sample_workflow.json.mustache

{
    "name": "test workflow",
    "resources": [
        {
            "id": "1",
            "name": "emr cluster",
            "resourceType": "aws.resource.EmrResource",
            "resourceSpec": {
                "releaseLabel": "emr-6.3.0",
                "applications": ["Spark"],
                "serviceRole": "EMR_DefaultRole",
                "resourceRole": "EMR_EC2_DefaultRole",
                "instancesConfig": {
                    "subnetId": "{{resources.subnetId}}",
                    "ec2KeyName": "orchard",
                    "instanceGroupConfigs": [
                        {
                            "instanceRoleType": "MASTER",
                            "instanceCount": 1,
                            "instanceType": "m5.xlarge"
                        },
                        {
                            "instanceRoleType": "CORE",
                            "instanceCount": 1,
                            "instanceType": "m5.xlarge"
                        }
                    ]
                },
                "sparkConfigs": [
                    {
                        "classification": "spark",
                        "properties": {
                            "maximizeResourceAllocation": "true"
                        }
                    },
                    {
                        "classification": "spark-defaults",
                        "properties": {
                            "spark.sql.defaultUrlStreamHandlerFactory.enabled": "false",
                            "spark.dynamicAllocation.enabled": "false"
                        }
                    },
                    {
                        "classification": "spark-env",
                        "configurations": [
                          {
                            "classification": "export",
                            "properties": {
                              "{{sparkConfig.env_key}}": "{{sparkConfig.env_val}}"
                            }
                          }
                        ]
                    }
                ]
            },
            "maxAttempt": 3
        }
    ],
    "activities": [
        {
            "id": "1",
            "name": "activity 1",
            "activityType": "aws.activity.EmrActivity",
            "activitySpec": {
                "steps": [
                    {
                        "jar": "command-runner.jar",
                        "args": [
                            "spark-submit",
                            "s3://{{s3bucket}}/managed/health_violations.py",
                            "--data_source",
                            "s3://{{s3bucket}}/data/food_establishment_data.csv",
                            "--output_uri",
                            "s3://{{s3bucket}}/data/output"
                        ]
                    }
                ]
            },
            "resourceId": "1",
            "maxAttempt": 3
        },
        {
            "id": "2",
            "name": "activity 2",
            "activityType": "aws.activity.EmrActivity",
            "activitySpec": {
                "steps": [
                    {
                        "jar": "command-runner.jar",
                        "args": [
                            "spark-submit",
                            "s3://{{s3bucket}}/managed/health_violations.py",
                            "--data_source",
                            "s3://{{s3bucket}}/data/food_establishment_data.csv",
                            "--output_uri",
                            "s3://{{s3bucket}}/data/output"
                        ]
                    }
                ]
            },
            "resourceId": "1",
            "onFailure": ["1"],
            "maxAttempt": 3
        }
    ],
    "actions": [
        {
            "id": "1",
            "name": "action 1",
            "actionType": "aws.action.SnsAction",
            "actionSpec": {
                "topicArn": "{{snsArn}}",
                "subject": "hi - subject",
                "message": "hi - message"
            }
        }
    ],
    "dependencies": {
        "2": ["1"]
    }
}