simple-impala-part.conf

application {
  name = Simple Impala Partitioned HDFS Example
}

include file("env.conf")

steps {
  loop_dates {
    type = loop
    mode = serial
    source = range
    range.start = 20190101
    range.end = 20190102
    parameter = processing_date
  }
  fsInput {
    dependencies = [loop_dates]
    input {
      type = filesystem
      path = "/tmp/example-input/example-input-${processing_date}.json"
      format = json
    }
  }
  fsProcess {
    dependencies = [fsInput, loop_dates]
    deriver {
      type = sql
      query.literal = "SELECT id,foo,blah,ymd FROM fsInput_${processing_date}"
    }
    planner = {
      type = overwrite
    }
    output = {
      type = filesystem
      path = "/tmp/example-output-partitioned/ymd=${processing_date}"
      format = parquet
    }
  }
  ddl {
    dependencies = [fsProcess, loop_dates]
    type = task
    task = ${env.impala} {
      type = impala_ddl
      query {
        type = "add_partition"
        table = "example_output_part"
        partition {
          spec = "ymd=${processing_date}"
        }
      }
    }
  }
}