Upgrade to 0.10.9, add in example plan using ODCS

.github/workflows/benchmark.yml

@@ -1,6 +1,4 @@
 name: Run performance benchmark tests
-permissions:
- pull-requests: write
 
 on:
  push:

gradle.properties

@@ -8,6 +8,6 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.15
-dataCatererVersion=0.10.8
+dataCatererVersion=0.10.9
 sparkVersion=3.5.1
 sparkMajorVersion=3.5

run.sh

@@ -14,7 +14,11 @@ else
  curr_class_name=$1
 fi
 
-full_class_name=io.github.datacatering.plan.$curr_class_name
+if [[ $curr_class_name == *".yaml"* ]]; then
+ full_class_name="PLAN_FILE_PATH=/opt/app/custom/plan/$curr_class_name"
+else
+ full_class_name="PLAN_CLASS=io.github.datacatering.plan.$curr_class_name"
+fi
 echo -n "$curr_class_name" > .tmp_prev_class_name
 
 image_suffix="-basic"
@@ -44,7 +48,9 @@ DOCKER_CMD=(
  -v "$(pwd)/docker/sample:/opt/app/data"
  -v "$(pwd)/docker/sample/tracking:/opt/app/record-tracking"
  -v "$(pwd)/docker/mount:/opt/app/mount"
- -e "PLAN_CLASS=$full_class_name"
+ -v "$(pwd)/docker/data/custom:/opt/app/custom"
+ -e "APPLICATION_CONFIG_PATH=/opt/app/custom/application.conf"
+ -e "$full_class_name"
  -e "DEPLOY_MODE=client"
  -e "DRIVER_MEMORY=2g"
  -e "EXECUTOR_MEMORY=2g"
@@ -54,4 +60,8 @@ DOCKER_CMD=(
 )
 
 eval "${DOCKER_CMD[@]}"
+if [[ $? != 0 ]]; then
+ echo "Failed to run"
+ exit 1
+fi
 echo "Finished!"

src/main/java/io/github/datacatering/plan/AdvancedODCSJavaPlanRun.java

@@ -0,0 +1,19 @@
+package io.github.datacatering.plan;
+
+import io.github.datacatering.datacaterer.javaapi.api.PlanRun;
+
+import java.util.Map;
+
+public class AdvancedODCSJavaPlanRun extends PlanRun {
+ {
+ var accountTask = csv("my_csv", "/opt/app/data/account-odcs", Map.of("saveMode", "overwrite", "header", "true"))
+ .schema(metadataSource().openDataContractStandard("/opt/app/mount/odcs/full-example.yaml"))
+ .count(count().records(100));
+
+ var conf = configuration().enableGeneratePlanAndTasks(true)
+ .enableGenerateValidations(true)
+ .generatedReportsFolderPath("/opt/app/data/report");
+
+ execute(conf, accountTask);
+ }
+}

src/main/scala/io/github/datacatering/plan/AdvancedODCSPlanRun.scala

@@ -0,0 +1,15 @@
+package io.github.datacatering.plan
+
+import io.github.datacatering.datacaterer.api.PlanRun
+
+class AdvancedODCSPlanRun extends PlanRun {
+
+ val accountTask = csv("customer_accounts", "/opt/app/data/customer/account-odcs", Map("header" -> "true"))
+ .schema(metadataSource.openDataContractStandard("/opt/app/mount/odcs/full-example.yaml"))
+ .count(count.records(100))
+
+ val conf = configuration.enableGeneratePlanAndTasks(true)
+ .generatedReportsFolderPath("/opt/app/data/report")
+
+ execute(conf, accountTask)
+}