SWOOP workflow-configuration chart (#103)

* SWOOP workflow configuration helm chart

charts/workflow-config/Chart.yaml

@@ -0,0 +1,7 @@
+name: workflow-config
+description: A Helm Chart for loading the workflow configuration and templates
+version: 0.1.0
+apiVersion: v2
+appVersion: "0.1.0"
+keywords:
+ - workflow-config

charts/workflow-config/README.md

@@ -0,0 +1,37 @@
+# Deployment
+
+This chart deploys the SWOOP configuration file and workflow templates onto a K8s cluster.
+
+The SWOOP configuration file contains the high-level configuration parameters and values used by SWOOP components such as workflows, callbacks, conductors, and handlers, to run workflows on the cluster.
+
+The workflow templates contain more detailed configurations used by each individual workflow to run, and describe their tasks, input and output artifacts, any additional workflow templates referenced by the main workflow (if any), and secrets being used.
+
+The workflow-config helm chart uses Argo Workflows since the WorkflowTemplate resources are Argo-specific configurations. Therefore, install the `swoop-caboose` helm chart first, which will also install Argo Workflows as a dependency.
+
+To add the FilmDrop Helm Chart repository, do:
+
+`helm repo add e84 https://element84.github.io/filmdrop-k8s-helm-charts`
+
+First, follow the stepwise instructions under [Deployment](https://github.com/Element84/filmdrop-k8s-helm-charts/blob/main/charts/swoop-caboose/README.md#deployment), **until and including** the following command:
+
+```
+mc cp ./payload_workflow.json swoopminio/payloadtest/
+```
+
+Do not create any further resources such as secrets or submit any workflows.
+
+Then, modify the `values.yaml` file with the `AWS Access Key ID`, `AWS Secret Access Key`, and `AWS Region` under the keys `s3.accessKeyId`, `s3.secretAccessKey`, `s3.region`, respectively. These values contain dummy data and should be modified to include your AWS IAM user credentials. Make sure you are using base-64 encoding for these values. Also, make sure that the correct values are provided for `minio.host`, `minio.port`, and `minio.bucket` as well. If you're running this example from a minio created by the MinIO helm chart, you should be able to use `minio.default` for the host and `9000` for the port.
+
+Then, do:
+
+`helm install workflow-config e84/workflow-config`
+
+Once the chart has been deployed, you can do:
+
+`kubectl get workflowtemplate` and
+
+`kubectl get configmap`
+
+to see that the workflow templates and SWOOP configmap were deployed.
+
+This will deploy the Swoop configuration file as a configmap and the copy-assets workflow as an Argo Workflow onto the cluster. The workflow will run to completion and you should then see the output thumbnail asset in the bucket that you created in AWS S3.

charts/workflow-config/swoop-config.yaml

@@ -0,0 +1,129 @@
+handlers:
+ argoHandler:
+ type: argoWorkflowHandler
+ #serverUrl: https://kubernetes:6443
+ #token: {secrets.argo_token}
+
+ cirrusHandler:
+ type: cirrusWorkflowHandler
+ #sqsUrl: https://sqs.aws.com/0142354653636::sqs::queue
+
+ noopHandler:
+ type: noop
+ parameters:
+ workflowUuid:
+ type: string
+ format: uuid
+
+ # We believe it is best to re-template the request
+ # (at least with regard to secrets) JIT for every request execution
+ publishS3Handler:
+ type: httpHandler
+ url: https://{secrets.minio-user}:{secrets.minio-password}@our-minio:9000
+ requestBody: |
+ {
+ "fixed": "a_value",
+ "name": "{{ .parameters.workflowName -}}",
+ "date": "{{ .parameters.feature.properties.datetime -}}"
+ }
+ operation: POST
+ secrets:
+ # simplest approach for secret renewal is to have a ttl
+ # and periodically re-resolve the secret values, but does mean
+ # invalid secrets may be used for some time until the ttl expires
+ #
+ # how to handle secrets that need to resolve a token,
+ # have some other initialization step, or are of some built-in
+ # limited duration?
+ - name: user
+ type: file
+ path: /secrets-mount/username-secret
+ ttl: 1200
+ headers:
+ Authorization: "Basic {{ .secrets.user }} {{ .secrets.password}}"
+ Content-Type: "application/json"
+ X-Workflow-Name: "{{ .parameters.workflowName }}"
+ backoff:
+ retries: 10
+ seconds: 5
+ factor: 2
+ max: 25
+ errors:
+ - status: 400
+ retryable: false
+ - status: 400
+ message: ".*timed out.*"
+ retryable: true
+ parameters:
+ workflowName:
+ type: string
+ default: a_value
+ feature:
+ type: object
+
+conductors:
+ instance-a:
+ handlers:
+ - argoHandler
+ - publishS3Handler
+
+callbacks:
+ publishS3Push: &callbacksPublishS3Push
+ handler: publishS3Handler
+ type: perFeature
+ when:
+ - "successful"
+ featureFilter: "@.id =~ 'fake*' & @.properties.gsd <= 0"
+ parameters:
+ workflowName:
+ path: .input.features[0].id
+ feature:
+ value: { "feature": true }
+ enabled: true
+
+workflows:
+ mirror:
+ description: "A workflow to copy STAC items into a local mirror"
+ version: 2
+ handler: argoHandler
+ argoTemplate: workflowtemplate/mirror-workflow
+ cacheKeyHashIncludes:
+ - .features[].id
+ - .features[].collection
+ cacheKeyHashExcludes: []
+ callbacks:
+ publishS3Push:
+ <<: *callbacksPublishS3Push
+ failed:
+ handler: noopHandler
+ type: single
+ when:
+ - "!successful"
+ parameters:
+ workflowUuid:
+ path: .workflow.uuid
+ cirrus-example:
+ callbacks:
+ publishS3Push:
+ <<: *callbacksPublishS3Push
+ when:
+ - "!failed"
+ - "successful"
+ title: "Cirrus example workflow"
+ description: "An example workflow config for a cirrus workflow"
+ version: 1
+ handler: cirrusHandler
+ sfnArn: arn:aws:states:us-west-2:09876543210:stateMachine:cirrus-example
+ cacheKeyHashIncludes:
+ - .features[].id
+ - .features[].collection
+ cacheKeyHashExcludes: []
+ links:
+ - href: https://example.com/repo
+ rel: external
+ type: text/html
+ title: "source repository"
+ - href: https://example.com/docs
+ rel: external
+ type: text/html
+ title: "process documentation"

charts/workflow-config/templates/NOTES.txt

@@ -0,0 +1,8 @@
+Thank you for installing {{ .Chart.Name }}.
+
+Your release is named {{ .Release.Name }}.
+
+To learn more about the release, try:
+
+ $ helm status {{ .Release.Name }}
+ $ helm get all {{ .Release.Name }}

charts/workflow-config/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "workflow-config.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "workflow-config.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "workflow-config.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "workflow-config.labels" -}}
+helm.sh/chart: {{ include "workflow-config.chart" . }}
+{{ include "workflow-config.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "workflow-config.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "workflow-config.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "workflow-config.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "workflow-config.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

charts/workflow-config/templates/configmap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ .Release.Name }}-configmap
+data:
+ swoop-config.yaml: |-
+{{ .Files.Get "swoop-config.yaml" | indent 4}}

charts/workflow-config/templates/copy-stac-asset-s3-secret.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.s3.createSecret -}}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: copy-stac-asset-s3-secret
+type: Opaque
+data:
+ access_key_id: {{ .Values.s3.accessKeyId }}
+ secret_access_key: {{ .Values.s3.secretAccessKey }}
+ region: {{ .Values.s3.region }}
+{{- end }}

charts/workflow-config/templates/workflows/copy-assets-workflow.yaml

@@ -0,0 +1,46 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+ name: copy-assets
+spec:
+# Start of variable mappings
+{{- $accessKeyIdSecretName := .Values.s3.accessKeyIdSecret.name -}}
+{{- $accessKeyIdSecretKey := .Values.s3.accessKeyIdSecret.key -}}
+{{- $secretAccessKeySecretName := .Values.s3.secretAccessKeySecret.name -}}
+{{- $secretAccessKeySecretKey := .Values.s3.secretAccessKeySecret.key -}}
+{{- $regionSecretName := .Values.s3.regionSecret.name -}}
+{{- $regionSecretKey := .Values.s3.regionSecret.key -}}
+# End of global variable mappings
+ entrypoint: copy-assets
+ serviceAccountName: argo # set this so the proper permissions are assigned
+ arguments:
+ parameters:
+ - name: input
+ value: http://{{ .Values.minio.host }}:{{ .Values.minio.port }}/{{ .Values.minio.bucket }}/payload_workflow.json
+
+ templates:
+ - name: copy-assets
+ inputs:
+ parameters:
+ - name: input # parameter declaration
+ container:
+ env:
+ - name: AWS_ACCESS_KEY_ID
+ valueFrom:
+ secretKeyRef:
+ name: {{ $accessKeyIdSecretName }}
+ key: {{ $accessKeyIdSecretKey }}
+ - name: AWS_SECRET_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ name: {{ $secretAccessKeySecretName }}
+ key: {{ $secretAccessKeySecretKey }}
+ - name: AWS_DEFAULT_REGION
+ valueFrom:
+ secretKeyRef:
+ name: {{ $regionSecretName }}
+ key: {{ $regionSecretKey }}
+ image: quay.io/element84/copy-assets-stac-task:latest
+ imagePullPolicy: IfNotPresent
+ command: ["python3", "./task.py", "run"]
+ args: [ {{ `"{{inputs.parameters.input}}"` }} ]

charts/workflow-config/values.yaml

@@ -0,0 +1,22 @@
+nameOverride: ""
+fullnameOverride: ""
+
+s3:
+ createSecret: true
+ accessKeyId: YWNjZXNza2V5aWQ=
+ secretAccessKey: c2VjcmV0YWNjZXNza2V5
+ region: cmVnaW9u
+ accessKeyIdSecret:
+ name: copy-stac-asset-s3-secret
+ key: access_key_id
+ secretAccessKeySecret:
+ name: copy-stac-asset-s3-secret
+ key: secret_access_key
+ regionSecret:
+ name: copy-stac-asset-s3-secret
+ key: region
+
+minio:
+ host: minio.default
+ port: 9000
+ bucket: payloadtest