adds cwl-wrapper and stage

.github/workflows/build.yaml

@@ -44,7 +44,7 @@ jobs:
  run: echo "::set-output name=version::$APP_VERSION"
 
  container-crop:
- 
+
  needs: version
 
  runs-on: ubuntu-latest
@@ -95,7 +95,7 @@ jobs:
  docker push $IMAGE_ID:${{needs.version.outputs.app-version}}
 
  container-stac:
- 
+
  needs: version
 
  runs-on: ubuntu-latest
@@ -110,7 +110,40 @@ jobs:
  docker build water-bodies/command-line-tools/stac --file water-bodies/command-line-tools/stac/Dockerfile --tag stac
  docker tag stac $IMAGE_ID:${{needs.version.outputs.app-version}}
  docker push $IMAGE_ID:${{needs.version.outputs.app-version}}
+ 
+ container-stage:
+
+ needs: version
+
+ runs-on: ubuntu-latest
 
+ steps:
+ - uses: actions/checkout@v2
+ - run: echo version ${{needs.version.outputs.app-version}}
+ - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+ - name: build & push stage image
+ run: |
+ IMAGE_ID=ghcr.io/terradue/ogc-eo-application-package-hands-on/stage
+ docker build . --file containers/stage/Dockerfile --tag stage
+ docker tag stage $IMAGE_ID:${{needs.version.outputs.app-version}}
+ docker push $IMAGE_ID:${{needs.version.outputs.app-version}}
+
+ container-cwl-wrapper:
+
+ needs: version
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ - run: echo version ${{needs.version.outputs.app-version}}
+ - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+ - name: build & push cwl-runner image
+ run: |
+ IMAGE_ID=ghcr.io/terradue/ogc-eo-application-package-hands-on/cwl-wrapper
+ docker build . --file containers/cwl-wrapper/Dockerfile --tag cwl-wrapper
+ docker tag cwl-wrapper $IMAGE_ID:${{needs.version.outputs.app-version}}
+ docker push $IMAGE_ID:${{needs.version.outputs.app-version}}
 
  publish:
 
@@ -133,9 +166,13 @@ jobs:
  - run: yq -i eval '(.$graph[] | select (.id == "norm_diff") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/norm_diff:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl
  - run: yq -i eval '(.$graph[] | select (.id == "otsu") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/otsu:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl
  - run: yq -i eval '(.$graph[] | select (.id == "stac") ).hints.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/stac:${{needs.version.outputs.app-version}}"' water-bodies/app-package-cloud-native.cwl
+ - run: yq -i eval '(.requirements.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/stage:${{needs.version.outputs.app-version}}"' stage/stage-in.yaml 
+ - run: yq -i eval '(.requirements.DockerRequirement.dockerPull = "ghcr.io/terradue/ogc-eo-application-package-hands-on/stage:${{needs.version.outputs.app-version}}"' stage/stage-out.yaml
  - run: mkdir downloads
  - run: cp water-bodies/app-package.cwl downloads/app-water-bodies.${{needs.version.outputs.app-version}}.cwl
  - run: cp water-bodies/app-package-cloud-native.cwl downloads/app-water-bodies-cloud-native.${{needs.version.outputs.app-version}}.cwl
+ - run: cp stage/stage-in.yaml downloads/stage-in.yaml
+ - run: cp stage/stage-out.yaml downloads/stage-out.yaml
  - run: ls downloads/app-water-bodies.${{needs.version.outputs.app-version}}.cwl
  - uses: actions/upload-artifact@v2
  with: 

containers/cwl-wrapper/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker.io/eoepca/cwl-wrapper@sha256:066566ab6dcee685bb95d4eeab20ef07eefc5aaaf320857a251a3a608c81e694
+
+COPY cwl-wrapper/templates /templates
+
+COPY cwl-wrapper/default.conf /home/jovyan/.cwlwrapper/default.conf
+
+ENTRYPOINT [ "cwl-wrapper" ]

containers/cwl-wrapper/README.md

@@ -0,0 +1,4 @@
+
+docker run --rm -i cwl-wrapper:latest $PWD:/app-package eoepca/
+
+docker run --rm -i -v $PWD:/app cwl-wrapper:latest --stagein /app/stage/stage-in.yaml --stageout /app/stage/stage-out.yaml app/app-water-bodies.1.3.1.cwl

containers/cwl-wrapper/default.conf

@@ -0,0 +1,2 @@
+maincwl="/templates/main.yaml"
+rulez="/templates/rules.yaml"

containers/cwl-wrapper/templates/main.yaml

@@ -0,0 +1,11 @@
+class: Workflow
+doc: Main stage manager
+id: main
+label: macro-cwl
+inputs: {}
+outputs: {}
+
+requirements:
+ SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ InlineJavascriptRequirement: {}

containers/cwl-wrapper/templates/rules.yaml

@@ -0,0 +1,79 @@
+rulez:
+ version: 1
+
+parser:
+ type: $graph
+ driver: cwl
+
+onstage:
+ driver: cwl
+
+ stage_in:
+ connection_node: node_stage_in
+ if_scatter:
+ scatterMethod: dotproduct
+ input:
+ template:
+ overwrite: True
+
+ on_stage:
+ connection_node: on_stage
+
+ stage_out:
+ connection_node: node_stage_out
+ scatter: False
+ if_scatter:
+ scatterMethod: dotproduct
+ follow_node: node_metrics_out
+
+
+output:
+ driver: cwl
+ name: '-'
+ type: $graph
+
+
+cwl:
+ GlobalInput:
+ Directory: string
+ Directory[]: string[]
+
+ OptionalInput:
+ Directory: string?
+ Directory[]: string[]?
+
+ stage_in:
+ Directory?:
+ type: string?
+
+ Directory:
+ type: string?
+
+ Directory[]:
+ type: string[]
+
+ stage_out:
+ Directory:
+ type: Directory
+
+ Directory[]:
+ type: Directory[]
+
+ outputBindingResult:
+ command:
+ Directory:
+ outputBinding:
+ glob: .
+ type: Directory
+ Directory[]:
+ outputBinding:
+ glob: .
+ type: Directory[]
+ Directory?:
+ outputBinding:
+ glob: ${ if (inputs.input == null) {return null } else {return ".";} }
+ type: Directory?
+ stepOut:
+ type:
+ items: Directory
+ type: array

containers/cwl-wrapper/templates/stage-in.yaml_

@@ -0,0 +1,52 @@
+cwlVersion: v1.0
+doc: "Run Stars for staging input data"
+class: CommandLineTool
+hints:
+ DockerRequirement:
+ dockerPull: ghcr.io/terradue/stars:2.14.5
+ "cwltool:Secrets":
+ secrets: []
+
+id: stars
+requirements:
+ EnvVarRequirement:
+ envDef:
+ PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ ResourceRequirement: {}
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: stagein.sh 
+ entry: |- 
+ #!/bin/bash
+ set -x 
+ res=0
+ input='$( inputs.input )'
+ usersettings='$( inputs.usersettings.path )'
+
+ [ "\${input}" != "null" ] && {
+
+ IFS='#' read -r -a reference <<< '$( inputs.input )'
+ input_len=\${#reference[@]}
+
+ [[ \${input_len} == 2 ]] && {
+ IFS=',' read -r -a assets <<< \${reference[1]}
+ af=" "
+ for asset in \${assets[@]}
+ do 
+ af="\${af} -af \${asset}"
+ done
+ } || {
+ af="--empty"
+ }
+ Stars copy -conf \${usersettings} -v -rel -r '4' \${af} -o ./ \${reference[0]}
+ res=$?
+ }
+ rm -fr stagein.sh
+ exit 0
+
+baseCommand: ['/bin/bash', 'stagein.sh']
+
+inputs: {}
+
+outputs: {}

containers/stage/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.io/python:3.9.18-slim-bullseye
+
+RUN pip install --no-cache-dir --upgrade pip stac-asset boto3

stage/stage-in.yaml

@@ -0,0 +1,65 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+
+class: CommandLineTool
+id: stage
+
+inputs: {}
+
+outputs: {}
+
+baseCommand: 
+- python
+- stage.py
+arguments:
+- $( inputs.input )
+
+requirements:
+ DockerRequirement:
+ dockerPull: stac-asset:latest
+ EnvVarRequirement:
+ envDef:
+ A: "2"
+ InlineJavascriptRequirement: {}
+ InitialWorkDirRequirement:
+ listing:
+ - entryname: stage.py
+ entry: |-
+ import pystac
+ import stac_asset
+ import asyncio
+ import os
+ import sys
+
+ config = stac_asset.Config(warn=True)
+
+ async def main(href: str):
+
+ item = pystac.read_file(href)
+
+ os.makedirs(item.id, exist_ok=True)
+ cwd = os.getcwd()
+
+ os.chdir(item.id)
+ item = await stac_asset.download_item(item=item, directory=".", config=config)
+ os.chdir(cwd)
+
+ cat = pystac.Catalog(
+ id="catalog",
+ description=f"catalog with staged {item.id}",
+ title=f"catalog with staged {item.id}",
+ )
+ cat.add_item(item)
+
+ cat.normalize_hrefs("./")
+ cat.save(catalog_type=pystac.CatalogType.SELF_CONTAINED)
+
+ return cat
+
+ href = sys.argv[1]
+
+ cat = asyncio.run(main(href))
+
+
+