data-pipeline.yaml

name: Peskas Malawi Data Pipeline

on:
  push:
  schedule:
    - cron: '0 0 */2 * *'

env:
  MALAWI_SSF: ${{ secrets.MALAWI_SSF }}
  FISHERIES_ECAS: ${{ secrets.FISHERIES_ECAS }}
  FIELDDATAAPP_2024: ${{ secrets.FIELDDATAAPP_2024 }}
  FIELDDATAAPP_2024A: ${{ secrets.FIELDDATAAPP_2024A }}
  FIELDDATAAPP_2023F: ${{ secrets.FIELDDATAAPP_2023F }}
  KOBO_USERNAME: ${{ secrets.KOBO_USERNAME }}
  KOBO_PASSWORD: ${{ secrets.KOBO_PASSWORD }}
  PDS_TOKEN: ${{ secrets.PDS_TOKEN }}
  PDS_SECRET: ${{ secrets.PDS_SECRET }}
  MONGODB_CONNECTION_STRING: ${{ secrets.MONGODB_CONNECTION_STRING }}
  GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
  GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }}

jobs:
  build-container:
    name: Build R container
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Build image with cache
        id: build-docker
        uses: whoan/docker-build-with-cache-action@v5
        with:
          username: $GITHUB_ACTOR
          password: "${{ secrets.GITHUB_TOKEN }}"
          registry: docker.pkg.github.com
          image_name: r-runner-peskas-malawi
          dockerfile: Dockerfile.prod

  ingest-landings:
    name: Ingest Malawi SSF landings
    needs: build-container
    runs-on: ubuntu-latest
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.malawi.data.pipeline/r-runner-peskas-malawi
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call ingest_landings()
        run: Rscript -e 'peskas.malawi.data.pipeline::ingest_landings()'

  preprocess-landings:
    name: Preprocess Malawi SSF landings
    needs: ingest-landings
    runs-on: ubuntu-latest
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.malawi.data.pipeline/r-runner-peskas-malawi
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call preprocess_landings()
        run: Rscript -e 'peskas.malawi.data.pipeline::preprocess_landings()'

  validate-landings:
    name: Validate Malawi SSF landings
    needs: preprocess-landings
    runs-on: ubuntu-latest
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.malawi.data.pipeline/r-runner-peskas-malawi
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Validate landings
        run: Rscript -e 'peskas.malawi.data.pipeline::validate_landings()'

  export-landings:
    name: Export Malawi SSF landings
    needs: preprocess-landings
    runs-on: ubuntu-latest
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.malawi.data.pipeline/r-runner-peskas-malawi
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Export landings
        run: Rscript -e 'peskas.malawi.data.pipeline::export_landings()'