data-pipeline.yaml

name: Peskas Timor Data Pipeline

on:
  push:
  schedule:
    - cron:  '0 0 */2 * *'

env:
  KOBO_PASSWORD: ${{ secrets.KOBO_PASSWORD }}
  KOBO_USERNAME: ${{ secrets.KOBO_USERNAME }}
  KOBO_PESKAS1: ${{ secrets.KOBO_PESKAS1 }}
  KOBO_PESKAS2: ${{ secrets.KOBO_PESKAS2 }}
  KOBO_PESKAS3: ${{ secrets.KOBO_PESKAS3 }}
  GCP_SA_KEY: ${{ secrets.PESKAS_DATAINGESTION_GCS_KEY }}
  AIRTABLE_KEY: ${{ secrets.AIRTABLE_KEY }}
  PDS_TOKEN: ${{ secrets.PESKAS_PDS_TOKEN }}
  PDS_SECRET: ${{ secrets.PESKAS_PDS_SECRET }}
  VALID_SHEET_ID: ${{ secrets.VALID_SHEET_ID }}
  GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }}

jobs:
  build-container:
    name: Build R container
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2
      - name: Build image with cache
        id: build-docker
        uses: whoan/docker-build-with-cache-action@v5
        with:
          username: $GITHUB_ACTOR
          password: "${{ secrets.GITHUB_TOKEN }}"
          registry: docker.pkg.github.com
          image_name: r-runner-peskas-timor
          dockerfile: Dockerfile.prod

  ingest-preprocess-metadata-tables:
    name: Ingest metadata tables
    needs: build-container
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call ingest_metadata_tables()
        run: Rscript -e 'peskas.timor.data.pipeline::ingest_metadata_tables()'
      - name: Call preprocess_metadata_tables()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_metadata_tables()'
      - name: Call ingest_rfish_table()
        continue-on-error: true
        run: Rscript -e 'peskas.timor.data.pipeline::ingest_rfish_table()'

  ingest-preprocess-landings-v1-v3:
    name: Ingest and preprocess legacy and ongoing landings
    needs: build-container
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call ingest_landings()
        run: Rscript -e 'peskas.timor.data.pipeline::ingest_landings_v1v3()'
      - name: Call preprocess_legacy_landings()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_legacy_landings()'
      - name: Call preprocess_updated_landings()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_updated_landings()'

  ingest-preprocess-v2-landings-step1:
    name: "Ingest-process landings 1/2"
    needs: build-container
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call ingest_landings_v2()
        run: Rscript -e 'peskas.timor.data.pipeline::ingest_landings_v2()'
      - name: Call preprocess_landings_step_1()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_landings_step_1()'

  ingest-preprocess-v2-landings-step2:
    name: "Ingest-process landings 2/2"
    needs: ingest-preprocess-v2-landings-step1
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call preprocess_landings_step_2()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_landings_step_2()'

  merge-landings:
    name: Merge landings
    needs: [ingest-preprocess-v2-landings-step2, ingest-preprocess-landings-v1-v3, ingest-preprocess-metadata-tables]
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call merge_landings()
        run: Rscript -e 'peskas.timor.data.pipeline::merge_landings()'
      - name: Call calculate_weights()
        run: Rscript -e 'peskas.timor.data.pipeline::calculate_weights()'

  validate-landings:
    name: Validate landings
    needs: [merge-landings]
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Validate landings
        run: Rscript -e 'peskas.timor.data.pipeline::validate_landings()'
      - name: Test validated landings
        run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_validated_landings.R", package = "peskas.timor.data.pipeline"))'

  ingest-pds-data:
    name: Ingest pds data
    needs: build-container
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call ingest_pds_trips()
        run: Rscript -e 'peskas.timor.data.pipeline::ingest_pds_trips()'
      - name: Call ingest_pds_tracks()
        run: Rscript -e 'peskas.timor.data.pipeline::ingest_pds_tracks()'

  preprocess-pds-data:
    name: Preprocess pds data
    needs: [ingest-pds-data]
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call preprocess_pds_trips()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_pds_trips()'
      - name: Call preprocess_pds_tracks()
        run: Rscript -e 'peskas.timor.data.pipeline::preprocess_pds_tracks()'

  validate-pds-data:
    name: Validate pds data
    needs: preprocess-pds-data
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call validate_pds_trips()
        run: Rscript -e 'peskas.timor.data.pipeline::validate_pds_trips()'
      - name: Test validated PDS trips
        run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_validated_pds_trips.R", package = "peskas.timor.data.pipeline"))'

  merge-trips:
    name: Merge trips
    needs: [validate-landings, validate-pds-data]
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call merge_trips()
        run: Rscript -e 'peskas.timor.data.pipeline::merge_trips()'
      - name: Test merged_trips
        run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_merged_trips.R", package = "peskas.timor.data.pipeline"))'

  model-indicators:
    name: Estimate fishery indicators
    needs: [merge-trips]
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call estimate_fishery_indicators()
        run: Rscript -e 'peskas.timor.data.pipeline::estimate_fishery_indicators()'

  export-trips:
    name: Export trips
    needs: [merge-trips, model-indicators]
    runs-on: ubuntu-20.04
    container:
      image: docker.pkg.github.com/worldfishcenter/peskas.timor.data.pipeline/r-runner-peskas-timor
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Set env to production
        if: endsWith(github.ref, '/main')
        run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV
      - name: Call format_public_data()
        run: Rscript -e 'peskas.timor.data.pipeline::format_public_data()'
      - name: Test public data
        run: Rscript -e 'tinytest::run_test_file(system.file("tinytest/test_public_data.R", package = "peskas.timor.data.pipeline"))'
      - name: Generate and upload enumerators report
        run: Rscript -e 'source(system.file("report/generate_enumerators_report.R", package = "peskas.timor.data.pipeline"))'