dataflow.yaml

name: Dataflow Integration Test

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]
    types: [ opened, reopened, synchronize, labeled ]
  schedule:
    - cron: '0 4 * * *' # run once a day at 4 AM

jobs:
  test-dataflow:
    # run on:
    #  - all pushes to main
    #  - schedule defined above
    #  - a PR was just labeled 'test-dataflow' or 'test-all'
    #  - a PR with 'test-dataflow' or 'test-all' label was opened, reopened, or synchronized
    if: |
      github.event_name == 'push' ||
      github.event_name == 'schedule' ||
      github.event.label.name == 'test-all' ||
      github.event.label.name == 'test-dataflow' ||
      contains( github.event.pull_request.labels.*.name, 'test-all') ||
      contains( github.event.pull_request.labels.*.name, 'test-dataflow')
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
        recipes-version: [
          "pangeo-forge-recipes==0.10.4",
        ]
        beam-version: [
          "apache-beam[gcp]==2.52.0",
        ]

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: 'Authenticate to Google Cloud'
      uses: 'google-github-actions/auth@v1'
      with:
        credentials_json: '${{ secrets.GCP_DATAFLOW_SERVICE_KEY }}'

    - name: Install dependencies & our package
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e .[test]

    - name: 'Run Dataflow Integration Test'
      run: |
        pytest -vvvxs tests/integration/test_dataflow_integration.py \
          --recipes-version=${{ matrix.recipes-version }} \
          --beam-version=${{ matrix.beam-version }}