regression_test.yaml

name: Regression Tests

on:
  schedule:
    # Runs at midnight every day
    - cron:  '0 0 * * *'

concurrency:
  group: regression-test-${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: read

defaults:
  run:
    shell: bash -l -eo pipefail {0}

jobs:
  regression_test:
    runs-on: linux.g5.12xlarge.nvidia.gpu
    strategy:
      matrix:
        python-version: ['3.11']
        torch-version: ["stable", "nightly"]
    steps:
      - name: Check out repo
        uses: actions/checkout@v3
      - name: Setup conda env
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          miniconda-version: "latest"
          activate-environment: test
          python-version: ${{ matrix.python-version }}
      - name: Update pip
        run: python -m pip install --upgrade pip
      - name: configure aws credentials
        id: aws_creds
        uses: aws-actions/configure-aws-credentials@v1.7.0
        with:
          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_torchtune_pytorch-multimodal
          aws-region: us-east-1
      - name: Install S3 CLI
        run: |
          python3 -m pip install awscli==1.32.6
      - name: Install torch nightly
        if: ${{ matrix.torch-version == 'nightly' }}
        run: python -m pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu118
      - name: Install torch stable
        if: ${{ matrix.torch-version == 'stable' }}
        run: python -m pip install torch
      - name: Install remaining dependencies
        run: |
          python -m pip install -e ".[dev]"
          python -m pip install lm-eval==0.4.*
      - name: Run regression tests with coverage
        run: pytest tests -m slow_integration_test --silence-s3-logs --cov=. --cov-report=xml --durations=20 -vv
      - name: Upload Coverage to Codecov
        uses: codecov/codecov-action@v3