eval_test.yaml

name: E2E Eval Tests

on:
  push:
    branches: [ main ]
  pull_request:

concurrency:
  group: eval-test-${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_number || github.ref }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: read

defaults:
  run:
    shell: bash -l -eo pipefail {0}

jobs:
  eval_test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11']
    steps:
      - name: Check out repo
        uses: actions/checkout@v3
      - name: Setup conda env
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          miniconda-version: "latest"
          activate-environment: test
          python-version: ${{ matrix.python-version }}
      - name: Update pip
        run: python -m pip install --upgrade pip
      - name: configure aws credentials
        id: aws_creds
        uses: aws-actions/configure-aws-credentials@v1.7.0
        with:
          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_torchtune_pytorch-multimodal
          aws-region: us-east-1
      - name: Download files from S3
        uses: nick-fields/retry@v2
        with:
          max_attempts: 3
          retry_on: error
          timeout_seconds: 14400
          command: |
            python3 -m pip install awscli==1.32.6
            mkdir -p /tmp/test-artifacts
            aws s3 cp s3://pytorch-multimodal/llama2-7b/tokenizer.model /tmp/test-artifacts
            aws s3 cp s3://pytorch-multimodal/small-ckpt-01242024 /tmp/test-artifacts
            aws s3 cp s3://pytorch-multimodal/small-ckpt-tune-03082024.pt /tmp/test-artifacts
      - name: Install dependencies
        run: |
          python -m pip install -r requirements.txt
          python -m pip install -r dev-requirements.txt
          python -m pip install -e .
      - name: Run unit tests with coverage
        run: pytest tests/evals --cov=. --cov-report=xml --durations=20 -vv
      - name: Upload Coverage to Codecov
        uses: codecov/codecov-action@v3