Skip to content

[TLM] Update max_tokens for TLM to match provider limits #71

[TLM] Update max_tokens for TLM to match provider limits

[TLM] Update max_tokens for TLM to match provider limits #71

Workflow file for this run

name: /test-tlm CI
on:
issue_comment:
types: [created]
jobs:
test:
name: "TLM Test: Python ${{ matrix.python }} on ${{ matrix.os }}"
runs-on: ${{ matrix.os }}
if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm"
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
python:
- "3.8"
- "3.9"
- "3.10"
- "3.11"
- "3.12"
steps:
- name: Checkout latest commit to pull request
uses: actions/checkout@v3
with:
ref: refs/pull/${{ github.event.issue.number }}/head
- name: Set up Python ${{ matrix.python }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python }}
- name: Install dependencies
run: |
pip install --upgrade pip
pip install .
pip install -r tests/requirements_test.txt
- name: Install Cleanlab Studio client
run: pip install -e .
- name: Set env variables (Linux/macOS/Windows)
run: |
echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV
shell: bash
- name: Cleanlab login
run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}"
- name: Run tests
run: |
pytest tests/tlm/ --ignore=tests/tlm/test_properties.py --verbose
propertytest:
name: "TLM Property Test: Python ${{ matrix.python }} on ${{ matrix.os }}"
runs-on: ${{ matrix.os }}
if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm"
strategy:
matrix:
os:
- macos-latest
python:
- "3.11"
steps:
- name: Checkout latest commit to pull request
uses: actions/checkout@v3
with:
ref: refs/pull/${{ github.event.issue.number }}/head
- name: Set up Python ${{ matrix.python }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python }}
- name: Install dependencies
run: |
pip install --upgrade pip
pip install .
pip install -r tests/requirements_test.txt
- name: Restore pytest cache
uses: actions/cache@v2
with:
path: .pytest_cache
key: pytest-cache-${{ github.run_id }}
restore-keys: pytest-cache-
- name: Install Cleanlab Studio client
run: pip install -e .
- name: Set env variables (Linux/macOS/Windows)
run: |
echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV
shell: bash
- name: Cleanlab login
run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}"
- name: Run tests
run: |
pytest -n auto tests/tlm/test_properties.py --verbose
- name: Cache pytest results
uses: actions/cache@v2
with:
path: .pytest_cache
key: pytest-cache-${{ github.run_id }}
launch-tlm-test-notif:
name: Comment TLM test start on PR
runs-on: ubuntu-latest
if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm"
steps:
- name: Log PR Number and Pull request ref
run: echo "PR Number is ${{ github.event.issue.number }}. PR ref is refs/pull/${{ github.event.issue.number }}/head."
- name: Find Comment
id: fc
uses: peter-evans/find-comment@v3
with:
issue-number: ${{ github.event.issue.number }}
body-regex: '^/test-tlm$'
direction: last
- name: Update comment
uses: peter-evans/create-or-update-comment@v4
if: steps.fc.outputs.comment-id != ''
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
body: |
:sparkles: **Starting TLM tests...** :sparkles:
If you want to run all the TLM tests again (because TLM code is ready for review), comment '/test-tlm' on this PR.
If you want to re-run only the failed tests (you are still developing), comment '/rerun-failed-test-tlm' on this PR.
[View full GitHub Actions run log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
reactions: rocket
finish-tlm-test-notif:
name: Comment TLM test completion on PR
runs-on: ubuntu-latest
needs:
- launch-tlm-test-notif
- test
- propertytest
if: always() && github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" but still run if needs failed
continue-on-error: true
steps:
- name: Find Comment
id: fc
uses: peter-evans/find-comment@v3
with:
issue-number: ${{ github.event.issue.number }}
body-includes: Starting TLM tests
direction: last
- name: Build Comment Body
run: |
# Set up emojis based on test results
if [[ "${{ needs.test.result }}" == "success" ]]; then
TLM_TEST_RESULT="✅"
else
TLM_TEST_RESULT="❌"
fi
if [[ "${{ needs.propertytest.result }}" == "success" ]]; then
PROPERTY_TEST_RESULT="✅"
else
PROPERTY_TEST_RESULT="❌"
fi
echo "PROPERTY_TEST_RESULT=$PROPERTY_TEST_RESULT" >> $GITHUB_ENV
echo "TLM_TEST_RESULT=$TLM_TEST_RESULT" >> $GITHUB_ENV
- name: Update comment
if: steps.fc.outputs.comment-id != ''
uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
body: |
:sparkles: **Tests completed!** :sparkles:
TLM Tests Results: ${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}
TLM Property Tests Results: ${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}
Click the Github Actions run log for more information.