Elyra CI #390
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright 2018-2023 Elyra Authors | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| name: Elyra CI | |
| on: | |
| push: # all branches | |
| pull_request: # all branches | |
| schedule: | |
| # once a day at 3 am (PST) (10 am (UTC)) | |
| - cron: '0 10 * * *' | |
| env: | |
| FORCE_COLOR: true | |
| NODE_VERSION: 20.11.0 | |
| YARN_VERSION: 3.5.0 | |
| PYTHON_VERSION: 3.11 | |
| QUAY_WORKBENCH_IMAGES_REPOSITORY: 'opendatahub/workbench-images' | |
| QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV: 'jupyter-trustyai-ubi9-python-3.11-20241,cuda-jupyter-tensorflow-ubi9-python-3.11-20241' | |
| jobs: | |
| prepare-yarn-cache: | |
| name: Prepare Cache | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| node_modules | |
| */*/node_modules | |
| /home/runner/.cache/Cypress | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| - name: Install Yarn ${{ env.YARN_VERSION }} | |
| run: | | |
| corepack prepare yarn@${{ env.YARN_VERSION }} --activate | |
| yarn set version ${{ env.YARN_VERSION }} | |
| yarn --version | |
| - name: Install dependencies | |
| run: | | |
| yarn install | |
| lint-server: | |
| name: Lint Server | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Lint | |
| run: make lint-server | |
| lint-ui: | |
| name: Lint UI | |
| needs: prepare-yarn-cache | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| node_modules | |
| */*/node_modules | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| - name: Install Yarn ${{ env.YARN_VERSION }} | |
| run: | | |
| corepack prepare yarn@${{ env.YARN_VERSION }} --activate | |
| yarn set version ${{ env.YARN_VERSION }} | |
| yarn --version | |
| - name: Install dependencies | |
| run: | | |
| make yarn-install | |
| - name: Lint | |
| run: make eslint-check-ui | |
| - name: Check format | |
| run: make prettier-check-ui | |
| test-server: | |
| name: Test Server | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: [3.8, 3.9, '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install | |
| # Note: Tests fail if we don't install `wheel` for some reason. | |
| run: | | |
| make install-server | |
| make test-dependencies | |
| - name: Version Snapshot | |
| run: pip freeze | |
| - name: Test | |
| id: Test | |
| run: make pytest | |
| - name: Fetch packages metadata (if Test step failed) | |
| # Note: checking for general failure() first is needed according to docs | |
| if: ${{ failure() && steps.Test.conclusion == 'failure' }} | |
| env: | |
| days: 7 # check for newer package versions in past X days | |
| run: | | |
| python3 ./.github/workflows/scripts/fetch_packages_metadata.py $days | |
| test-ui: | |
| name: Test UI | |
| needs: prepare-yarn-cache | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| node_modules | |
| */*/node_modules | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| - name: Install Yarn ${{ env.YARN_VERSION }} | |
| run: | | |
| corepack prepare yarn@${{ env.YARN_VERSION }} --activate | |
| yarn set version ${{ env.YARN_VERSION }} | |
| yarn --version | |
| - name: Build | |
| run: | | |
| make build-dependencies | |
| make yarn-install | |
| make build-ui-prod | |
| - name: Install | |
| run: make install-server | |
| - name: Test | |
| run: make test-ui-unit | |
| test-integration: | |
| name: Run Integration Tests | |
| needs: prepare-yarn-cache | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| node_modules | |
| */*/node_modules | |
| /home/runner/.cache/Cypress | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| - name: Install Yarn ${{ env.YARN_VERSION }} | |
| run: | | |
| corepack prepare yarn@${{ env.YARN_VERSION }} --activate | |
| yarn set version ${{ env.YARN_VERSION }} | |
| yarn --version | |
| - name: Build | |
| run: | | |
| make build-dependencies | |
| make yarn-install | |
| make build-ui-prod | |
| - name: Install | |
| run: | | |
| make install-server | |
| make install-examples | |
| - name: Cypress | |
| run: make test-integration | |
| - name: Collect logs | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: elyra_test_artifacts | |
| path: | | |
| ${{ github.workspace }}/build/cypress-tests/*.log | |
| ${{ github.workspace }}/build/cypress-tests/screenshots//**/* | |
| ${{ github.workspace }}/build/cypress-tests/videos//**/* | |
| /home/runner/.npm/_logs/*.log | |
| test-documentation-build: | |
| name: Test documentation build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Build | |
| run: make docs | |
| validate-image-env: | |
| name: Validate Image Environment | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: [3.8, 3.9, '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Validate image environment | |
| run: make PYTHON_VERSION=${{ matrix.python-version }} elyra-image-env | |
| validate-images: | |
| name: Validate Images | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Validate runtime images | |
| run: make REMOVE_RUNTIME_IMAGE=1 validate-runtime-images | |
| publish-artifacts: | |
| permissions: | |
| packages: write | |
| name: Publish Artifacts | |
| needs: prepare-yarn-cache | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| node_modules | |
| */*/node_modules | |
| /home/runner/.cache/Cypress | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| - name: Install Yarn ${{ env.YARN_VERSION }} | |
| run: | | |
| corepack prepare yarn@${{ env.YARN_VERSION }} --activate | |
| yarn set version ${{ env.YARN_VERSION }} | |
| yarn --version | |
| - name: Check whether to build container images | |
| id: should-build-images | |
| run: | | |
| echo "result=$([[ '${{ github.event_name }}' != 'pull_request' || '${{ github.event.pull_request.head.repo.full_name }}' == '${{ github.repository }}' ]] && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT | |
| - name: Free up additional disk space | |
| if: steps.should-build-images.outputs.result == 'true' | |
| run: | | |
| ./.github/workflows/scripts/free_gha_space.sh | |
| - name: Build | |
| run: | | |
| make install-prod | |
| - name: Login to GitHub Container Registry | |
| if: steps.should-build-images.outputs.result == 'true' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build notebook images | |
| if: steps.should-build-images.outputs.result == 'true' | |
| id: build-notebook-images | |
| run: | | |
| TAG_SUFFIX=$([ -n "${{ github.event.pull_request.number }}" ] && echo "-pr-${{ github.event.pull_request.number }}" || echo "-sha-$(echo "${{ github.sha }}" | cut -c1-8)") | |
| echo "Tag suffix: $TAG_SUFFIX" | |
| WHL_FILE_PATH=$(ls dist/*.whl | head -n 1) | |
| WHL_FILE_NAME=$(basename $WHL_FILE_PATH) | |
| echo "Wheel file: $WHL_FILE_PATH" | |
| URLS="" | |
| for tag_prefix in $(echo "${{ env.QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV }}" | tr ',' '\n'); do | |
| LATEST_TAG=$(curl -s "https://quay.io/api/v1/repository/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}/tag/?onlyActiveTags=true&filter_tag_name=like:${tag_prefix}&limit=1" | jq -r ".tags[].name") | |
| echo "Latest tag: $LATEST_TAG" | |
| CONTAINER_IMAGE=ghcr.io/${{ github.repository }}/workbench-images:${LATEST_TAG}${TAG_SUFFIX} | |
| echo "Build container image: $CONTAINER_IMAGE" | |
| docker build -t ${CONTAINER_IMAGE} -f - . <<EOF | |
| FROM quay.io/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}:${LATEST_TAG} | |
| COPY $WHL_FILE_PATH /tmp/ | |
| RUN pip uninstall -y odh-elyra \ | |
| && pip install /tmp/$WHL_FILE_NAME \ | |
| && jupyter labextension list | |
| EOF | |
| [ -z "${URLS}" ] && URLS="${CONTAINER_IMAGE}" || URLS="${URLS},${CONTAINER_IMAGE}" | |
| echo "Publish image to registry" | |
| docker push ${CONTAINER_IMAGE} | |
| echo "Show container image information" | |
| docker images | |
| echo "Clean up resources" | |
| docker system prune -af | |
| echo "urls=${URLS}" >> "$GITHUB_OUTPUT" | |
| done | |
| - name: Print container image URLs | |
| if: steps.should-build-images.outputs.result == 'true' | |
| run: | | |
| echo ${{ steps.build-notebook-images.outputs.urls }} | tr ',' '\n' | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: elyra_build_artifacts | |
| path: | | |
| ${{ github.workspace }}/dist |