build.yml

#
# Copyright 2018-2023 Elyra Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Elyra CI
on:
  push: # all branches
  pull_request: # all branches
  schedule:
    # once a day at 3 am (PST) (10 am (UTC))
    - cron: '0 10 * * *'

env:
  FORCE_COLOR: true
  NODE_VERSION: 20.11.0
  YARN_VERSION: 3.5.0
  PYTHON_VERSION: 3.11
  QUAY_WORKBENCH_IMAGES_REPOSITORY: 'opendatahub/workbench-images'
  QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV: 'jupyter-trustyai-ubi9-python-3.11-20241,cuda-jupyter-tensorflow-ubi9-python-3.11-20241'

jobs:
  prepare-yarn-cache:
    name: Prepare Cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/cache@v4
        with:
          path: |
            node_modules
            */*/node_modules
            /home/runner/.cache/Cypress
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
      - name: Install Yarn ${{ env.YARN_VERSION }}
        run: |
          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
          yarn set version ${{ env.YARN_VERSION }}
          yarn --version
      - name: Install dependencies
        run: |
          yarn install
  lint-server:
    name: Lint Server
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Lint
        run: make lint-server

  lint-ui:
    name: Lint UI
    needs: prepare-yarn-cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/cache@v4
        with:
          path: |
            node_modules
            */*/node_modules
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
      - name: Install Yarn ${{ env.YARN_VERSION }}
        run: |
          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
          yarn set version ${{ env.YARN_VERSION }}
          yarn --version
      - name: Install dependencies
        run: |
          make yarn-install
      - name: Lint
        run: make eslint-check-ui
      - name: Check format
        run: make prettier-check-ui

  test-server:
    name: Test Server
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11']
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install
        # Note: Tests fail if we don't install `wheel` for some reason.
        run: |
          make install-server
          make test-dependencies
      - name: Version Snapshot
        run: pip freeze
      - name: Test
        id: Test
        run: make pytest
      - name: Fetch packages metadata (if Test step failed)
        # Note: checking for general failure() first is needed according to docs
        if: ${{ failure() && steps.Test.conclusion == 'failure' }}
        env:
          days: 7 # check for newer package versions in past X days
        run: |
          python3 ./.github/workflows/scripts/fetch_packages_metadata.py $days

  test-ui:
    name: Test UI
    needs: prepare-yarn-cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - uses: actions/cache@v4
        with:
          path: |
            node_modules
            */*/node_modules
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
      - name: Install Yarn ${{ env.YARN_VERSION }}
        run: |
          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
          yarn set version ${{ env.YARN_VERSION }}
          yarn --version
      - name: Build
        run: |
          make build-dependencies
          make yarn-install
          make build-ui-prod
      - name: Install
        run: make install-server
      - name: Test
        run: make test-ui-unit

  test-integration:
    name: Run Integration Tests
    needs: prepare-yarn-cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - uses: actions/cache@v4
        with:
          path: |
            node_modules
            */*/node_modules
            /home/runner/.cache/Cypress
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
      - name: Install Yarn ${{ env.YARN_VERSION }}
        run: |
          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
          yarn set version ${{ env.YARN_VERSION }}
          yarn --version
      - name: Build
        run: |
          make build-dependencies
          make yarn-install
          make build-ui-prod
      - name: Install
        run: |
          make install-server
          make install-examples
      - name: Cypress
        run: make test-integration
      - name: Collect logs
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: elyra_test_artifacts
          path: |
            ${{ github.workspace }}/build/cypress-tests/*.log
            ${{ github.workspace }}/build/cypress-tests/screenshots//**/*
            ${{ github.workspace }}/build/cypress-tests/videos//**/*
            /home/runner/.npm/_logs/*.log

  test-documentation-build:
    name: Test documentation build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Build
        run: make docs

  validate-image-env:
    name: Validate Image Environment
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11']
    steps:
      - uses: actions/checkout@v4
      - name: Validate image environment
        run: make PYTHON_VERSION=${{ matrix.python-version }} elyra-image-env

  validate-images:
    name: Validate Images
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Validate runtime images
        run: make REMOVE_RUNTIME_IMAGE=1 validate-runtime-images

  publish-artifacts:
    permissions:
      packages: write
    name: Publish Artifacts
    needs: prepare-yarn-cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - uses: actions/cache@v4
        with:
          path: |
            node_modules
            */*/node_modules
            /home/runner/.cache/Cypress
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
      - name: Install Yarn ${{ env.YARN_VERSION }}
        run: |
          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
          yarn set version ${{ env.YARN_VERSION }}
          yarn --version
      - name: Check whether to build container images
        id: should-build-images
        run: |
          echo "result=$([[ '${{ github.event_name }}' != 'pull_request' || '${{ github.event.pull_request.head.repo.full_name }}' == '${{ github.repository }}' ]] && echo 'true' || echo 'false')" >> $GITHUB_OUTPUT
      - name: Free up additional disk space
        if: steps.should-build-images.outputs.result == 'true'
        run: |
          ./.github/workflows/scripts/free_gha_space.sh
      - name: Build
        run: |
          make install-prod
      - name: Login to GitHub Container Registry
        if: steps.should-build-images.outputs.result == 'true'
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build notebook images
        if: steps.should-build-images.outputs.result == 'true'
        id: build-notebook-images
        run: |
          TAG_SUFFIX=$([ -n "${{ github.event.pull_request.number }}" ] && echo "-pr-${{ github.event.pull_request.number }}" || echo "-sha-$(echo "${{ github.sha }}" | cut -c1-8)")
          echo "Tag suffix: $TAG_SUFFIX"

          WHL_FILE_PATH=$(ls dist/*.whl | head -n 1)
          WHL_FILE_NAME=$(basename $WHL_FILE_PATH)
          echo "Wheel file: $WHL_FILE_PATH"

          URLS=""
          for tag_prefix in $(echo "${{ env.QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV }}" | tr ',' '\n'); do
            LATEST_TAG=$(curl -s "https://quay.io/api/v1/repository/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}/tag/?onlyActiveTags=true" | \
              jq -r ".tags | map(select(.name | startswith(\"${tag_prefix}\"))) | .[].name" | \
              sort -r | head -n 1)
            echo "Latest tag: $LATEST_TAG"

            CONTAINER_IMAGE=ghcr.io/${{ github.repository }}/workbench-images:${LATEST_TAG}${TAG_SUFFIX}
            echo "Build container image: $CONTAINER_IMAGE"

            docker build -t ${CONTAINER_IMAGE} -f - . <<EOF
            FROM quay.io/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}:${LATEST_TAG}
            COPY $WHL_FILE_PATH /tmp/
            RUN pip uninstall -y odh-elyra \
              && pip install /tmp/$WHL_FILE_NAME \
              && jupyter labextension list
          EOF

            [ -z "${URLS}" ] && URLS="${CONTAINER_IMAGE}" || URLS="${URLS},${CONTAINER_IMAGE}"

            echo "Publish image to registry"
            docker push ${CONTAINER_IMAGE}

            echo "Show container image information"
            docker images

            echo "Clean up resources"
            docker system prune -af

            echo "urls=${URLS}" >> "$GITHUB_OUTPUT"
          done
      - name: Print container image URLs
        if: steps.should-build-images.outputs.result == 'true'
        run: |
          echo ${{ steps.build-notebook-images.outputs.urls }} | tr ',' '\n'
      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: elyra_build_artifacts
          path: |
            ${{ github.workspace }}/dist