build_test_push.yaml

name: Build, test & push executor

on:
  workflow_call:
    # WARNING: Duplicate everything here into workflow_dispatch
    inputs:
      rondb_version:
        required: true
        type: string
      rondb_x86_tarball_name:
        required: true
        type: string
      rondb_arm64_tarball_name:
        required: true
        type: string
      is_latest_rondb_release:
        required: true
        description: "Is latest stable RonDB release; affects the image tag"
        type: boolean
        default: false
      skip_test:
        required: true
        description: "Skip running a docker-compose cluster with benchmarking"
        type: boolean
        default: false
      base_download_url:
        type: string
        required: true
        description: "Base URL for downloading RonDB tarballs; affects the image name"
        default: https://repo.hops.works/master
        
  # Inputs here have to be duplicated from workflow_call...
  # https://github.com/orgs/community/discussions/39357
  workflow_dispatch:
    inputs:
      rondb_version:
        required: true
        type: string
      rondb_x86_tarball_name:
        required: true
        type: string
      rondb_arm64_tarball_name:
        required: true
        type: string
      is_latest_rondb_release:
        required: true
        description: "Is latest stable RonDB release; affects the image tag"
        type: boolean
        default: false
      skip_test:
        required: true
        description: "Skip running a docker-compose cluster with benchmarking"
        type: boolean
        default: false
      base_download_url:
        type: choice
        required: true
        description: "Base URL for downloading RonDB tarballs; affects the image name"
        default: https://repo.hops.works/master
        options:
        - https://repo.hops.works/master
        - https://repo.hops.works/master/rondb-dev

jobs:
  payload-validation:
    runs-on: ubuntu-latest
    steps:
      - name: Tarball name is not a path
        run: |
          if [[ "${{ inputs.rondb_x86_tarball_name }}" == */* || "${{ inputs.rondb_arm64_tarball_name }}" == */* ]]; then
            echo "The tarball names contain '/'s, which are not allowed."
            exit 1
          fi

      - name: Check base_download_url is /master if is_latest_rondb_release is selected
        run: |
          if [ "${{ inputs.is_latest_rondb_release }}" = "true" ] && [ "${{ inputs.base_download_url }}" != "https://repo.hops.works/master" ]; then
            echo "is_latest_rondb_release can only be true if base_download_url is set to https://repo.hops.works/master"
            exit 1
          fi

  process-info:
    runs-on: ubuntu-latest
    needs: [payload-validation]
    outputs:
      is_highest_release: ${{ steps.is_highest_release.outputs.is_highest }}
      arm_image_name: ${{ steps.image_names.outputs.arm }}
      x86_image_name: ${{ steps.image_names.outputs.x86 }}
      multi_image_name: ${{ steps.image_names.outputs.multi }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Check if current branch is the highest release branch
        if: github.repository == 'logicalclocks/rondb-docker'
        id: is_highest_release
        run: |
          ALL_RELEASES=$(git branch -r | grep 'origin/release-' | sed 's|origin/||')
          HIGHEST_RELEASE=$(echo "$ALL_RELEASES" | sort -V | tail -n 1 | xargs)
          echo "Highest release branch is '$HIGHEST_RELEASE'"
          echo "GITHUB_REF_NAME is '${GITHUB_REF_NAME}'"

          if [ "${GITHUB_REF_NAME}" = "$HIGHEST_RELEASE" ]; then
            echo "Current branch is the highest release branch."
            echo "is_highest=true" >> $GITHUB_OUTPUT
          else
            echo "Current branch is not the highest release branch."
            echo "is_highest=false" >> $GITHUB_OUTPUT
          fi

      # When using tarballs from rondb-dev, we'll use the image name "rondb-dev"
      - name: Get image names
        id: image_names
        run: |
          if [ "${{ inputs.base_download_url }}" == "https://repo.hops.works/master" ]; then
            echo "arm=rondb-arm64" >> $GITHUB_OUTPUT
            echo "x86=rondb-amd64" >> $GITHUB_OUTPUT
            echo "multi=rondb" >> $GITHUB_OUTPUT
          else
            echo "arm=rondb-dev-arm64" >> $GITHUB_OUTPUT
            echo "x86=rondb-dev-amd64" >> $GITHUB_OUTPUT
            echo "multi=rondb-dev" >> $GITHUB_OUTPUT
          fi

  build-test-push-x86:
    runs-on: ubuntu-latest
    needs: [process-info]
    if: ${{ !inputs.skip_test || (github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main'))}}
    steps:
      - uses: actions/checkout@v4

      - name: Login to Dockerhub
        uses: docker/login-action@v3
        if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main')
        with:
          username: hopsworks
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build and run Docker Compose cluster with benchmarking
        if: ${{ !inputs.skip_test }}
        run: |
          X86_IMAGE_NAME="${{ needs.process-info.outputs.x86_image_name }}"

          ./run.sh -lv \
            --rondb-version ${{ inputs.rondb_version }} \
            --rondb-image-name $X86_IMAGE_NAME \
            --rondb-tarball-url ${{ inputs.base_download_url }}/${{ inputs.rondb_x86_tarball_name }} \
            --size mini \
            --run-benchmark sysbench_single \
            --detached

      - name: Wait for one container exit or timeout
        if: ${{ !inputs.skip_test }}
        run: |
          start=`date +%s`
          while true; do
              end=`date +%s`
              runtime=$((end-start))
              if [ $( docker container ls --filter "status=exited" | grep rondb | wc -l ) -gt 0 ]; then
                  echo "One container is down. We can continue"
                  docker container ls --filter "status=exited"
                  exit 0
              elif [ $runtime -gt 800 ]; then
                  echo "The benchmarking seems to be stuck. We're aborting now."
                  docker ps
                  exit 1
              fi
              sleep 2
          done

      - run: docker container ls
        if: ${{ !inputs.skip_test }}
      - run: docker logs mgmd_1
        if: ${{ !inputs.skip_test }}
      - run: docker logs ndbd_1
        if: ${{ !inputs.skip_test }}
      - run: docker logs mysqld_1
        if: ${{ !inputs.skip_test }}
      - run: docker logs rest_1
        if: ${{ !inputs.skip_test }}
      - run: docker logs bench_1
        if: ${{ !inputs.skip_test }}

      # At this point we only know that one container has exited. We want to
      # check whether the bench container has exited with exit code 0. We need
      # both status and exit code to do so, since Docker reports exit code 0
      # for running containers.
      - name: Check Benchmarking Exit Code
        if: ${{ !inputs.skip_test }}
        run: |
          if [ "$(docker inspect bench_1 --format='{{.State.Status}}')" != "exited" ]
          then
            echo "Some container other than bench_1 exited unexpectedly."
            docker ps -a
            exit 1
          elif [ "$(docker inspect bench_1 --format='{{.State.ExitCode}}')" != "0" ]
          then
            echo "Benchmarking failed."
            cat autogenerated_files/*/volumes/sysbench_single/sysbench_results/oltp_rw_0_0.res
            exit 1
          fi

      - name: Printing Sysbench results
        if: ${{ !inputs.skip_test }}
        run: cat autogenerated_files/*/volumes/sysbench_single/final_result.txt

      - name: Get X86 image ID
        run: |
          VERSION="$(sed -e 's/^[[:space:]]*//' -e '/-SNAPSHOT$/s/.*/dev/' ./VERSION)"
          X86_IMAGE_NAME="${{ needs.process-info.outputs.x86_image_name }}"
          TAG=${{ inputs.rondb_version }}-$VERSION

          echo "X86_IMAGE_NAME=$X86_IMAGE_NAME" >> $GITHUB_ENV
          echo "X86_IMAGE_ID=$X86_IMAGE_NAME:$TAG" >> $GITHUB_ENV

      - name: Build X86 image
        if: ${{ inputs.skip_test && github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main')}}
        run: |
          docker buildx build . \
              --tag $X86_IMAGE_ID \
              --build-arg RONDB_VERSION=${{ inputs.rondb_version }} \
              --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \
              --build-arg RONDB_X86_TARBALL_URI=${{ inputs.base_download_url }}/${{ inputs.rondb_x86_tarball_name }}

      - name: Push versioned image
        if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main')
        run: |
          NEW_IMAGE_ID=hopsworks/$X86_IMAGE_ID

          docker tag $X86_IMAGE_ID $NEW_IMAGE_ID
          docker push $NEW_IMAGE_ID

      - name: Push with tag `latest`
        if: ${{ needs.process-info.outputs.is_highest_release == 'true' && inputs.is_latest_rondb_release }}
        run: |
          NEW_IMAGE_ID=hopsworks/$X86_IMAGE_NAME:latest

          docker tag $X86_IMAGE_ID $NEW_IMAGE_ID
          docker push $NEW_IMAGE_ID

  build-push-arm64:
    runs-on: ubuntu-latest
    if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main')
    needs: [process-info]
    steps:
      - uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Dockerhub
        uses: docker/login-action@v3
        with:
          username: hopsworks
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      # We're skipping the benchmarking on ARM64 as we assume this will be run on a regular basis
      # during development. ARM64 images are only for development anyways. It is more important to add
      # all types of benchmarking to the tests.
      - name: Build and push ARM64 image
        run: |
          VERSION="$(sed -e 's/^[[:space:]]*//' -e '/-SNAPSHOT$/s/.*/dev/' ./VERSION)"
          ARM_IMAGE_NAME="${{ needs.process-info.outputs.arm_image_name }}"

          TAGS="--tag hopsworks/$ARM_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION"
          if [[ "${{ needs.process-info.outputs.is_highest_release }}" == "true" &&
                "${{ inputs.is_latest_rondb_release }}" == "true" ]]; then
            TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:latest"
          fi

          docker buildx build . \
              $TAGS \
              --platform=linux/arm64 \
              --output type=registry \
              --build-arg RONDB_VERSION=${{ inputs.rondb_version }} \
              --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \
              --build-arg RONDB_ARM_TARBALL_URI=${{ inputs.base_download_url }}/${{ inputs.rondb_arm64_tarball_name }} \
              --cache-to type=registry,ref=hopsworks/rondb-cache,mode=max \
              --cache-from type=registry,ref=hopsworks/rondb-cache,mode=max

  make-multi-platform-image:
    runs-on: ubuntu-latest
    if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main')
    needs: [process-info, build-test-push-x86, build-push-arm64]
    steps:
      - uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Dockerhub
        uses: docker/login-action@v3
        with:
          username: hopsworks
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to Nexus
        uses: docker/login-action@v3
        with:
          registry: docker.hops.works
          username: ${{ secrets.NEXUS_USER }}
          password: ${{ secrets.NEXUS_PASSWORD }}

      - name: Create and push multi-platform image
        run: |
          VERSION="$(sed -e 's/^[[:space:]]*//' -e '/-SNAPSHOT$/s/.*/dev/' ./VERSION)"

          X86_IMAGE_NAME="${{ needs.process-info.outputs.x86_image_name }}"
          ARM_IMAGE_NAME="${{ needs.process-info.outputs.arm_image_name }}"
          MULTI_IMAGE_NAME="${{ needs.process-info.outputs.multi_image_name }}"

          for repo_url in docker.io docker.hops.works; do

            echo "Creating and pushing multi-platform image for $repo_url"

            docker buildx imagetools create -t $repo_url/hopsworks/$MULTI_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION \
              hopsworks/$X86_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION \
              hopsworks/$ARM_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION

            if [[ "${{ needs.process-info.outputs.is_highest_release }}" == "true" && 
                  "${{ inputs.is_latest_rondb_release }}" == "true" ]]; then
              docker buildx imagetools create -t $repo_url/hopsworks/$MULTI_IMAGE_NAME:latest \
                hopsworks/$X86_IMAGE_NAME:latest \
                hopsworks/$ARM_IMAGE_NAME:latest
            fi

          done