Build, test & push executor #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, test & push executor | |
| on: | |
| workflow_call: | |
| # WARNING: Duplicate everything here into workflow_dispatch | |
| inputs: | |
| rondb_version: | |
| required: true | |
| type: string | |
| rondb_x86_tarball_name: | |
| required: true | |
| type: string | |
| rondb_arm64_tarball_name: | |
| required: true | |
| type: string | |
| is_latest_rondb_release: | |
| required: true | |
| description: "Is latest stable RonDB release; affects the image tag" | |
| type: boolean | |
| default: false | |
| skip_test: | |
| required: true | |
| description: "Skip running a docker-compose cluster with benchmarking" | |
| type: boolean | |
| default: false | |
| base_download_url: | |
| type: string | |
| required: true | |
| description: "Base URL for downloading RonDB tarballs; affects the image name" | |
| default: https://repo.hops.works/master | |
| # Inputs here have to be duplicated from workflow_call... | |
| # https://github.com/orgs/community/discussions/39357 | |
| workflow_dispatch: | |
| inputs: | |
| rondb_version: | |
| required: true | |
| type: string | |
| rondb_x86_tarball_name: | |
| required: true | |
| type: string | |
| rondb_arm64_tarball_name: | |
| required: true | |
| type: string | |
| is_latest_rondb_release: | |
| required: true | |
| description: "Is latest stable RonDB release; affects the image tag" | |
| type: boolean | |
| default: false | |
| skip_test: | |
| required: true | |
| description: "Skip running a docker-compose cluster with benchmarking" | |
| type: boolean | |
| default: false | |
| base_download_url: | |
| type: choice | |
| required: true | |
| description: "Base URL for downloading RonDB tarballs; affects the image name" | |
| default: https://repo.hops.works/master | |
| options: | |
| - https://repo.hops.works/master | |
| - https://repo.hops.works/master/rondb-dev | |
| jobs: | |
| payload-validation: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Tarball name is not a path | |
| run: | | |
| if [[ "${{ inputs.rondb_x86_tarball_name }}" == */* || "${{ inputs.rondb_arm64_tarball_name }}" == */* ]]; then | |
| echo "The tarball names contain '/'s, which are not allowed." | |
| exit 1 | |
| fi | |
| - name: Check base_download_url is /master if is_latest_rondb_release is selected | |
| run: | | |
| if [ "${{ inputs.is_latest_rondb_release }}" = "true" ] && [ "${{ inputs.base_download_url }}" != "https://repo.hops.works/master" ]; then | |
| echo "is_latest_rondb_release can only be true if base_download_url is set to https://repo.hops.works/master" | |
| exit 1 | |
| fi | |
| process-info: | |
| runs-on: ubuntu-latest | |
| needs: [payload-validation] | |
| outputs: | |
| is_highest_release: ${{ steps.is_highest_release.outputs.is_highest }} | |
| arm_image_name: ${{ steps.image_names.outputs.arm }} | |
| x86_image_name: ${{ steps.image_names.outputs.x86 }} | |
| multi_image_name: ${{ steps.image_names.outputs.multi }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check if current branch is the highest release branch | |
| if: github.repository == 'logicalclocks/rondb-docker' | |
| id: is_highest_release | |
| run: | | |
| ALL_RELEASES=$(git branch -r | grep 'origin/release-' | sed 's|origin/||') | |
| HIGHEST_RELEASE=$(echo "$ALL_RELEASES" | sort -V | tail -n 1 | xargs) | |
| echo "Highest release branch is '$HIGHEST_RELEASE'" | |
| echo "GITHUB_REF_NAME is '${GITHUB_REF_NAME}'" | |
| if [ "${GITHUB_REF_NAME}" = "$HIGHEST_RELEASE" ]; then | |
| echo "Current branch is the highest release branch." | |
| echo "is_highest=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Current branch is not the highest release branch." | |
| echo "is_highest=false" >> $GITHUB_OUTPUT | |
| fi | |
| # When using tarballs from rondb-dev, we'll use the image name "rondb-dev" | |
| - name: Get image names | |
| id: image_names | |
| run: | | |
| if [ "${{ inputs.base_download_url }}" == "https://repo.hops.works/master" ]; then | |
| echo "arm=rondb-arm64" >> $GITHUB_OUTPUT | |
| echo "x86=rondb-amd64" >> $GITHUB_OUTPUT | |
| echo "multi=rondb" >> $GITHUB_OUTPUT | |
| else | |
| echo "arm=rondb-dev-arm64" >> $GITHUB_OUTPUT | |
| echo "x86=rondb-dev-amd64" >> $GITHUB_OUTPUT | |
| echo "multi=rondb-dev" >> $GITHUB_OUTPUT | |
| fi | |
| build-test-push-x86: | |
| runs-on: ubuntu-latest | |
| needs: [process-info] | |
| if: ${{ !inputs.skip_test || (github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main'))}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to Dockerhub | |
| uses: docker/login-action@v3 | |
| if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') | |
| with: | |
| username: hopsworks | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and run Docker Compose cluster with benchmarking | |
| if: ${{ !inputs.skip_test }} | |
| run: | | |
| X86_IMAGE_NAME="${{ needs.process-info.outputs.x86_image_name }}" | |
| ./run.sh -lv \ | |
| --rondb-version ${{ inputs.rondb_version }} \ | |
| --rondb-image-name $X86_IMAGE_NAME \ | |
| --rondb-tarball-url ${{ inputs.base_download_url }}/${{ inputs.rondb_x86_tarball_name }} \ | |
| --size mini \ | |
| --run-benchmark sysbench_single \ | |
| --detached | |
| - name: Wait for one container exit or timeout | |
| if: ${{ !inputs.skip_test }} | |
| run: | | |
| start=`date +%s` | |
| while true; do | |
| end=`date +%s` | |
| runtime=$((end-start)) | |
| if [ $( docker container ls --filter "status=exited" | grep rondb | wc -l ) -gt 0 ]; then | |
| echo "One container is down. We can continue" | |
| docker container ls --filter "status=exited" | |
| exit 0 | |
| elif [ $runtime -gt 800 ]; then | |
| echo "The benchmarking seems to be stuck. We're aborting now." | |
| docker ps | |
| exit 1 | |
| fi | |
| sleep 2 | |
| done | |
| - run: docker container ls | |
| if: ${{ !inputs.skip_test }} | |
| - run: docker logs mgmd_1 | |
| if: ${{ !inputs.skip_test }} | |
| - run: docker logs ndbd_1 | |
| if: ${{ !inputs.skip_test }} | |
| - run: docker logs mysqld_1 | |
| if: ${{ !inputs.skip_test }} | |
| - run: docker logs rest_1 | |
| if: ${{ !inputs.skip_test }} | |
| - run: docker logs bench_1 | |
| if: ${{ !inputs.skip_test }} | |
| # At this point we only know that one container has exited. We want to | |
| # check whether the bench container has exited with exit code 0. We need | |
| # both status and exit code to do so, since Docker reports exit code 0 | |
| # for running containers. | |
| - name: Check Benchmarking Exit Code | |
| if: ${{ !inputs.skip_test }} | |
| run: | | |
| if [ "$(docker inspect bench_1 --format='{{.State.Status}}')" != "exited" ] | |
| then | |
| echo "Some container other than bench_1 exited unexpectedly." | |
| docker ps -a | |
| exit 1 | |
| elif [ "$(docker inspect bench_1 --format='{{.State.ExitCode}}')" != "0" ] | |
| then | |
| echo "Benchmarking failed." | |
| cat autogenerated_files/*/volumes/sysbench_single/sysbench_results/oltp_rw_0_0.res | |
| exit 1 | |
| fi | |
| - name: Printing Sysbench results | |
| if: ${{ !inputs.skip_test }} | |
| run: cat autogenerated_files/*/volumes/sysbench_single/final_result.txt | |
| - name: Get X86 image ID | |
| run: | | |
| VERSION="$(sed -e 's/^[[:space:]]*//' -e '/-SNAPSHOT$/s/.*/dev/' ./VERSION)" | |
| X86_IMAGE_NAME="${{ needs.process-info.outputs.x86_image_name }}" | |
| TAG=${{ inputs.rondb_version }}-$VERSION | |
| echo "X86_IMAGE_NAME=$X86_IMAGE_NAME" >> $GITHUB_ENV | |
| echo "X86_IMAGE_ID=$X86_IMAGE_NAME:$TAG" >> $GITHUB_ENV | |
| - name: Build X86 image | |
| if: ${{ inputs.skip_test && github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main')}} | |
| run: | | |
| docker buildx build . \ | |
| --tag $X86_IMAGE_ID \ | |
| --build-arg RONDB_VERSION=${{ inputs.rondb_version }} \ | |
| --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \ | |
| --build-arg RONDB_X86_TARBALL_URI=${{ inputs.base_download_url }}/${{ inputs.rondb_x86_tarball_name }} | |
| - name: Push versioned image | |
| if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') | |
| run: | | |
| NEW_IMAGE_ID=hopsworks/$X86_IMAGE_ID | |
| docker tag $X86_IMAGE_ID $NEW_IMAGE_ID | |
| docker push $NEW_IMAGE_ID | |
| - name: Push with tag `latest` | |
| if: ${{ needs.process-info.outputs.is_highest_release == 'true' && inputs.is_latest_rondb_release }} | |
| run: | | |
| NEW_IMAGE_ID=hopsworks/$X86_IMAGE_NAME:latest | |
| docker tag $X86_IMAGE_ID $NEW_IMAGE_ID | |
| docker push $NEW_IMAGE_ID | |
| build-push-arm64: | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') | |
| needs: [process-info] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Dockerhub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: hopsworks | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # We're skipping the benchmarking on ARM64 as we assume this will be run on a regular basis | |
| # during development. ARM64 images are only for development anyways. It is more important to add | |
| # all types of benchmarking to the tests. | |
| - name: Build and push ARM64 image | |
| run: | | |
| VERSION="$(sed -e 's/^[[:space:]]*//' -e '/-SNAPSHOT$/s/.*/dev/' ./VERSION)" | |
| ARM_IMAGE_NAME="${{ needs.process-info.outputs.arm_image_name }}" | |
| TAGS="--tag hopsworks/$ARM_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION" | |
| if [[ "${{ needs.process-info.outputs.is_highest_release }}" == "true" && | |
| "${{ inputs.is_latest_rondb_release }}" == "true" ]]; then | |
| TAGS+=" --tag hopsworks/$ARM_IMAGE_NAME:latest" | |
| fi | |
| docker buildx build . \ | |
| $TAGS \ | |
| --platform=linux/arm64 \ | |
| --output type=registry \ | |
| --build-arg RONDB_VERSION=${{ inputs.rondb_version }} \ | |
| --build-arg RONDB_TARBALL_LOCAL_REMOTE=remote \ | |
| --build-arg RONDB_ARM_TARBALL_URI=${{ inputs.base_download_url }}/${{ inputs.rondb_arm64_tarball_name }} \ | |
| --cache-to type=registry,ref=hopsworks/rondb-cache,mode=max \ | |
| --cache-from type=registry,ref=hopsworks/rondb-cache,mode=max | |
| make-multi-platform-image: | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'logicalclocks/rondb-docker' && (startsWith(github.ref_name, 'release-') || github.ref_name == 'main') | |
| needs: [process-info, build-test-push-x86, build-push-arm64] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Dockerhub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: hopsworks | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Login to Nexus | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: docker.hops.works | |
| username: ${{ secrets.NEXUS_USER }} | |
| password: ${{ secrets.NEXUS_PASSWORD }} | |
| - name: Create and push multi-platform image | |
| run: | | |
| VERSION="$(sed -e 's/^[[:space:]]*//' -e '/-SNAPSHOT$/s/.*/dev/' ./VERSION)" | |
| X86_IMAGE_NAME="${{ needs.process-info.outputs.x86_image_name }}" | |
| ARM_IMAGE_NAME="${{ needs.process-info.outputs.arm_image_name }}" | |
| MULTI_IMAGE_NAME="${{ needs.process-info.outputs.multi_image_name }}" | |
| for repo_url in docker.io docker.hops.works; do | |
| echo "Creating and pushing multi-platform image for $repo_url" | |
| docker buildx imagetools create -t $repo_url/hopsworks/$MULTI_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION \ | |
| hopsworks/$X86_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION \ | |
| hopsworks/$ARM_IMAGE_NAME:${{ inputs.rondb_version }}-$VERSION | |
| if [[ "${{ needs.process-info.outputs.is_highest_release }}" == "true" && | |
| "${{ inputs.is_latest_rondb_release }}" == "true" ]]; then | |
| docker buildx imagetools create -t $repo_url/hopsworks/$MULTI_IMAGE_NAME:latest \ | |
| hopsworks/$X86_IMAGE_NAME:latest \ | |
| hopsworks/$ARM_IMAGE_NAME:latest | |
| fi | |
| done |