From 0866100dbfa4b1716039a14d5d5793d4140be4f0 Mon Sep 17 00:00:00 2001 From: "David E. Wheeler" Date: Sun, 24 Sep 2023 20:10:36 -0400 Subject: [PATCH] Build images for ARM64 Update the CI/CD workflow to use the latest Docker actions for assembling metadata, building, testing, and pushing multi-platform Docker images. Cease using the `./build` script in the workflow, but update it to build images for multiple platforms. To have both the CI/CD workflow and the `./build` script in sync, add the `.envrc` file and teach both to read from it for common values, especially the Sqitch version. Fix an issue building Sqitch on ARM64 by installing List::MoreUtils::XS before building Sqitch; otherwise it hangs and times out during the configure phase for that module. Update the Oracle `Dockerfile` to always install the latest Instant Client; no more need to edit it for new versions. Also teach it to properly install ARM64 client libraries, but disable building for ARM64 in the `Makefile`, for now, until a [bug building DBD::Oracle] can be fixed. Update the Snowflake `Dockerfile` to install the ARM64 ODBC driver on ARM64 Linux, but limit builds to AMD64 in the `Makefile`, for now, until [SnowSQL gets ARM64 support]. Limit the Exasol build to AMD64, as there are currently no Exasol libraries for ARM64. Remove reference to Vertica, for which there is not yet Docker support. [bug building DBD::Oracle]: https://rt.cpan.org/Ticket/Display.html?id=149876 [SnowSQL gets ARM64 support]: https://community.snowflake.com/s/question/0D5Do00000ltxpVKAQ/snowsql-linuxarm64-support --- .envrc | 11 ++++++ .github/workflows/cicd.yml | 74 +++++++++++++++++++++++++++++++++----- Dockerfile | 2 +- Makefile | 13 ++++--- README.md | 9 ++--- build | 41 +++++++++++---------- exasol/Dockerfile | 4 +-- oracle/Dockerfile | 22 ++++++------ snowflake/Dockerfile | 11 ++++-- 9 files changed, 135 insertions(+), 52 deletions(-) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..b06c478 --- /dev/null +++ b/.envrc @@ -0,0 +1,11 @@ +NAME=sqitch +TITLE=Sqitch +VERSION=1.4.0 +IMAGE=sqitch/sqitch +DESCRIPTION="Sensible database change management" +VENDOR="The Sqitch Community" +AUTHORS="Sqitch Hackers " +URL=https://hub.docker.com/r/sqitch/sqitch/ +DOCS="https://github.com/sqitchers/docker-sqitch#readme" +SOURCE=https://github.com/sqitchers/docker-sqitch +LICENSE=MIT diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 10d6e10..8763626 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -4,20 +4,76 @@ on: pull_request: schedule: - cron: '0 10 25 * *' # Monthly at 10am on the 25th + jobs: build: - name: Build and Push to Docker Hub + # https://docs.docker.com/build/ci/github-actions/multi-platform/ + name: 📦 Build and Push to Docker Hub runs-on: ubuntu-latest steps: - - name: Check out the repo - uses: actions/checkout@v3 - - name: Build the Image - run: ./build + # https://github.com/orgs/community/discussions/25678#discussioncomment-5242449 + - name: Free Disk Space + run: rm -rf /opt/hostedtoolcache + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Environment + run: cat .envrc >> "$GITHUB_ENV" + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # https://docs.docker.com/build/ci/github-actions/manage-tags-labels/ + - name: Docker meta + uses: docker/metadata-action@v5 + id: meta + with: + # list of Docker images to use as base name for tags + images: ${{ env.IMAGE }} + # generate Docker tags based on the following events/attributes + tags: | + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=match,pattern=v\d.\d.\d + type=match,pattern=v\d.\d + # Customize OCI labels. + labels: | + org.opencontainers.image.title=${{ env.TITLE }} + org.opencontainers.image.description=${{ env.DESCRIPTION }} + org.opencontainers.image.vendor=${{ env.VENDOR }} + org.opencontainers.image.authors=${{ env.AUTHORS }} + org.opencontainers.image.url=${{ env.URL }} + org.opencontainers.image.documentation=${{ env.DOCS }} + org.opencontainers.image.source=${{ env.SOURCE }} + org.opencontainers.image.version=v${{ env.VERSION }} + org.opencontainers.image.licenses=${{ env.LICENSE }} + org.opencontainers.image.ref.name=${{ env.NAME }}-v${{ env.VERSION }} + - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Push the Image - if: startsWith(github.ref, 'refs/tags/v') - run: docker push -a sqitch/sqitch + + - name: Build and Export to Docker + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ env.IMAGE }}:test + build-args: VERSION=${{ env.VERSION }} + + - name: Test + run: docker run --rm "${{ env.IMAGE }}:test" + + # Release on a v* tag. + - name: Build and Push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ startsWith(github.ref, 'refs/tags/v') }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: VERSION=${{ env.VERSION }} diff --git a/Dockerfile b/Dockerfile index d093b50..40fd609 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN mkdir -p /usr/share/man/man1 /usr/share/man/man7 \ # Install cpan and build dependencies. ENV PERL5LIB /work/local/lib/perl5 RUN curl -sL --compressed https://git.io/cpm > cpm && chmod +x cpm \ - && ./cpm install -L local --verbose --no-test --show-build-log-on-failure ExtUtils::MakeMaker \ + && ./cpm install -L local --verbose --no-test --show-build-log-on-failure ExtUtils::MakeMaker List::MoreUtils::XS \ && ./cpm install -L local --verbose --no-test --show-build-log-on-failure --with-recommends \ --with-configure --cpanfile src/dist/cpanfile diff --git a/Makefile b/Makefile index 0451075..85e8f8a 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,19 @@ .DEFAULT_GOAL := sqitch .PHONY: oracle snowflake vertica exasol firebird postgres mysql sqlite -sqitch: Dockerfile +sqitch: Dockerfile .envrc ./build +# Oracle only on amd64 till an issue building DBD::Oracle is sorted. +# https://rt.cpan.org/Ticket/Display.html?id=149876 oracle: oracle/Dockerfile - env DIR=oracle REGISTRY=sqitch ./build + env DIR=oracle REGISTRY=sqitch ARCHS=amd64 ./build +# Snowflake only on amd64 till SnowSQL ARM support released. +# https://community.snowflake.com/s/question/0D5Do00000ltxpVKAQ/snowsql-linuxarm64-support snowflake: snowflake/Dockerfile - env DIR=snowflake REGISTRY=sqitch ./build --build-arg sf_account=example + env DIR=snowflake REGISTRY=sqitch ARCHS=amd64 ./build --build-arg sf_account=example +# Exasol currently offers no ARM support. exasol: exasol/Dockerfile - env DIR=exasol REGISTRY=sqitch ./build + env DIR=exasol REGISTRY=sqitch ARCHS=amd64 ./build diff --git a/README.md b/README.md index 68544f1..74ae759 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,11 @@ Notes for it to land). In the meantime you can [use a NAT gateway container](https://github.com/qoomon/docker-host) to forward traffic to the Docker host. -* Custom images for [Oracle], [Snowflake], [Exasol], or [Vertica] can be built - by downloading the appropriate binary files and using the `Dockerfile`s in - the appropriately-named subdirectories of this repository. +* Custom images for [Oracle], [Snowflake], or [Exasol] can be built by + downloading the appropriate binary files and using the `Dockerfile`s in the + appropriately-named subdirectories of this repository. +* The Sqitch image is built with AMD64 and ARM64 support, but the [Oracle], + [Snowflake] and [Exasol] images can currently only be built for AMD64. * In an effort to keep things as simple as possible, the only editor included and configured for use in the image is [nano]. This is a very simple, tiny text editor suitable for editing change descriptions and the like. Its @@ -91,5 +93,4 @@ Notes [Oracle]: https://www.oracle.com/database/ [Snowflake]:https://www.snowflake.com [Exasol]:https://www.exasol.com/ - [Vertica]: https://www.vertica.com [nano]: https://www.nano-editor.org/ diff --git a/build b/build index 8b6d0e6..7041b5d 100755 --- a/build +++ b/build @@ -2,11 +2,11 @@ set -e -NAME=sqitch -VERSION=1.4.0 +source .envrc DIR=${DIR:=.} REGISTRY=${REGISTRY:="$NAME"} +ARCHS=${ARCHS:='amd64 arm64'} # For main Sqitch build, the main tag is "latest" and there # is no sub-package. @@ -34,20 +34,23 @@ if [ "$GITHUB_REF_TYPE" == "tag" ]; then fi fi -# The Oracle and Snowflake images require amd64. -docker buildx build --platform=linux/amd64 --pull \ - --label org.opencontainers.image.created=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ - --label org.opencontainers.image.authors='Sqitch Hackers ' \ - --label org.opencontainers.image.url="https://hub.docker.com/r/sqitch/sqitch/" \ - --label org.opencontainers.image.documentation="https://github.com/sqitchers/docker-sqitch#readme" \ - --label org.opencontainers.image.source="https://github.com/sqitchers/docker-sqitch" \ - --label org.opencontainers.image.version="v$VERSION" \ - --label org.opencontainers.image.revision="$(git rev-parse --abbrev-ref HEAD)" \ - --label org.opencontainers.image.vendor="The Sqitch Community" \ - --label org.opencontainers.image.licenses="MIT" \ - --label org.opencontainers.image.ref.name="sqitch${PKG}-v${VERSION}" \ - --label org.opencontainers.image.title="Sqitch" \ - --label org.opencontainers.image.description="Sensible database change management" \ - "${tagopt[@]}" \ - --build-arg "VERSION=${VERSION}" \ - "$@" . +# Build and export imaages to Docker for all supported architectures. +for arch in $ARCHS; do + printf "Building for linux/%s\n" "${arch}" + docker buildx build --platform="linux/${arch}" --pull --load \ + --label org.opencontainers.image.created=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --label org.opencontainers.image.authors="${AUTHORS}" \ + --label org.opencontainers.image.url="${URL}" \ + --label org.opencontainers.image.documentation="${DOCS}" \ + --label org.opencontainers.image.source="${SOURCE}" \ + --label org.opencontainers.image.version="v${VERSION}" \ + --label org.opencontainers.image.revision="$(git rev-parse --abbrev-ref HEAD)" \ + --label org.opencontainers.image.vendor="${VENDOR}" \ + --label org.opencontainers.image.licenses="${LICENSE}" \ + --label org.opencontainers.image.ref.name="${NAME}${PKG}-v${VERSION}" \ + --label org.opencontainers.image.title="${TITLE}" \ + --label org.opencontainers.image.description="${DESCRIPTION}" \ + "${tagopt[@]}" \ + --build-arg "VERSION=${VERSION}" \ + "$@" . +done diff --git a/exasol/Dockerfile b/exasol/Dockerfile index 04431c7..aec7703 100644 --- a/exasol/Dockerfile +++ b/exasol/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS exa-build +FROM --platform=linux/amd64 debian:bullseye-slim AS exa-build WORKDIR /work @@ -17,7 +17,7 @@ RUN tar zxf EXASOL_ODBC-$version.tar.gz \ && mv EXAplus-$version/ exaplus \ && rm -rf exaplus/doc -FROM sqitch/sqitch:latest +FROM --platform=linux/amd64 sqitch/sqitch:latest # Install runtime dependencies, remove unnecesary files, and create symlink. USER root diff --git a/oracle/Dockerfile b/oracle/Dockerfile index 5c758d4..a5e6a6e 100644 --- a/oracle/Dockerfile +++ b/oracle/Dockerfile @@ -1,28 +1,30 @@ FROM debian:bullseye-slim AS ora-build WORKDIR /work -# https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html -ARG INSTANTCLIENT_VERSION=21.11.0.0.0dbru -ARG INSTANTCLIENT_VDIR=2111000 -ADD https://download.oracle.com/otn_software/linux/instantclient/${INSTANTCLIENT_VDIR}/instantclient-basic-linux.x64-${INSTANTCLIENT_VERSION}.zip ./ -ADD https://download.oracle.com/otn_software/linux/instantclient/${INSTANTCLIENT_VDIR}/instantclient-sqlplus-linux.x64-${INSTANTCLIENT_VERSION}.zip ./ -ADD https://download.oracle.com/otn_software/linux/instantclient/${INSTANTCLIENT_VDIR}/instantclient-sdk-linux.x64-${INSTANTCLIENT_VERSION}.zip ./ +# Alwyays install the latest version of instantclient. +# https://www.oracle.com/database/technologies/instant-client/linux-x86-64-downloads.html +# https://www.oracle.com/database/technologies/instant-client/linux-arm-aarch64-downloads.html +ENV BASEURI=https://download.oracle.com/otn_software/linux/instantclient/instantclient ENV ORACLE_HOME /work/instantclient ENV LD_LIBRARY_PATH /work/instantclient ENV PERL5LIB /work/tmp/lib/perl5 RUN mkdir -p /usr/share/man/man1 /usr/share/man/man7 \ - # Install dependen cies + # Install dependencies, including latest platform-specific instantclient. && apt-get -qq update \ && apt-get -qq install build-essential libarchive-tools curl libaio-dev \ + && case "$(arch)" in aarch64) export ORAPLAT=-arm64 ;; x86_64) export ORAPLAT=x64 ;; esac \ + && curl "${BASEURI}-basic-linux${ORAPLAT}.zip" -o instantclient-basic.zip \ + && curl "${BASEURI}-sqlplus-linux${ORAPLAT}.zip" -o instantclient-sqlplus.zip \ + && curl "${BASEURI}-sdk-linux${ORAPLAT}.zip" -o instantclient-sdk.zip \ && mkdir instantclient \ - && bsdtar -C instantclient --strip-components 1 -zxf instantclient-basic-linux.x64-${INSTANTCLIENT_VERSION}.zip \ - && bsdtar -C instantclient --strip-components 1 -zxf instantclient-sqlplus-linux.x64-${INSTANTCLIENT_VERSION}.zip \ + && bsdtar -C instantclient --strip-components 1 -zxf instantclient-basic.zip \ + && bsdtar -C instantclient --strip-components 1 -zxf instantclient-sqlplus.zip \ # Copy basic and SQL*Plus for installation below. && cp -rf instantclient instantclient.install \ - && bsdtar -C instantclient --strip-components 1 -zxf instantclient-sdk-linux.x64-${INSTANTCLIENT_VERSION}.zip \ + && bsdtar -C instantclient --strip-components 1 -zxf instantclient-sdk.zip \ # Install DBI in its own directory, then install DBD::Oracle. && curl https://cpanmin.us > cpanm && chmod +x cpanm \ && ./cpanm install -l tmp --quiet --notest DBI \ diff --git a/snowflake/Dockerfile b/snowflake/Dockerfile index 9a7b6f9..48f616d 100644 --- a/snowflake/Dockerfile +++ b/snowflake/Dockerfile @@ -6,8 +6,10 @@ WORKDIR /work # https://docs.snowflake.com/en/user-guide/snowsql-install-config # https://docs.snowflake.com/en/release-notes/client-change-log-snowsql # https://sfc-repo.snowflakecomputing.com/index.html +ENV ODBC_VERSION=3.1.0 +# No ARM-specific SnowSQL yet. +# https://community.snowflake.com/s/question/0D5Do00000ltxpVKAQ/snowsql-linuxarm64-support ADD https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.28-linux_x86_64.bash snowsql.bash -ADD https://sfc-repo.snowflakecomputing.com/odbc/linuxaarch64/3.0.2/snowflake_linux_aarch64_odbc-3.0.2.tgz snowflake_linux_x8664_odbc.tgz COPY conf ./ # Tell SnowSQL where to store its versions and config. Need to keep it inside @@ -21,11 +23,14 @@ ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 # Install prereqs. ARG sf_account RUN apt-get -qq update \ - && apt-get -qq --no-install-recommends install odbcinst \ + && apt-get -qq --no-install-recommends install odbcinst curl ca-certificates \ # Configure ODBC. https://docs.snowflake.net/manuals/user-guide/odbc-linux.html - && gunzip -f *.tgz && tar xf *.tar \ + && case "$(arch)" in aarch64) export SNOWPLAT=aarch64 DIRARCH=aarch64 ;; x86_64) export SNOWPLAT=x8664 ;; esac \ + && curl https://sfc-repo.snowflakecomputing.com/odbc/linux${DIRARCH}/${ODBC_VERSION}/snowflake_linux_${SNOWPLAT}_odbc-${ODBC_VERSION}.tgz -o snowflake_odbc.tgz \ + && gunzip -f *.tgz && tar vxf *.tar \ && mkdir odbc \ && mv snowflake_odbc/lib snowflake_odbc/ErrorMessages odbc/ \ + && perl -i -pe "s/x86_64/$(arch)/g" simba.snowflake.ini \ && mv simba.snowflake.ini odbc/lib/ \ && perl -i -pe "s/SF_ACCOUNT/$sf_account/g" odbc.ini \ && cat odbc.ini >> /etc/odbc.ini \