From 91727778902130e29101df7438aefaed503c7efb Mon Sep 17 00:00:00 2001 From: Doug Schaapveld Date: Tue, 5 Dec 2023 13:37:06 -0600 Subject: [PATCH 1/3] Pin package_cloud gem to 0.2.45 (#524) --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd6c20d..c18539d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,7 @@ jobs: - run: name: Install and Push to Package Cloud command: | - gem install package_cloud + gem install package_cloud -v 0.2.45 bash scripts/push_packagecloud.sh internal deploy-prod: @@ -58,7 +58,7 @@ jobs: - run: name: Install and Push to Package Cloud command: | - gem install package_cloud + gem install package_cloud -v 0.2.45 bash scripts/push_packagecloud.sh internal workflows: From ad330733eade83051bc1461c845d1c494e10b0ab Mon Sep 17 00:00:00 2001 From: Doug Schaapveld Date: Wed, 1 Nov 2023 12:38:01 -0500 Subject: [PATCH 2/3] Remove CircleCI CI/CD config --- .circleci/config.yml | 100 ---------------------------------------- .dockerignore | 1 - .gitignore | 1 - Dockerfile | 3 -- scripts/README.md | 82 -------------------------------- scripts/docker-inner.sh | 90 ------------------------------------ scripts/docker-outer.sh | 55 ---------------------- scripts/version.sh | 61 ------------------------ 8 files changed, 393 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 scripts/README.md delete mode 100755 scripts/docker-inner.sh delete mode 100755 scripts/docker-outer.sh delete mode 100755 scripts/version.sh diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index c18539d..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,100 +0,0 @@ ---- -# all-purpose workspace persister -persist_to_workspace: &persist_to_workspace - persist_to_workspace: - root: . - paths: - - ./ - -version: 2.1 - -# consolidate -executors: - python-executor: - working_directory: ~/pantheon-systems/fusedav - docker: - - image: cimg/python:2.7.18 - ruby-executor: - working_directory: ~/pantheon-systems/fusedav - docker: - - image: cimg/ruby:2.4.10 - -jobs: - build-test: - executor: python-executor - parallelism: 1 - steps: - - checkout - - setup_remote_docker - - run: - name: Hack GitHub credentials into git URL - command: | - git config --global url."https://$GITHUB_TOKEN:x-oauth-basic@github.com/pantheon-systems/".insteadOf "git@github.com:pantheon-systems/" - - run: - name: Set up version - command: bash scripts/version.sh - - run: - name: Build the image - command: bash scripts/docker-outer.sh - - *persist_to_workspace - - deploy-dev: - executor: ruby-executor - steps: - - attach_workspace: - at: . - - run: - name: Install and Push to Package Cloud - command: | - gem install package_cloud -v 0.2.45 - bash scripts/push_packagecloud.sh internal - - deploy-prod: - executor: ruby-executor - steps: - - checkout - - attach_workspace: - at: . - - run: - name: Install and Push to Package Cloud - command: | - gem install package_cloud -v 0.2.45 - bash scripts/push_packagecloud.sh internal - -workflows: - build-test-deploy: - jobs: - - build-test: - name: build-test-dev - filters: - branches: - ignore: - - master - - build-test: - name: build-test-prod - context: - - sig-go-release - filters: - branches: - only: - - master - - deploy-dev: - requires: - - build-test-dev - context: - - sig-go-release - filters: - branches: - only: - - dev - - yolo - - stage - - deploy-prod: - requires: - - build-test-prod - context: - - sig-go-release - filters: - branches: - only: - - master diff --git a/.dockerignore b/.dockerignore index 294d021..b2fa676 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,6 @@ .git/ .gitignore .vscode/ -CHANNEL LATEST_RPM README.md VERSION diff --git a/.gitignore b/.gitignore index 7e182ce..1815439 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,6 @@ /tests/readwhatwaswritten /tests/rename /tests/trunc -CHANNEL LATEST_RPM Makefile.in VERSION diff --git a/Dockerfile b/Dockerfile index eaa3476..770f218 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,6 @@ RUN \ make \ procps-ng \ rpm-build \ - ruby-devel \ strace \ systemd-devel \ tcpdump \ @@ -74,8 +73,6 @@ RUN \ zlib-devel \ && dnf clean all \ && rm -rf /var/cache/dnf \ - && gem install fpm --no-rdoc --no-ri \ - && gem install package_cloud -v 0.2.45 \ && curl -fsSL https://github.com/pantheon-systems/autotag/releases/latest/download/autotag_linux_amd64 \ -o /usr/local/bin/autotag \ && chmod 0755 /usr/local/bin/autotag diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 1966b51..0000000 --- a/scripts/README.md +++ /dev/null @@ -1,82 +0,0 @@ -Fusedav CI/CD -------------- -The scripts herein are invoked by CircleCI via the circle.yml. They use docker containers to build and package Fusedav for our target platforms (fedora 20/22). - -## Environment Variables -* CIRCLE_BUILD_NUM - Used to determine the build iterator. Defaults to 0 if not present -* BUILD_VERSIONS - a shell array of fedora versions to build for. -* BUILD_DEBUG - If present triggers the build scripts to drop you into a shell for the docker container. - -## Local building of dev packages -Building local is simple, first invoke the version.sh to generate the VERSION and CHANNEL files, and call docker-outer.sh. This assumes you have a working docker setup. - -``` - ./scripts/version.sh - ./scripts/docker-outer.sh -``` - -## Debuging Builds -To get a shell in a docker build container use `BUILD_DEBUG=1` `BUILD_VERSIONS` and invoke `docker-outer.sh` - -``` -BUILD_DEBUG=1 BUILD_VERSIONS=22 ./scripts/docker-outer.sh -==> Running RPM builds for these Fedora version(s): 22 -==> Building rpm for fedora 22 -.... -[root@144a1b05fb05 src]# cat /etc/redhat-release -Fedora release 22 (Twenty Two) -``` - -## Scripts -### version.sh -This will build a VERSION file and CHANNEL file in the root of the project. The autoconf and CI scripts utilize these files for building and for packaging. - -This file is ignored by git, because build-time controls their creation - -### docker-outer.sh -This script is the main outer execution loop. It loops over our desired platforms and executes docker in that environment. The script will mounting the source directory into the docker container for building, and execute `docer-inner.sh` inside the container. - -If you want to get into a container for debugging you can export BUILD_DEBUG with any value to be dropped into an interactive shell on the docker container. - -``` -╭─jnelson@prefect ~/panth/fusedav ‹rpmbuild*› -╰─➤ BUILD_DEBUG=1 ./scripts/docker-outer.sh -==> Running RPM builds for these Fedora version(s): 20 22 -==> Building rpm for fedora 20 -20: Pulling from getpantheon/rpmbuild-fusedav -70568946e5cd: Already exists -03dc8acc8238: Already exists -... -Digest: sha256:b0f4562429925a8be579eb7b86fea8fe8e676b7a962bba8d6bf372fb68b396d7 -Status: Image is up to date for quay.io/getpantheon/rpmbuild-fusedav:20 -Running: docker run --rm -ti -e "build=0" -w /src -v /Users/jnelson/orgs/pantheon/fusedav/scripts/../:/src quay.io/getpantheon/rpmbuild-fusedav:20 /bin/bash -[root@d868541bc6ae src]# -``` - -### docker-inner.sh -This script is build and packaging execution for fusedav. It runs the build and sets up an rpm based on the CHANNEL file. - -It accepts the arguments `channel`, `rpm_dir`, `build_number`, `revision`. Generally this is invoked from `docker_outer.sh` but if you are debugging you can invoke it by hand from inside a build container - -``` -[root@d868541bc6ae src]# ./scripts/docker-inner.sh dev /src/pkg 0 0 - ----------------------------------------------------------------- -Initialized build system. For a common configuration please run: ----------------------------------------------------------------- - -./configure CFLAGS='-g -O0' - -checking for a BSD-compatible install... /usr/bin/install -c -checking whether build environment is sane... yes -checking for a thread-safe mkdir -p... /usr/bin/mkdir -p -... - -Created package {:path=>"fusedav-dev-2.0.1+0-0.a63cceb.x86_64.rpm", :file=>"clamp/command.rb", :line=>"67", :method=>"run"} -``` - -### push_packagecloud.sh -This is a wrapper to push the builds up to package cloud. -Execution is controlled by branches specified in the circle.yml - -It uses `BUILD_VERSIONS` env var to determine the platforms to push, and takes a single argument for the repo to push too `internal` or `internal-staging` diff --git a/scripts/docker-inner.sh b/scripts/docker-inner.sh deleted file mode 100755 index 3a9d147..0000000 --- a/scripts/docker-inner.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh -# -# -set -ex -bin="$(cd -P -- "$(dirname -- "$0")" && pwd -P)" - -if [ "$#" -ne 4 ]; then - echo "specify a channel, rpm dir, build num, and revision" - exit 1 -fi - -fusedav_channel=$1 -rpm_dir=$2 -build=$3 -epoch=$4 - -fedora_release=$(rpm -q --queryformat '%{VERSION}\n' fedora-release) -GITSHA=$(git log -1 --format="%h") -name="fusedav-$fusedav_channel" - -version=$(cat $bin/../VERSION) -iteration=${epoch}.${GITSHA} -arch='x86_64' -url="https://github.com/pantheon-systems/${name}" -vendor='Pantheon' -description='Fusedav: Pantheon fuse-based DAV client' -fusedav_name="fusedav-$fusedav_channel" -install_prefix="/opt/pantheon/$name" - -# If the "/curl-7.4.6.0" directory exists in the build container it means we are using a -# custom build of curl linked with openssl. This is because libcurl on fedora < 27 used NSS instead of openssl -# https://fedoraproject.org/wiki/Changes/libcurlBackToOpenSSL -if [[ -d "/curl-7.46.0" ]]; then - curl_libdir=$install_prefix/libs - - # copy pre-compiled vanilla libcurl into $install_prefix/$name/libs if the curl lib is part of the upstream container - if [ ! -d "$curl_libdir" ]; then - mkdir -p $curl_libdir - fi - cp -R /curl-7.46.0/lib/.libs/* $curl_libdir - - # use our custom curl, and compile fusedav - export CFLAGS="-Wl,-rpath,$curl_libdir,-rpath-link,$curl_libdir -L$curl_libdir -lcurl" -fi - -./autogen.sh -CURL_LIBS="-lcurl" ./configure - -make -make install - -# this could be in the make-install, but for now lets keep the rpm sepparate from the build -if [ ! -d "$install_prefix" ] ; then - mkdir -p $install_prefix -fi - -# test that fusedav at least runs -set +e -/usr/local/bin/fusedav -V -/usr/local/bin/fusedav --help -if [ "1" != "$?" ] ; then - echo "fusedav binary seems broken, failing to continue" - exit 1 -fi -set -e - -mv /usr/local/bin/fusedav $install_prefix/$name -cp $bin/exec_wrapper/mount.fusedav_chan /usr/sbin/mount.$name -chmod 755 /usr/sbin/mount.$name - -fpm -s dir -t rpm \ - --name "${name}" \ - --version "${version}" \ - --iteration "${iteration}" \ - --architecture "${arch}" \ - --url "${url}" \ - --vendor "${vendor}" \ - --description "${description}" \ - --depends uriparser \ - --depends fuse-libs \ - --depends leveldb \ - --log=debug \ - $install_prefix \ - /usr/sbin/mount.$name - -if [ ! -d "$rpm_dir/$fedora_release/fusedav" ] ; then - mkdir -p $rpm_dir/$fedora_release/fusedav -fi - -mv *.rpm $rpm_dir/$fedora_release/fusedav/ diff --git a/scripts/docker-outer.sh b/scripts/docker-outer.sh deleted file mode 100755 index 80342b7..0000000 --- a/scripts/docker-outer.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -set -e -bin="$(cd -P -- "$(dirname -- "$0")" && pwd -P)" -docker=$(which docker) - -# which fedora distros to build this rpm for -BUILD_VERSIONS=${BUILD_VERSIONS:-22 28} - -echo "==> Running RPM builds for these Fedora version(s): $BUILD_VERSIONS" - -RUN_ARGS="--rm" - -# set a default build -> 0 for when it doesn't exist -CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM:-0} - -# location to mount the source in the container -inner_mount="/fusedav" - -echo "==> Creating docker volume for fusedav files" -$docker volume create fusedav_vol -$docker run --name cp-vol -v fusedav_vol:/fusedav busybox true -$docker cp $bin/../. cp-vol:/fusedav/ - -# epoch to use for -revision -epoch=$(date +%s) - -for ver in $BUILD_VERSIONS; do - echo "==> Building rpm for fedora $ver " - - build_image=quay.io/getpantheon/rpmbuild-fusedav:${ver} - $docker pull $build_image - - channel=$(tr -d "\n\r" < $bin/../CHANNEL) - exec_cmd="$inner_mount/scripts/docker-inner.sh $channel $inner_mount/pkg $CIRCLE_BUILD_NUM $epoch" - if [ -n "$BUILD_DEBUG" ] ; then - RUN_ARGS="$RUN_ARGS -ti " - exec_cmd="/bin/bash" - fi - - read docker_cmd <<-EOL - $docker run $RUN_ARGS \ - -e "build=$CIRCLE_BUILD_NUM" \ - -w $inner_mount \ - -v fusedav_vol:$inner_mount \ - $build_image $exec_cmd -EOL - - echo "Running: $docker_cmd" - $docker_cmd - -done - -$docker cp cp-vol:/fusedav/pkg $bin/../pkg/ -$docker rm cp-vol -$docker volume rm fusedav_vol diff --git a/scripts/version.sh b/scripts/version.sh deleted file mode 100755 index 8f6661a..0000000 --- a/scripts/version.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -set -eou pipefail -AUTOTAG_URL=${AUTOTAG_URL:-} - -# ensure we have autotag -if [ ! -d "$HOME/bin" ]; then - mkdir -p ~/bin -fi - -if [ ! -f "$HOME/bin/autotag" ]; then - AUTOTAG_URL=$(curl -silent -o - -L https://api.github.com/repos/pantheon-systems/autotag/releases/latest | grep 'browser_' | grep 'Linux' | cut -d\" -f4 | awk '{print $0}') - # handle the off chance that this wont work with some pre-set version - if [ -z "$AUTOTAG_URL" ] ; then - AUTOTAG_URL="https://github.com/pantheon-systems/autotag/releases/download/v0.0.3/autotag.linux.x86_64" - fi - echo "Pulling $AUTOTAG_URL" - curl -sf -L $AUTOTAG_URL -o ~/bin/autotag > /dev/null - chmod 755 ~/bin/autotag -fi - -if ! grep -q 'email' ~/.gitconfig ; then - git config --global user.email "infrastructure+circleci@getpantheon.com" - git config --global user.name "CI" -fi - - -GITSHA=$(git log -1 --format="%h") - -function gittag { - git tag -a v$VERSION -m "auto release $VERSION" # one day we will parse metadata and add deps here - git push origin --tags -} - -BUILD=${CIRCLE_BUILD_NUM:-} -if [ -z "$BUILD" ] ; then - BUILD=0 -fi - -# tag/autoversion and write out metadata -TAG=$(~/bin/autotag -n) -VERSION=${TAG}+${BUILD} - -# push -case $CIRCLE_BRANCH in -"master") - CHANNEL="release" - gittag - ;; -"stage") - CHANNEL="stage" - ;; -"yolo") - CHANNEL="yolo" - ;; -*) - CHANNEL="dev" - ;; -esac - -echo $CHANNEL > CHANNEL -echo $VERSION > VERSION From 94a4ace4cfba6a06687fb068d021975ee60389c2 Mon Sep 17 00:00:00 2001 From: Doug Schaapveld Date: Fri, 17 Nov 2023 10:49:05 -0600 Subject: [PATCH 3/3] [major] Add GitHub Actions configuration [FS-1505] (#523) - build and publish both Fedora 28 RPM and image - adjust build scripts for GitHub Actions - s/CIRCLE_BRANCH/GITHUB_REF_NAME/ - s/CIRCLE_BUILD_NUM/GITHUB_RUN_NUMBER/ - s/CIRCLE_SHA1/GITHUB_SHA/ - rename RPM to simply `fusedav` (was `fusedav-release`, `fusedav-dev`, `fusedav-stage`, or `fusedav-yolo`) - refactor/relocate computation of fusedav version string - also: add GitHub Actions plugin and mount .netrc within dev container --- .devcontainer/devcontainer.json | 7 ++ .dockerignore | 1 - .github/workflows/build.yml | 95 ++++++++++++++++++++++ .gitignore | 1 - Dockerfile | 55 +++++-------- README.md | 118 +++++++++++++++++++++------ configure.ac | 5 +- fusedav-template.spec | 70 ++++++++++++++++ scripts/build-rpm.sh | 138 +++++++++----------------------- scripts/compute-version.sh | 113 ++++++++++++++++++++++++++ scripts/upload-gh-assets.sh | 25 ++++++ 11 files changed, 459 insertions(+), 169 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 fusedav-template.spec create mode 100755 scripts/compute-version.sh create mode 100644 scripts/upload-gh-assets.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e056762..2a0473d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,6 +10,7 @@ "settings": {}, "extensions": [ "davidanson.vscode-markdownlint", + "github.vscode-github-actions", "ms-azuretools.vscode-docker", "ms-vscode.cpptools-extension-pack", "ms-vscode.makefile-tools", @@ -20,6 +21,7 @@ "settings": {}, "extensions": [ "davidanson.vscode-markdownlint", + "github.vscode-github-actions", "ms-azuretools.vscode-docker", "ms-vscode.cpptools-extension-pack", "ms-vscode.makefile-tools", @@ -27,6 +29,11 @@ ] } }, + // Bind mounting ~/.netrc allows use of the GitHub CLI (`gh`) by running + // `export GH_TOKEN=$(awk '{print $6}' ~/.netrc)` within the dev container. + "mounts": [ + "type=bind,source=${localEnv:HOME}/.netrc,target=/home/vscode/.netrc,readonly" + ], "containerUser": "vscode", "updateRemoteUserUID": true } diff --git a/.dockerignore b/.dockerignore index b2fa676..fb78a7b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,6 @@ .git/ .gitignore .vscode/ -LATEST_RPM README.md VERSION cache/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..a7cbae5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,95 @@ +name: Build everything +on: + - push +env: + REGISTRY: ghcr.io +defaults: + run: + shell: bash + # specifying `bash` here ensures `set -eo pipefail` is active +jobs: + build-everything: + runs-on: ubuntu-latest + permissions: + # contents:write allows creating a GitHub Release. + # packages:write allows publishing an image to GitHub Packages. + contents: write + packages: write + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + # autotag requires a reasonably complete git history. + fetch-depth: 0 + - name: Build fusedav-dev image + run: docker build --progress plain --target dev -t fusedav-dev . + - name: Install autotag + run: | + curl -fsSL https://github.com/pantheon-systems/autotag/releases/latest/download/autotag_linux_amd64 \ + -o /usr/local/bin/autotag + chmod 0755 /usr/local/bin/autotag + - name: Generate new version strings and tag(s) + env: + IMAGE_NAME: ${{ github.repository }} + run: | + echo "new-version.sh:" + scripts/compute-version.sh | tee new-version.sh + - name: Build/tag fusedav image + run: | + echo START build target extract + docker build --progress plain --target extract . --output extract + echo DONE build target extract + echo + echo START build final image + . new-version.sh + # Use older "maintainer" label instead of "org.opencontainers.image.maintainer" + # to overwrite the docker.io/library/fedora:28 value. + docker build --progress plain --target runtime \ + --label "maintainer=nobody@pantheon.io" \ + --label "org.opencontainers.image.description=FUSE-based DAV client with extensions for performance" \ + --label "org.opencontainers.image.licenses=GPLv2" \ + --label "org.opencontainers.image.source=https://github.com/pantheon-systems/fusedav" \ + --label "org.opencontainers.image.vendor=Pantheon Systems, Inc." \ + --label "org.opencontainers.image.version=${SEMVER}" \ + -t fusedav . + echo DONE build final image + for tag in ${IMAGE_TAGS[@]}; do + echo "tag image as ${tag}" + docker tag fusedav $tag + done + - name: Log in to GitHub Container Registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push final image/tag(s) to container registry + run: | + . new-version.sh + for tag in ${IMAGE_TAGS[@]}; do + echo "push ${tag}" + docker push $tag + done + - name: Create (pre-release) GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + . new-version.sh + # If NOT master, create a pre-release. + # Note that GitHub converts the tilde (`~`) which indicates a pre-release + # RPM version to a period (`.`) in the filename. This should not affect + # RPM version comparison operations. + echo "Create pre-release release:" + gh release create $GITHUB_RELEASE_NAME -p --generate-notes --target $GITHUB_REF_NAME + . scripts/upload-gh-assets.sh + if: github.ref != 'refs/heads/master' + - name: Create GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + . new-version.sh + # Iff master, create a regular release. + echo "Create regular release:" + gh release create $GITHUB_RELEASE_NAME --generate-notes + . scripts/upload-gh-assets.sh + if: github.ref == 'refs/heads/master' diff --git a/.gitignore b/.gitignore index 1815439..3652209 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,6 @@ /tests/readwhatwaswritten /tests/rename /tests/trunc -LATEST_RPM Makefile.in VERSION _trial_temp* diff --git a/Dockerfile b/Dockerfile index 770f218..e5ec20b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,6 @@ # # runtime is the final runtime image. # -# To build image locally: -# docker build --progress plain --build-arg CIRCLE_SHA1=$(git rev-parse --short HEAD) -t fusedav . -# -# To compile/build and extract the RPM into the `extract` directory: -# docker build --progress plain --build-arg GITHUB_SHA=$(git rev-parse --short HEAD) --target extract -t fusedav-extract . --output=extract -# FROM docker.io/library/fedora:28 AS base SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -50,6 +44,7 @@ FROM base AS dev RUN \ dnf install -y \ + 'dnf-command(config-manager)' \ autoconf \ automake \ bind-utils \ @@ -71,6 +66,8 @@ RUN \ tcpdump \ uriparser-devel \ zlib-devel \ + && dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo \ + && dnf install -y gh \ && dnf clean all \ && rm -rf /var/cache/dnf \ && curl -fsSL https://github.com/pantheon-systems/autotag/releases/latest/download/autotag_linux_amd64 \ @@ -80,34 +77,19 @@ RUN \ # Installing autotag above makes it available within a dev container. # When building via CI/CD, autotag is installed/called elsewhere. +# Installing gh above makes it available within a dev container. +# When building via GitHub Actions, gh is installed/called elsewhere. + USER vscode ######################################## FROM dev AS compile +# new-version.sh MUST be created before we get here COPY . /build WORKDIR /build -ARG CIRCLE_BRANCH="unknown" -ARG CIRCLE_BUILD_NUM="" -ARG CIRCLE_SHA1=0000000 - -# CHANNEL is always `release` now. -# Historically, CHANNEL could be: dev, stage, yolo, release -ARG CHANNEL=release - -# Set PACKAGECLOUD_REPO to `internal` or `internal-staging` to publish RPM. -ARG PACKAGECLOUD_REPO="" - -# RPM_VERSION is set here for local/direct `docker build` use; CircleCI builds -# will set their own value. -ARG RPM_VERSION="0.0.0+0" - -# SEMVER is set here for local/direct `docker build` use; CircleCI builds -# will set their own value. -ARG SEMVER="0.0.0-local" - # Using explicit USER instructions instead of sudo to satisfy Guardrails. USER root @@ -117,27 +99,28 @@ RUN \ USER vscode RUN \ - echo "${RPM_VERSION}" > VERSION \ - && scripts/build-rpm.sh "${CHANNEL}" \ - && if [ -n "${PACKAGECLOUD_REPO}" ] ; then \ - echo SKIPPING scripts/push_packagecloud.sh ; \ - else \ - echo "NOT pushing RPM to Packagecloud as this is a pre-release build" ; \ - fi + scripts/build-rpm.sh ######################################## FROM scratch AS extract -COPY --from=compile /build/pkg pkg +COPY --from=compile /home/vscode/rpmbuild/RPMS RPMS +COPY --from=compile /home/vscode/rpmbuild/SRPMS SRPMS +COPY --from=compile /build/LATEST-RPM-VER-REL LATEST-RPM-VER-REL ######################################## FROM base AS runtime -ARG CHANNEL=release +COPY --from=compile \ + /build/LATEST-RPM-VER-REL \ + /home/vscode/rpmbuild/RPMS/x86_64/fusedav-*.rpm \ + /tmp/ -COPY --from=compile /build/src/fusedav "/opt/pantheon/fusedav-${CHANNEL}/fusedav-${CHANNEL}" -COPY scripts/exec_wrapper/mount.fusedav_chan "/usr/sbin/mount.fusedav-${CHANNEL}" +# BEWARE: `.fc28` is the RPM release suffix normally added by rpmbuild. +RUN \ + LATEST=$(cat /tmp/LATEST-RPM-VER-REL) \ + && rpm -i "/tmp/fusedav-${LATEST}.fc28.x86_64.rpm" USER fusedav diff --git a/README.md b/README.md index ad0c79d..c89984e 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,101 @@ -Fusedav -======= +# Fusedav [![Unsupported](https://img.shields.io/badge/Pantheon-Unsupported-yellow?logo=pantheon&color=FFDC28)](https://pantheon.io/docs/oss-support-levels#unsupported) -Fusedav Client --------------- +## Fusedav Client This is a fuse-based DAV client with extensions for performance. This will run against any standard DAV implementation, i.e. `pywebdav` (see tests). -Installation ------------- +## Production (CI/CD) Build -1. ```git clone git://github.com/pantheon-systems/fusedav.git``` -2. ```git clean -f -x -d && ./autogen.sh && ./configure && make``` +Merging anything to the `master` branch will trigger GitHub Actions to build +a production release. -Usage ------ +The RPM(s) are published as a [Release](https://github.com/pantheon-systems/fusedav/releases). -Use the ```-V``` flag to see libraries. +Container image(s) are published as [Packages](https://github.com/pantheon-systems/fusedav/pkgs/container/fusedav). +Images are generally accessed as `ghcr.io/pantheon-systems/fusedav:0.0.0`, where +`0.0.0` is the desired version. + +### RPM Retrieval / Installation + +Note that, while the RPM version is removed from the filename entirely, the +RPM release `.fc28` suffix is retained. This is being done to allow for use +of other base images/distributions in the future. + +```sh +# Install latest release +curl -fsSL https://github.com/pantheon-systems/fusedav/releases/latest/download/fusedav.fc28.x86_64.rpm \ + -o fusedav.x86_64.rpm \ + && dnf install -y fusedav.x86_64.rpm + +# Install specific release +RELEASE_NAME=v0.0.0-branch.1 +curl -fsSL "https://github.com/pantheon-systems/fusedav/releases/download/${RELEASE_NAME}/fusedav.fc28.x86_64.rpm" \ + -o fusedav.x86_64.rpm \ + && dnf install -y fusedav.x86_64.rpm + +# Install specific release, alternate method +RELEASE_NAME=v0.0.0-branch.1 +curl -fsSL "https://api.github.com/repos/pantheon-systems/fusedav/releases/tags/${RELEASE_NAME}" \ + | jq ".assets[] | select(.name==\"fusedav.fc28.x86_64.rpm\") | .browser_download_url" \ + | xargs curl -o fusedav.x86_64.rpm -fsSL \ + && dnf install -y fusedav.x86_64.rpm +``` + +## Development Build + +### Development Image Build + +Building a local container image requires only: + +1) a local Docker (or Podman) installation, and +1) [autotag](https://github.com/pantheon-systems/autotag) in the PATH. +PATH. + +```sh +scripts/compute-version.sh | tee new-version.sh +docker build --progress plain -t fusedav . ``` + +### Development RPM Build + +Building a set of RPMs within a containerized environment requires only: + +1) a local Docker (or Podman) installation, and +1) [autotag](https://github.com/pantheon-systems/autotag) in the PATH. + +The generated RPMs will be written to the `extract` directory. + +```sh +scripts/compute-version.sh | tee new-version.sh +docker build --progress plain --target extract . --output=extract +``` + +### Development Code Build + +1. Clone the git repository. + - You may have done this already. + - `git clone git://github.com/pantheon-systems/fusedav.git` +1. Install build dependencies. + - See `BuildRequires` in [fusedav-template.spec](fusedav-template.spec) + for required Fedora Linux 28 packages. + - You may have done this already. + - ALTERNATIVELY, open this git repository in Visual Studio Code with + the [ms-vscode-remote.remote-containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + extension active and work within the `fedora28` (default) dev + container. +1. Compile the code. + - `git clean -f -x -d && ./autogen.sh && ./configure && make` + - The executable will be written to `src/fusedav`. + +## Usage + +Use the `-V` flag to see libraries. + +```text $ src/fusedav -V fusedav version 2.0.42-bccf93b LevelDB version 1.20 @@ -28,25 +103,16 @@ libcurl/7.59.0 OpenSSL/1.1.0i zlib/1.2.11 libidn2/2.0.5 libpsl/0.20.2 (+libidn2/ FUSE library version: 2.9.7 ``` -Debug/Develop ------ -Running this docker script in debug mode will build a fedora-22 container with the local source mounted inside it suitable to build fusedav. -``` -BUILD_VERSIONS=22 BUILD_DEBUG=1 ./scripts/docker-outer.sh -``` - -libcurl and OpenSSL -------------------- +## libcurl and OpenSSL FuseDAV requires libcurl linked with OpenSSL. On Fedora versions before 27 the provided libcurl is linked against NSS and you need to provide your own libcurl linked against OpenSSL. -Contributing ------------- +## Contributing 1. Fork it. -2. Create a branch (`git checkout -b my_new_features`) -3. Commit your changes (`git commit -am "Adding a nice new feature"`) -4. Push to the branch (`git push origin my_new_feature`) -5. Open a Pull Request with relevant information +2. Create a branch (`git checkout -b my_new_features`). +3. Commit your changes (`git commit -am "Adding a nice new feature"`). +4. Push to the branch (`git push origin my_new_feature`). +5. Open a Pull Request with relevant information. diff --git a/configure.ac b/configure.ac index ee47438..7bc994c 100644 --- a/configure.ac +++ b/configure.ac @@ -19,10 +19,7 @@ AC_PREREQ([2.68]) -define([gitcommit], esyscmd([sh -c "git rev-parse --short HEAD|tr -d '\n'"]))dnl - -AC_DEFINE(GIT_COMMIT, "gitcommit", [Git Commit]) -AC_INIT([fusedav],m4_esyscmd([tr -d '\n' < VERSION])-[gitcommit],[david@davidstrauss.net]) +AC_INIT([fusedav],m4_esyscmd([tr -d '\n' < VERSION]),[david@davidstrauss.net]) AC_CONFIG_SRCDIR([src/fusedav.c]) AC_CONFIG_HEADERS([config.h]) AM_INIT_AUTOMAKE([foreign 1.11 -Wall]) diff --git a/fusedav-template.spec b/fusedav-template.spec new file mode 100644 index 0000000..395b35c --- /dev/null +++ b/fusedav-template.spec @@ -0,0 +1,70 @@ +Name: fusedav +Version: RPM_VERSION +Release: RPM_RELEASE%{?dist} +License: GPLv2 +URL: https://github.com/pantheon-systems/fusedav +Vendor: Pantheon +Summary: Fusedav: Pantheon fuse-based DAV client +Source: fusedav-RPM_VERSION.tar.gz + +BuildRequires: autoconf +BuildRequires: automake +BuildRequires: curl-devel +BuildRequires: expat-devel +BuildRequires: fuse-devel +BuildRequires: gcc +BuildRequires: glib2-devel +BuildRequires: jemalloc-devel +BuildRequires: leveldb-devel +BuildRequires: make +BuildRequires: systemd-devel +BuildRequires: uriparser-devel +BuildRequires: zlib-devel + +Requires: fuse +Requires: fuse-libs +Requires: jemalloc +Requires: leveldb +Requires: uriparser + +%description +fusedav is a fuse-based DAV client with extensions for performance. + +%prep +%setup + +%build +./autogen.sh +echo RPM_BUILD_ROOT = $RPM_BUILD_ROOT +CURL_LIBS="-lcurl" ./configure --prefix="${RPM_BUILD_ROOT}" \ + --bindir="/opt/pantheon/fusedav-release" +make + +%install +%make_install + +# HACK: rename binary +mv "${RPM_BUILD_ROOT}/opt/pantheon/fusedav-release/fusedav" \ + "${RPM_BUILD_ROOT}/opt/pantheon/fusedav-release/fusedav-release" + +mkdir -p "${RPM_BUILD_ROOT}/usr/sbin" +install -m 0755 scripts/exec_wrapper/mount.fusedav_chan \ + "${RPM_BUILD_ROOT}/usr/sbin/mount.fusedav-release" + +%check + +# test that fusedav at least runs +echo "TEST OUTPUT: fusedav -V" +set +e +"${RPM_BUILD_ROOT}/opt/pantheon/fusedav-release/fusedav-release" -V +if [ "$?" != "0" ] ; then + echo + echo "fusedav binary seems broken, aborting" + exit 1 +fi +set -e + +%files +%defattr(0644, root, root, 0755) +%attr(0755, root, root) /opt/pantheon/fusedav-release/fusedav-release +%attr(0755, root, root) /usr/sbin/mount.fusedav-release diff --git a/scripts/build-rpm.sh b/scripts/build-rpm.sh index 1c52170..3e88021 100755 --- a/scripts/build-rpm.sh +++ b/scripts/build-rpm.sh @@ -5,119 +5,55 @@ # # It should be started from the root of the repo. # -# Upon success, the fusedav RPM will be written to the pkg/28/fusedav -# directory. -# -# RPM version string creation is a bit weird. The VERSION file is expected to -# contain something like `0.0.1+3`, where 0.0.1 is the proper version and 3 -# is the build number (which should be populated from CIRCLE_BUILD_NUM -# upstream). -# -# `iteration` is the timestamp (in seconds since the epoch) followed by a -# period and the 7-character git commit hash. -# -# All together, the RPM version/iteration string ends up looking like -# `0.0.1+3-1698000000.abc1234`. -# -# Note that the RPM version and iteration strings are SEPARATE from the -# SemVer string embedded within the fusedav binary. -# set -euo pipefail -if [ "$#" -ne 1 ]; then - echo "specify a channel" +if [ ! -r new-version.sh ] ; then + echo "new-version.sh is missing, try: \"scripts/compute-version.sh | tee new-version.sh\"" exit 1 fi -if [ ! -r VERSION ] ; then - echo "VERSION file is missing; cannot continue" -fi - -# fusedav_channel may be: dev, stage, yolo, release -fusedav_channel=$1 - -# build vars from environment -fedora_release=$(rpm -q --queryformat '%{VERSION}\n' fedora-release) -name="fusedav-${fusedav_channel}" -version=$(cat VERSION) - -arch=$(uname -m) # typically x86_64 -url="https://github.com/pantheon-systems/${name}" -vendor='Pantheon' -description='Fusedav: Pantheon fuse-based DAV client' - -# Note that the ".fc28" RPM release/iteration suffix is arbitrarily -# added here. Provision should be made for using a different string -# if/when building on other distributions/versions. -if [ -n "${CIRCLE_SHA1:-}" ]; then - iteration="$(date +%s).$(echo $CIRCLE_SHA1 | cut -c -7).fc28" -else - iteration="$(date +%s).$(git rev-parse --short HEAD).fc28" -# if [ -n "$(git status --porcelain)" ] ; then -# iteration="${iteration}-dirty" -# fi -fi +eval $(cat new-version.sh) -# rpm_build_root is used by `make install` to write fusedav. -rpm_build_root="${HOME}/fusedav_build_root" +# Capture latest RPM version/release to simplify later RPM extraction. +# +# Note this does NOT include the `%{?dist}` suffix that rpmbuild appends +# to its release. +echo "${RPM_VERSION}-${RPM_RELEASE}" > LATEST-RPM-VER-REL -# install_prefix is the final home of the fusedav binary, relative to rpm_build_root. -install_prefix="opt/pantheon/${name}" +# Create VERSION for autoconf. +echo "${SEMVER}" > VERSION -# start the build -./autogen.sh -mkdir -p $rpm_build_root -CURL_LIBS="-lcurl" ./configure --prefix="${rpm_build_root}" -make -make install +mkdir -p "${HOME}/rpmbuild/SOURCES" "${HOME}/rpmbuild/SPECS" -# test that fusedav at least runs -set +e -fusedav_bin="${rpm_build_root}/bin/fusedav" -echo -echo "TEST OUTPUT: fusedav -V" -"${fusedav_bin}" -V -echo -echo "TEST OUTPUT: fusedav --help" -"${fusedav_bin}" --help -if [ "$?" != "1" ] ; then - echo - echo "fusedav binary at ${fusedav_bin} seems broken, aborting" +spec_path="${HOME}/rpmbuild/SPECS/fusedav.spec" +if [ -e $spec_path ] ; then + echo "Cowardly refusing to overwrite ${spec_path}" exit 1 fi -echo -set -e -mkdir -p "${rpm_build_root}/${install_prefix}" -mv $fusedav_bin "${rpm_build_root}/${install_prefix}" - -mkdir -p "${rpm_build_root}/usr/sbin" -install -m 0755 scripts/exec_wrapper/mount.fusedav_chan "${rpm_build_root}/usr/sbin/mount.${name}" +# build vars from environment +fedora_release=$(rpm -q --queryformat '%{VERSION}\n' fedora-release) -# fpm will not clobber, so ensure file is not present -#rpm_version=$(echo $version | sed -e 's/-/_/g') -#rpm_iteration=$(echo $iteration | sed -e 's/-/_/g') -rpm_target="${name}-${version}-${iteration}.${arch}.rpm" -rm -f $rpm_target +echo "CREATE SOURCE ARCHIVE" +tar czf "${HOME}/rpmbuild/SOURCES/fusedav-${RPM_VERSION}.tar.gz" \ + --transform "s,^,fusedav-${RPM_VERSION}/," \ + LICENSE \ + Makefile.am \ + VERSION \ + autogen.sh \ + configure.ac \ + scripts/exec_wrapper/mount.fusedav_chan \ + src/*.c \ + src/*.h \ + src/Makefile.am + +sed -e "s/RPM_VERSION/${RPM_VERSION}/" \ + -e "s/RPM_RELEASE/${RPM_RELEASE}/" \ + fusedav-template.spec \ + > $spec_path echo "BUILD RPM" -fpm -s dir -t rpm \ - --name "${name}" \ - --version "${version}" \ - --iteration "${iteration}" \ - --architecture "${arch}" \ - --url "${url}" \ - --vendor "${vendor}" \ - --description "${description}" \ - --depends uriparser \ - --depends fuse-libs \ - --depends leveldb \ - --log=info \ - --chdir=$rpm_build_root \ - $install_prefix \ - "usr/sbin/mount.${name}" - -mkdir -p "pkg/${fedora_release}/fusedav" -mv $rpm_target "pkg/${fedora_release}/fusedav/" -echo "${rpm_target}" > LATEST_RPM -echo "fusedav RPM written to pkg/${fedora_release}/fusedav/${rpm_target}" +echo "SEMVER=${SEMVER}" +echo "RPM_VERSION=${RPM_VERSION}" +echo "RPM_RELEASE=${RPM_RELEASE}" +rpmbuild -ba $spec_path diff --git a/scripts/compute-version.sh b/scripts/compute-version.sh new file mode 100755 index 0000000..a9ab266 --- /dev/null +++ b/scripts/compute-version.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# +# compute-version.sh generates version strings and image tags. +# +# See end of this file for format specifics, including examples. +# +# When run from GitHub Actions, the GITHUB_* variables will be used. +# +# When run locally (perhaps within a dev container), the GITHUB_* variables +# are not available, but `.git/` should exist so we can get the necessary +# information ourselves. +# +set -eou pipefail + +if [ -n "${REGISTRY:-}" ] ; then + IMAGE_REPOSITORY="${REGISTRY}/${IMAGE_NAME:-fusedav}" +else + # if no REGISTRY, build image for local use + IMAGE_REPOSITORY="${IMAGE_NAME:-fusedav}" +fi + +# Ensure we have autotag. +if ! command -v autotag > /dev/null ; then + echo "autotag is not available; aborting" > /dev/stderr + exit 1 +fi + +# Ensure we have build_num. +build_num=${GITHUB_RUN_NUMBER:-} +if [ -z "$build_num" ] ; then + echo "GITHUB_RUN_NUMBER not set, assuming '1'" > /dev/stderr + build_num=1 +fi + +# Ensure we have branch name. +raw_branch=${GITHUB_REF_NAME:-} +if [ -z "${raw_branch}" ] ; then + echo "GITHUB_REF_NAME not set, extracting from git directly" > /dev/stderr + raw_branch=$(git rev-parse --abbrev-ref HEAD) +fi + +# Sanitize branch name. +# +# Yes, this is overly restrictive. The git rules for naming branches are +# quite complex. +# https://git-scm.com/docs/git-check-ref-format +safe_branch=$(sed -e 's/[^0-9A-Za-z-]/-/g' <<<$raw_branch) + +# Ensure we have git commit hash. +if [ -n "${GITHUB_SHA:-}" ]; then + git_commit="$(echo $GITHUB_SHA | cut -c -7)" +else + echo "GITHUB_SHA not set, extracting from git directly" > /dev/stderr + git_commit="$(git rev-parse --short HEAD)" + if [ -n "$(git status --porcelain)" ] ; then + git_commit="${git_commit}-dirty" + fi +fi + +# Compute new SemVer string and image tag(s). +# +# $raw_branch is being directly appended in some places to avoid shell +# interpolation issues. +# (Example: Consider a branch name which contains a single quote, double +# quote, and/or semicolon.) +IMAGE_TAGS=( "${IMAGE_REPOSITORY}:${build_num}-"$safe_branch ) +if [ "${safe_branch}" == "main" -o "${safe_branch}" == "master" ] ; then + full_semver="$(autotag -n -m "${build_num}.${git_commit}" -b $raw_branch)" + + # Originally, the actual `vN.N.N` git release tag was created by this + # invocation of autotag. However, `gh` appears to do that on its own + # when creating a release, so we now include `-n` here to suppress + # the generation of a potentially confusing/conflicting git tag. + bare_ver="$(autotag -n -b $raw_branch)" + + GITHUB_RELEASE_NAME="v${bare_ver}" + rpm_ver=$bare_ver + + # When main/master, also create SemVer tag. + IMAGE_TAGS+=( "${IMAGE_REPOSITORY}:${bare_ver}" ) +else + full_semver="$(autotag -n -p "${safe_branch}.${build_num}" -m $git_commit -b $raw_branch)" + bare_ver=$(autotag -n -b $raw_branch) + GITHUB_RELEASE_NAME="v$(autotag -n -p "${safe_branch}.${build_num}" -b $raw_branch)" + rpm_ver="${bare_ver}~${safe_branch}" +fi + +# In the examples below: +# - latest tag on the git repo is v0.0.1. +# - GITHUB_REF_NAME is fix-bug. +# - GITHUB_RUN_NUMBER is 3. +# - GITHUB_SHA is abc1234fffff. + +# iff main/master: v0.0.2 +# else: v0.0.2-fix-bug.3 +echo "export GITHUB_RELEASE_NAME=${GITHUB_RELEASE_NAME}" + +# image tag(s) to be pushed to registry/repository +# - always: ...:3-fix-bug +# - iff main/master: also ...:0.0.2 +echo "export IMAGE_TAGS=( ${IMAGE_TAGS[@]} )" # keep args separate + +# iff main/master: 0.0.2+3.abc1234 +# else: 0.0.2-fix-bug.3+abc1234 +echo "export SEMVER=${full_semver}" + +# iff main/master: RPM_VERSION=0.0.2 RPM_RELEASE=1 +# else: RPM_VERSION=0.0.2~fix_bug RPM_RELEASE=1 +# +# Originally, we set RPM_RELEASE to $build_num; it is unclear if this is +# useful or merely distracting. +echo "export RPM_VERSION=${rpm_ver//-/_}" +echo "export RPM_RELEASE=1" diff --git a/scripts/upload-gh-assets.sh b/scripts/upload-gh-assets.sh new file mode 100644 index 0000000..6bdb278 --- /dev/null +++ b/scripts/upload-gh-assets.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eou pipefail + +# upload-gh-assets.sh is meant to be called automatically by Github Actions. +# Before calling this script, new-version.sh MUST already be loaded into our +# env. + +T=$(mktemp -d) + +# Upload artifacts with static/fixed names for ease of consumption. +for F in extract/RPMS/x86_64/*.rpm ; do + n=$(echo $F | sed -e "s,.*/\([0-9A-Za-z-]*\)-${RPM_VERSION}-${RPM_RELEASE}\(.[0-9A-Za-z]*.x86_64.rpm\),\1\2,") + echo "include ${F} as ${n}" + cp $F "${T}/${n}" +done +for F in extract/SRPMS/*.rpm ; do + n=$(echo $F | sed -e "s,.*/\([0-9A-Za-z-]*\)-${RPM_VERSION}-${RPM_RELEASE}\(.[0-9A-Za-z]*.src.rpm\),\1\2,") + echo "include ${F} as ${n}" + cp $F "${T}/${n}" +done + +gh release upload $GITHUB_RELEASE_NAME ${T}/*.rpm +echo Done uploading release artifacts + +rm -rf $T