From 92ba5b97bc45dcbec096aec22112b72aeee1cdfc Mon Sep 17 00:00:00 2001 From: Michal Rostecki Date: Sat, 21 Sep 2024 09:15:57 +0200 Subject: [PATCH] ci: Build statically linked bpf-linker in CI Build LLVM and bpf-linker with statically linked libLLVM for multiple architectures. Include the resulting build artifacts on the release pages, so they can be installed with `cargo binstall` --- .github/workflows/ci.yml | 83 ++++++++++++++----------- .github/workflows/llvm.yml | 113 ++++++++++++++++++++++++---------- .github/workflows/release.yml | 54 +++++++++++++++- Cargo.lock | 4 +- Cargo.toml | 1 + 5 files changed, 184 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8f124d4..ecdf0d68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: run: cargo fmt --all -- --check build: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.target.os }} strategy: fail-fast: false matrix: @@ -56,13 +56,50 @@ jobs: - stable - beta - nightly - llvm: - - 19 - - source + target: + - os: macos-13 + target: aarch64-apple-darwin + target-llvm: aarch64-apple-darwin + - os: macos-13 + target: x86_64-apple-darwin + target-llvm: x86_64-apple-darwin + # We use the GNU builds of LLVM both for GNU and musl builds of + # bpf-linker - it doesn't affect the type of libc being picked by + # Cargo and musl Rust toolchains come with their own copies of + # libc.a. + - os: ubuntu-22.04 + packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross + target: aarch64-unknown-linux-gnu + target-llvm: aarch64-linux-gnu + - os: ubuntu-22.04 + packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross + target: aarch64-unknown-linux-musl + target-llvm: aarch64-linux-gnu + - os: ubuntu-22.04 + packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross + target: riscv64gc-unknown-linux-gnu + target-llvm: riscv64-linux-gnu + - os: ubuntu-22.04 + packages: + target: x86_64-unknown-linux-gnu + target-llvm: x86_64-linux-gnu + - os: ubuntu-22.04 + packages: + target: x86_64-unknown-linux-musl + target-llvm: x86_64-linux-gnu name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }} needs: llvm env: + CARGO_BUILD_TARGET: ${{ matrix.target.target }} + # We use the GNU sysroot as an LD path both for GNU and musl builds of + # bpf-linker - the user-space emulator and the path are used only for + # executing `llvm-config` and test binaries. + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu + CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/riscv64-linux-gnu + CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/riscv64-linux-gnu + LLVM_SYS_CROSS_LD_PREFIX: /usr/${{ matrix.target.target-llvm }} RUST_BACKTRACE: full steps: @@ -80,8 +117,7 @@ jobs: with: toolchain: ${{ matrix.rust }} components: rust-src - # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. - targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl + targets: ${{ matrix.target.target }} - uses: Swatinem/rust-cache@v2 @@ -92,7 +128,7 @@ jobs: run: cargo build - name: Install dependencies - if: matrix.rust == 'nightly' + if: runner.os == 'Linux' # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit # enum values which was added in clang 15[1]. # @@ -108,42 +144,19 @@ jobs: wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt update - sudo apt -y install clang gcc-multilib - - - name: Install LLVM - if: matrix.llvm != 'source' - run: | - set -euxo pipefail - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc - echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list - - sudo apt update - # TODO(vadorovsky): Remove the requirement of libpolly. - # - # Packages from apt.llvm.org are being built all at once, with one - # cmake build with superset of options, then different binaries and - # libraries are being included in different packages. - # - # That results in `llvm-config --libname --link-static` mentioning - # libpolly, even if it's not installed. The output of that command is - # being used in build.rs of llvm-sys, so building llvm-sys on such - # system is complaining about lack of libpolly. - # - # Hopefully that nightmare goes away once we switch to binstalls and - # ditch the system LLVM option. - sudo apt -y install llvm-${{ matrix.llvm }}-dev libpolly-${{ matrix.llvm }}-dev - echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH + sudo apt -y install \ + clang \ + gcc-multilib \ + ${{ matrix.target.packages }} - name: Restore LLVM - if: matrix.llvm == 'source' uses: actions/cache/restore@v4 with: path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} + key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target.target-llvm)] }} fail-on-cache-miss: true - name: Add LLVM to PATH && LD_LIBRARY_PATH - if: matrix.llvm == 'source' run: | set -euxo pipefail echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index bef08b21..6e9d8ab3 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -3,13 +3,52 @@ name: LLVM on: workflow_call: outputs: - cache-key: - value: ${{ jobs.llvm.outputs.cache-key }} + cache-key-aarch64-apple-darwin: + value: ${{ jobs.llvm.outputs.cache-key-aarch64-apple-darwin }} + cache-key-x86_64-apple-darwin: + value: ${{ jobs.llvm.outputs.cache-key-x86_64-apple-darwin }} + cache-key-aarch64-linux-gnu: + value: ${{ jobs.llvm.outputs.cache-key-aarch64-linux-gnu }} + cache-key-riscv64-linux-gnu: + value: ${{ jobs.llvm.outputs.cache-key-riscv64-linux-gnu }} + cache-key-x86_64-linux-gnu: + value: ${{ jobs.llvm.outputs.cache-key-x86_64-linux-gnu }} jobs: llvm: - runs-on: ubuntu-22.04 - name: llvm + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-13 + os-name: macOS + processor: arm64 + system: Darwin + target: aarch64-apple-darwin + - os: macos-13 + os-name: macOS + processor: x86_64 + system: Darwin + target: x86_64-apple-darwin + - os: ubuntu-22.04 + os-name: Linux + packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross + processor: aarch64 + system: Linux + target: aarch64-linux-gnu + - os: ubuntu-22.04 + os-name: Linux + packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross + processor: riscv64 + system: Linux + target: riscv64-linux-gnu + - os: ubuntu-22.04 + os-name: Linux + packages: + processor: x86_64 + system: Linux + target: x86_64-linux-gnu + name: llvm ${{ matrix.target }} outputs: cache-key: ${{ steps.cache-key.outputs.cache-key }} steps: @@ -20,18 +59,18 @@ jobs: echo "sha=$value" >> "$GITHUB_OUTPUT" - id: cache-key - run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" + run: echo "cache-key-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" - name: Cache id: cache-llvm uses: actions/cache@v4 with: path: llvm-install - key: ${{ steps.cache-key.outputs.cache-key }} + key: ${{ steps.cache-key.outputs[format('cache-key-{0}', matrix.target)] }} lookup-only: true - name: Install Tools - if: steps.cache-llvm.outputs.cache-hit != 'true' + if: runner.os == 'Linux' && steps.cache-llvm.outputs.cache-hit != 'true' run: | set -euxo pipefail wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ @@ -41,7 +80,31 @@ jobs: sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null sudo apt update - sudo apt -y install cmake ninja-build clang lld + sudo apt -y install \ + cmake \ + ninja-build \ + ${{ matrix.packages }} + + - name: Install Tools + if: runner.os == 'macOS' && steps.cache-llvm.outputs.cache-hit != 'true' + run: | + set -euxo pipefail + brew update + brew install ninja + + - name: Install LLVM + # We use the latest LLVM binaries to build our LLVM fork. + run: | + set -euxo pipefail + llvm_tarball_url=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + https://api.github.com/repos/llvm/llvm-project/releases | jq -r \ + 'first | .assets | map(select(.name | + endswith("${{ matrix.os-name }}-X64.tar.xz"))) | first | + .browser_download_url') + mkdir -p /tmp/llvm-upstream + wget -q -O - $llvm_tarball_url | tar -xJ --strip-components 1 \ + -C /tmp/llvm-upstream + echo /tmp/llvm-upstream/bin >> $GITHUB_PATH - name: Checkout LLVM Source if: steps.cache-llvm.outputs.cache-hit != 'true' @@ -60,13 +123,21 @@ jobs: -B llvm-build \ -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_ASM_COMPILER=clang \ + -DCMAKE_ASM_COMPILER_TARGET="${{ matrix.target }}" \ -DCMAKE_C_COMPILER=clang \ + -DCMAKE_C_COMPILER_TARGET="${{ matrix.target }}" \ -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_CXX_COMPILER_TARGET="${{ matrix.target }}" \ + -DCMAKE_LINKER="lld" \ -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \ + -DCMAKE_SYSTEM_NAME="${{ matrix.system }}" \ + -DCMAKE_SYSTEM_PROCESSOR="${{ matrix.processor }}" \ -DLLVM_BUILD_LLVM_DYLIB=ON \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_ENABLE_PROJECTS= \ -DLLVM_ENABLE_RUNTIMES= \ + -DLLVM_HOST_TRIPLE="${{ matrix.target }}" \ -DLLVM_INSTALL_UTILS=ON \ -DLLVM_LINK_LLVM_DYLIB=ON \ -DLLVM_TARGETS_TO_BUILD=BPF \ @@ -74,28 +145,8 @@ jobs: - name: Install LLVM if: steps.cache-llvm.outputs.cache-hit != 'true' - env: - # Create symlinks rather than copies to conserve disk space. At the time of this writing, - # GitHub-hosted runners have 14GB of SSD space - # (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources). - # - # Since the LLVM build creates a bunch of symlinks (and this setting does not turn those - # into symlinks-to-symlinks), use absolute symlinks so we can distinguish the two cases. - CMAKE_INSTALL_MODE: ABS_SYMLINK run: cmake --build llvm-build --target install - - name: Rewrite LLVM Symlinks - if: steps.cache-llvm.outputs.cache-hit != 'true' - # Move targets over the symlinks that point to them. - # - # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE. - run: | - set -euxo pipefail - find llvm-install -type l -execdir sh -eux -c ' - for link in "$@"; do - target=$(readlink "$link") - case $target in - /*) mv "$target" "$link" ;; - esac - done - ' sh {} + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cb4c1c3..8385b302 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,15 +9,63 @@ jobs: uses: ./.github/workflows/llvm.yml upload-bins: - # TODO: Build for macos someday. - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: macos-13 + target: aarch64-apple-darwin + target-llvm: aarch64-apple-darwin + - os: macos-13 + target: x86_64-apple-darwin + target-llvm: x86_64-apple-darwin + - os: ubuntu-22.04 + packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross + target: aarch64-unknown-linux-gnu + target-llvm: aarch64-linux-gnu + - os: ubuntu-22.04 + packages: gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross + target: aarch64-unknown-linux-musl + target-llvm: aarch64-linux-gnu + - os: ubuntu-22.04 + packages: gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libc6-dev-riscv64-cross + target: riscv64-unknown-linux-gnu + target-llvm: riscv64-linux-gnu + - os: ubuntu-22.04 + packages: + target: x86_64-unknown-linux-gnu + target-llvm: x86_64-linux-gnu + - os: ubuntu-22.04 + packages: + target: x86_64-unknown-linux-musl + target-llvm: x86_64-linux-gnu needs: llvm + + env: + CARGO_BUILD_TARGET: ${{ matrix.target }} + # We use the GNU sysroot as an LD path both for GNU and musl builds of + # bpf-linker - the user-space emulator and the path are used only for + # executing `llvm-config` and test binaries. + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu + CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu + CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 -L /usr/aarch64-linux-gnu + LLVM_SYS_CROSS_LD_PREFIX: /usr/${{ matrix.target-llvm }} + RUST_BACKTRACE: full + steps: + - name: Install Tools + if: runner.os == 'Linux' + run: | + set -euxo pipefail + sudo apt update + sudo apt -y install ${{ matrix.packages }} + - name: Restore LLVM uses: actions/cache/restore@v4 with: path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} + key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target-llvm)] }} fail-on-cache-miss: true - name: Add LLVM to PATH diff --git a/Cargo.lock b/Cargo.lock index f4ee1740..eef62b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -448,8 +448,7 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "llvm-sys" version = "191.0.0-rc1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc981f56df5430a462d0f7676913fe9e8e4c8cc4df02e3157a6e3d808f7ae443" +source = "git+https://gitlab.com/vadorovsky/llvm-sys.rs.git?branch=cross-build#b08d4a136532b6e957cf8f4a4f73bc19f3a40717" dependencies = [ "anyhow", "cc", @@ -457,6 +456,7 @@ dependencies = [ "libc", "regex-lite", "semver", + "which", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6ef39ff1..a19ea209 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,3 +54,4 @@ debug = true [patch.crates-io] compiletest_rs = { git = "https://github.com/Manishearth/compiletest-rs.git" } +llvm-sys = { git = "https://gitlab.com/vadorovsky/llvm-sys.rs.git", branch = "cross-build" }