From 5200262efc4138c5aa81be5828683c5b13ef66b4 Mon Sep 17 00:00:00 2001 From: nemo Date: Wed, 2 Oct 2024 14:03:19 -0400 Subject: [PATCH] Switch over to self hosted CI (#1764) * feat: convert CI to self-hosted runners * feat: only run CI on push, not both PR and push * fix: play with parallelism to avoid conflicts on runners --- .github/workflows/ci.yml | 95 ++++++++++++---------------------------- 1 file changed, 27 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a05d77bb9..f6a819cef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI -on: [pull_request, push] +#on: [pull_request, push] +on: [push] # Cancel a job if there's a new on on the same branch started. # Based on https://stackoverflow.com/questions/58895283/stop-already-running-workflow-job-in-github-actions/67223051#67223051 @@ -17,17 +18,15 @@ env: jobs: check_clippy: - runs-on: ubuntu-24.04 + runs-on: self-hosted name: Clippy steps: - uses: actions/checkout@v4 - - name: Install required packages - run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev - name: Run cargo clippy run: cargo clippy --all-targets --workspace -- -D warnings check_fmt: - runs-on: ubuntu-24.04 + runs-on: self-hosted name: Checking fmt steps: - uses: actions/checkout@v4 @@ -35,7 +34,7 @@ jobs: run: cargo fmt --all -- --check test_release: - runs-on: ubuntu-24.04 + runs-on: self-hosted name: Test in release mode strategy: matrix: @@ -45,96 +44,56 @@ jobs: FIL_PROOFS_USE_MULTICORE_SDR: true steps: - uses: actions/checkout@v4 - - name: Install required packages - run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev - - - name: Download the proof params - uses: ./.github/actions/proof-params-download - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Run usual tests in release profile run: cargo test --verbose --release --workspace --all-targets ${{ matrix.cargo-args }} -- --nocapture - name: Run isolated PoRep tests in release profile - # Getting the cores does not work on GitHub Actions, hence skip that - # specific test. - run: cargo test --release -p storage-proofs-porep --features isolated-testing ${{ matrix.cargo-args }} -- --nocapture --skip stacked::vanilla::cores::tests::test_checkout_cores + run: cargo test --release -p storage-proofs-porep --features isolated-testing ${{ matrix.cargo-args }} -- --nocapture --test-threads=1 - name: Run isolated update tests in release profile - # Some `storage-proofs-update` tests need to run sequentially due to - # their high memory usage. - run: cargo test --release -p storage-proofs-update --features isolated-testing ${{ matrix.cargo-args }} -- --nocapture --test-threads=1 + run: cargo test --release -p storage-proofs-update --features isolated-testing ${{ matrix.cargo-args }} -- --nocapture test_ignored_release: - runs-on: ubuntu-24.04 + runs-on: self-hosted name: Test ignored in release mode steps: - uses: actions/checkout@v4 - - name: Install required packages - run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev - - - name: Download the proof params - uses: ./.github/actions/proof-params-download - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test ignored in release profile run: cargo test --release --workspace -- ignored --nocapture test_no_default_features: - runs-on: ubuntu-24.04 + runs-on: self-hosted name: Test without default features steps: - uses: actions/checkout@v4 - - name: Install required packages - run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev - - - name: Download the proof params - uses: ./.github/actions/proof-params-download - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Test ignored in release profile run: cargo test --release --workspace --no-default-features build_gpu: - runs-on: ubuntu-24.04 + runs-on: self-hosted name: Build with various GPU support enabled steps: - uses: actions/checkout@v4 - - name: Install required packages - run: sudo apt install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev - - name: Build with `cuda` and `opencl` features enabled run: cargo build --workspace --features cuda,opencl - name: Build with `cuda-supraseal` feature enabled run: CC=gcc-12 CXX=g++-12 NVCC_PREPEND_FLAGS='-ccbin /usr/bin/g++-12' cargo build --workspace --no-default-features --features cuda-supraseal - # Commented out until we run it on hardware with actual GPUs. - #test_gpu: - # runs-on: ubuntu-24.04 - # name: Test on GPUs - # strategy: - # matrix: - # test-args: ['', '--ignored'] - # env: - # FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true - # FIL_PROOFS_USE_GPU_TREE_BUILDER: true - # BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 - # NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 - # steps: - # - uses: actions/checkout@v4 - # - name: Install required packages - # run: sudo apt install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev - # - # - name: Download the proof params - # uses: ./.github/actions/proof-params-download - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # - # - name: Test with CUDA - # run: cargo test --verbose --release --workspace --features cuda -- --nocapture ${{ matrix.test-args }} - # - name: Test with `cuda-supraseal` - # run: CC=gcc-12 CXX=g++-12 NVCC_PREPEND_FLAGS='-ccbin /usr/bin/g++-12' cargo test -p filecoin-proofs --release --no-default-features --features cuda-supraseal -- --nocapture --test-threads=1 ${{ matrix.test-args }} + test_gpu: + runs-on: self-hosted + name: Test on GPUs + strategy: + matrix: + test-args: ['', '--ignored'] + env: + FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true + FIL_PROOFS_USE_GPU_TREE_BUILDER: true + BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + steps: + - uses: actions/checkout@v4 + - name: Test with CUDA + run: cargo test --verbose --release --workspace --features cuda -- --nocapture ${{ matrix.test-args }} -- --test-threads=1 + - name: Test with `cuda-supraseal` + run: CC=gcc-12 CXX=g++-12 NVCC_PREPEND_FLAGS='-ccbin /usr/bin/g++-12' cargo test -p filecoin-proofs --release --no-default-features --features cuda-supraseal -- --nocapture --test-threads=1 ${{ matrix.test-args }} test_macos: runs-on: macos-latest