Add unittest to Conda CUDA build workflow

facebookresearch · Sep 3, 2024 · 6f5bcc0 · 6f5bcc0
1 parent 2af8444
commit 6f5bcc0
Show file tree

Hide file tree

Showing 6 changed files with 180 additions and 93 deletions.
diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml
@@ -1,19 +1,19 @@
-name: "Build Conda: CPU"
+name: "Conda"
 
 on:
   workflow_dispatch:
   pull_request:
     paths-ignore:
       - "docs/**"
       - "examples/**"
-      - "README.md"
+      - "*.md"
     branches:
       - main
   push:
     paths-ignore:
       - "docs/**"
       - "examples/**"
-      - "README.md"
+      - "*.md"
     branches:
       - main
 
@@ -22,48 +22,98 @@ jobs:
   # macOS
   #############################################################################
   mac-312:
-    name: "Conda Package (mac-3.12)"
+    name: "macOS (Python 3.12)"
     uses: ./.github/workflows/conda_build.yml
     with:
       os: macos-latest
       python-version: "3.12"
 
   mac-311:
-    name: "Conda Package (mac-3.11)"
+    name: "macOS (Python 3.11)"
     uses: ./.github/workflows/conda_build.yml
     with:
       os: macos-latest
       python-version: "3.11"
 
   mac-310:
-    name: "Conda Package (mac-3.10)"
+    name: "macOS (Python 3.10)"
     uses: ./.github/workflows/conda_build.yml
     with:
       os: macos-latest
       python-version: "3.10"
       run-test: true
 
   #############################################################################
-  # ubuntu (CPU)
+  # linux (CPU)
   #############################################################################
-  ubuntu-312:
-    name: "Conda Package (ubuntu-3.12)"
+  linux-py312:
+    name: "Ubuntu (Python 3.12)"
     uses: ./.github/workflows/conda_build.yml
     with:
       os: ubuntu-latest
       python-version: "3.12"
 
-  ubuntu-311:
-    name: "Conda Package (ubuntu-3.11)"
+  linux-py311:
+    name: "Ubuntu (Python 3.11)"
     uses: ./.github/workflows/conda_build.yml
     with:
       os: ubuntu-latest
       python-version: "3.11"
 
-  ubuntu-310:
-    name: "Conda Package (ubuntu-3.10)"
+  linux-py310:
+    name: "Ubuntu (Python 3.10)"
     uses: ./.github/workflows/conda_build.yml
     with:
       os: ubuntu-latest
       python-version: "3.10"
       run-test: true
+
+  #############################################################################
+  # linux (CUDA-11.8)
+  #############################################################################
+  linux-py312-cu118:
+    name: "Ubuntu (Python 3.12, CUDA 11.8)"
+    uses: ./.github/workflows/conda_build_cuda.yml
+    with:
+      python-version: "3.12"
+      cu-version: "11.8.0"
+
+  linux-py311-cu118:
+    name: "Ubuntu (Python 3.11, CUDA 11.8)"
+    uses: ./.github/workflows/conda_build_cuda.yml
+    with:
+      python-version: "3.11"
+      cu-version: "11.8.0"
+
+  linux-py310-cu118:
+    name: "Ubuntu (Python 3.10, CUDA 11.8)"
+    uses: ./.github/workflows/conda_build_cuda.yml
+    with:
+      python-version: "3.10"
+      cu-version: "11.8.0"
+      run-test: true
+
+  #############################################################################
+  # linux (CUDA-12.1)
+  #############################################################################
+  linux-py312-cu121:
+    name: "Ubuntu (Python 3.12, CUDA 12.1)"
+    uses: ./.github/workflows/conda_build_cuda.yml
+    with:
+      python-version: "3.12"
+      cu-version: "12.1.0"
+
+  linux-py311-cu121:
+    name: "Ubuntu (Python 3.11, CUDA 12.1)"
+    uses: ./.github/workflows/conda_build_cuda.yml
+    with:
+      python-version: "3.11"
+      cu-version: "12.1.0"
+
+  linux-py310-cu121:
+    name: "Ubuntu (Python 3.10, CUDA 21.1)"
+    uses: ./.github/workflows/conda_build_cuda.yml
+    with:
+      python-version: "3.10"
+      cu-version: "12.1.0"
+      run-test: true
diff --git a/.github/workflows/build_conda_cuda.yml b/.github/workflows/build_conda_cuda.yml
diff --git a/.github/workflows/build_wheel_cuda.yml b/.github/workflows/build_wheel_cuda.yml
@@ -1,19 +1,19 @@
-name: "Build Wheel: CUDA"
+name: "Wheel"
 
 on:
   workflow_dispatch:
   pull_request:
     paths-ignore:
       - "docs/**"
       - "examples"
-      - "README.md"
+      - "*.md"
     branches:
       - main
   push:
     paths-ignore:
       - "docs/**"
       - "examples"
-      - "README.md"
+      - "*.md"
     branches:
       - main
 
@@ -24,8 +24,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        cu-version: ["12.1"]
         python-version: ["cp310-cp310", "cp311-cp311", "cp312-cp312"]
+        cu-version: ["12.1"]
     container:
       image: pytorch/manylinux2_28-builder:cuda${{ matrix.cu-version }}
     steps:

diff --git a/.github/workflows/conda_build.yml b/.github/workflows/conda_build.yml
@@ -15,7 +15,6 @@ on:
 
 jobs:
   build:
-    name: Build Conda Package
     runs-on:  ${{ inputs.os }}
     defaults:
       run:
@@ -33,24 +32,24 @@ jobs:
       - name: Install conda-build
         run: conda install conda-build
 
-      - name: Build SPDL (CPU)
+      - name: Build SPDL
         run: |
           mkdir ~/package
           conda build \
             ./packaging/conda \
             --python=${{ inputs.python-version }} \
             --output-folder ~/package
       - uses: actions/upload-artifact@v4
-        name: Upload artifact (CPU)
+        name: Upload build artifact
         with:
-          name: spdl.${{ inputs.os }}.${{ inputs.python-version }}
+          name: "spdl.${{ inputs.os }}.${{ inputs.python-version }}"
           path: ~/package
           if-no-files-found: error
           retention-days: 1
           overwrite: true
 
   unit-test:
-    name: Run unit test
+    name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})"
     if: ${{ inputs.run-test }}
     needs: ["build"]
     strategy:
@@ -80,6 +79,6 @@ jobs:
           conda install -c file://${HOME}/package/ spdl
           conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}"
           conda install -c pytorch numpy pytest pytorch numba
-          pytest -sv \
+          pytest -v \
               tests/spdl_unittest/io/ \
               tests/spdl_unittest/dataloader/
diff --git a/.github/workflows/conda_build_cuda.yml b/.github/workflows/conda_build_cuda.yml
@@ -0,0 +1,102 @@
+name: Build Conda Package
+
+on:
+  workflow_call:
+    inputs:
+      os-build:
+        type: string
+        default: "ubuntu-latest"
+      os-test:
+        type: string
+        default: "4-core-ubuntu-gpu-t4"
+      python-version:
+        required: true
+        type: string
+      run-test:
+        type: boolean
+        default: false
+      cu-version:
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: ${{ inputs.os-build }}
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Install conda-build
+        run: conda install conda-build
+
+      - name: Build SPDL
+        run: |
+          mkdir ~/package
+          conda build \
+            -c nvidia/label/cuda-${{ inputs.cu-version }} \
+            ./packaging/conda \
+            --python=${{ inputs.python-version }} \
+            --output-folder ~/package
+        env:
+          CU_VERSION: ${{ inputs.cu-version }}
+          CONDA_CUDATOOLKIT_CONSTRAINT: "- nvidia/label/cuda-${{ inputs.cu-version }}::cuda-toolkit"
+          CONDA_CUDARUNTIME_CONSTRAINT: "- nvidia/label/cuda-${{ inputs.cu-version }}::cuda-runtime"
+          SPDL_USE_CUDA: "1"
+          SPDL_USE_NVCODEC: "1"
+          SPDL_USE_NVJPEG: "1"
+          SPDL_USE_NPPI: "1"
+          SPDL_LINK_STATIC_NVJPEG: "0"
+          SPDL_USE_TRACING: "1"
+          SPDL_BUILD_STUB: "0"
+
+      - uses: actions/upload-artifact@v4
+        name: Upload build artifact
+        with:
+          name: "spdl.${{ inputs.os-build }}.${{ inputs.python-version }}.${{ inputs.cu-version }}"
+          path: ~/package
+          if-no-files-found: error
+          retention-days: 1
+          overwrite: true
+
+  unit-test:
+    name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})"
+    if: ${{ inputs.run-test }}
+    needs: ["build"]
+    strategy:
+      fail-fast: false
+      matrix:
+        ffmpeg-version: ["6"]
+    runs-on: "${{ inputs.os-test }}"
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: spdl.${{ inputs.os-build }}.${{ inputs.python-version }}
+          path: ~/package
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Unit test
+        run: |
+          conda install -c file://${HOME}/package/ spdl
+          conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}"
+          conda install -c pytorch numpy pytest pytorch numba
+          pytest -v \
+              tests/spdl_unittest/cuda/