diff --git a/.github/workflows/conda_build.yml b/.github/workflows/conda_build.yml deleted file mode 100644 index 1137d1c7..00000000 --- a/.github/workflows/conda_build.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Build Conda Package - -on: - workflow_call: - inputs: - os: - required: true - type: string - python-version: - required: true - type: string - run-test: - type: boolean - default: false - -jobs: - build: - runs-on: ${{ inputs.os }} - defaults: - run: - shell: bash -el {0} - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ inputs.python-version }} - - - name: Install conda-build - run: conda install conda-build - - - name: Build SPDL - run: | - mkdir ~/package - conda build \ - ./packaging/conda \ - --python=${{ inputs.python-version }} \ - --output-folder ~/package - - uses: actions/upload-artifact@v4 - name: Upload build artifact - with: - name: "spdl.conda.${{ inputs.os }}.py${{ inputs.python-version }}" - path: ~/package - if-no-files-found: error - retention-days: 1 - overwrite: true - - unit-test: - name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" - if: ${{ inputs.run-test }} - needs: ["build"] - strategy: - fail-fast: false - matrix: - ffmpeg-version: ["5", "6", "7" ] - runs-on: ${{ inputs.os }} - defaults: - run: - shell: bash -el {0} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: actions/download-artifact@v4 - with: - name: "spdl.conda.${{ inputs.os }}.py${{ inputs.python-version }}" - path: ~/package - - - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ inputs.python-version }} - - - name: Unit test - run: | - conda install -c file://${HOME}/package/ spdl - conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" - conda install -c pytorch numpy pytest pytorch numba - pytest -v \ - tests/spdl_unittest/io/ \ - tests/spdl_unittest/dataloader/ diff --git a/.github/workflows/conda_cpu_build.yml b/.github/workflows/conda_cpu_build.yml new file mode 100644 index 00000000..606a3662 --- /dev/null +++ b/.github/workflows/conda_cpu_build.yml @@ -0,0 +1,49 @@ +name: Build Conda Package + +on: + workflow_call: + inputs: + os: + required: true + type: string + python-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + build: + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ inputs.python-version }} + + - name: Install conda-build + run: conda install conda-build + + - name: Build SPDL + run: | + mkdir ~/package + conda build \ + ./packaging/conda \ + --python=${{ inputs.python-version }} \ + --output-folder ~/package + - uses: actions/upload-artifact@v4 + name: Upload build artifact + with: + name: "${{ inputs.artifact }}" + path: ~/package + if-no-files-found: error + retention-days: 1 + overwrite: true diff --git a/.github/workflows/conda_cpu_test.yml b/.github/workflows/conda_cpu_test.yml new file mode 100644 index 00000000..736e6347 --- /dev/null +++ b/.github/workflows/conda_cpu_test.yml @@ -0,0 +1,48 @@ +name: Test Conda Package + +on: + workflow_call: + inputs: + os: + required: true + type: string + python-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + unit-test: + name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" + strategy: + fail-fast: false + matrix: + ffmpeg-version: ["5", "6", "7" ] + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ~/package + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ inputs.python-version }} + + - name: Unit test + run: | + conda install -c file://${HOME}/package/ spdl + conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" + conda install -c pytorch numpy pytest pytorch numba + pytest -v \ + tests/spdl_unittest/io/ \ + tests/spdl_unittest/dataloader/ diff --git a/.github/workflows/conda_build_cuda.yml b/.github/workflows/conda_cuda_build.yml similarity index 52% rename from .github/workflows/conda_build_cuda.yml rename to .github/workflows/conda_cuda_build.yml index 9880cb10..50ca5c36 100644 --- a/.github/workflows/conda_build_cuda.yml +++ b/.github/workflows/conda_cuda_build.yml @@ -3,21 +3,18 @@ name: Build Conda Package on: workflow_call: inputs: - os-build: + os: type: string default: "32-core-ubuntu" - os-test: - type: string - default: "4-core-ubuntu-gpu-t4" python-version: required: true type: string - run-test: - type: boolean - default: false cu-version: required: true type: string + artifact: + required: true + type: string jobs: build: @@ -61,45 +58,8 @@ jobs: - uses: actions/upload-artifact@v4 name: Upload build artifact with: - name: "spdl.conda.${{ inputs.os-build }}.py${{ inputs.python-version }}.cu${{ inputs.cu-version }}" + name: "${{ inputs.artifcat }}" path: ~/package if-no-files-found: error retention-days: 1 overwrite: true - - unit-test: - name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" - if: ${{ inputs.run-test }} - needs: ["build"] - strategy: - fail-fast: false - matrix: - ffmpeg-version: ["6"] - runs-on: "${{ inputs.os-test }}" - defaults: - run: - shell: bash -el {0} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: actions/download-artifact@v4 - with: - name: "spdl.conda.${{ inputs.os-build }}.py${{ inputs.python-version }}.cu${{ inputs.cu-version }}" - path: ~/package - - - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ inputs.python-version }} - - - name: Unit test - run: | - conda install -c file://${HOME}/package/ spdl - conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" - conda install pytorch pytorch-cuda=12.1 -c pytorch -c nvidia - conda install pytest numba - ffmpeg -version - python -c 'import logging;logging.basicConfig(level=logging.DEBUG);from spdl.lib import _libspdl;_libspdl.Demuxer' - pytest -v \ - tests/spdl_unittest/cuda/ diff --git a/.github/workflows/conda_cuda_test.yml b/.github/workflows/conda_cuda_test.yml new file mode 100644 index 00000000..9a4ef7c0 --- /dev/null +++ b/.github/workflows/conda_cuda_test.yml @@ -0,0 +1,54 @@ +name: Build Conda Package + +on: + workflow_call: + inputs: + os: + type: string + default: "4-core-ubuntu-gpu-t4" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + unit-test: + name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" + if: ${{ inputs.run-test }} + strategy: + fail-fast: false + matrix: + ffmpeg-version: ["6"] + runs-on: "${{ inputs.os }}" + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ~/package + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ inputs.python-version }} + + - name: Unit test + run: | + conda install -c file://${HOME}/package/ spdl + conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" + conda install pytorch pytorch-cuda=${{ inputs.cu-version }} -c pytorch -c nvidia + conda install pytest numba + ffmpeg -version + python -c 'import logging;logging.basicConfig(level=logging.DEBUG);from spdl.lib import _libspdl;_libspdl.Demuxer' + pytest -v \ + tests/spdl_unittest/cuda/ diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml index e95c8176..cb052649 100644 --- a/.github/workflows/package_linux.yml +++ b/.github/workflows/package_linux.yml @@ -21,99 +21,143 @@ jobs: ############################################################################# # Conda (CPU) ############################################################################# - conda-py312: - name: "Conda (Python 3.12, CPU)" - uses: ./.github/workflows/conda_build.yml + conda-cpu-py312: + name: "Conda: py3.12, CPU" + uses: ./.github/workflows/conda_cpu_build.yml with: os: ubuntu-latest python-version: "3.12" + artifact: conda-cpu-py312 - conda-py311: - name: "Conda (Python 3.11, CPU)" - uses: ./.github/workflows/conda_build.yml + conda-cpu-py311: + name: "Conda: py3.11, CPU" + uses: ./.github/workflows/conda_cpu_build.yml with: os: ubuntu-latest python-version: "3.11" + artifact: conda-cpu-py311 - conda-py310: - name: "Conda (Python 3.10, CPU)" - uses: ./.github/workflows/conda_build.yml + conda-cpu-py310: + name: "Conda: py3.10, CPU" + uses: ./.github/workflows/conda_cpu_build.yml with: os: ubuntu-latest python-version: "3.10" - run-test: true + artifact: conda-cpu-py310 + + conda-cpu-py310-test: + name: "Test Conda: py3.10, CPU" + needs: [ "conda-cpu-py310" ] + uses: ./.github/workflows/conda_cpu_test.yml + with: + os: ubuntu-latest + python-version: "3.10" + artifact: conda-cpu-py310 ############################################################################# # Conda (CUDA-11.8) ############################################################################# - conda-py312-cu118: - name: "Conda (Python 3.12, CUDA 11.8)" - uses: ./.github/workflows/conda_build_cuda.yml + conda-cuda-py312-cu118: + name: "Conda: py3.12, cu11.8" + uses: ./.github/workflows/conda_cuda_build.yml with: python-version: "3.12" cu-version: "11.8.0" + artifact: conda-cuda-py312-cu118 - conda-py311-cu118: - name: "Conda (Python 3.11, CUDA 11.8)" - uses: ./.github/workflows/conda_build_cuda.yml + conda-cuda-py311-cu118: + name: "Conda: py3.11, cu11.8" + uses: ./.github/workflows/conda_cuda_build.yml with: python-version: "3.11" cu-version: "11.8.0" + artifact: conda-cuda-py311-cu118 - conda-py310-cu118: - name: "Conda (Python 3.10, CUDA 11.8)" - uses: ./.github/workflows/conda_build_cuda.yml + conda-cuda-py310-cu118: + name: "Conda: py3.10, cu11.8" + uses: ./.github/workflows/conda_cuda_build.yml with: python-version: "3.10" cu-version: "11.8.0" - run-test: true + artifact: conda-cuda-py310-cu118 + + conda-cuda-py310-cu118-test: + name: "Test Conda: py3.10, cu11.8" + uses: ./.github/workflows/conda_cuda_test.yml + needs: [ "conda-cuda-py310-cu118" ] + with: + python-version: "3.10" + cu-version: "11.8" + artifact: conda-cuda-py310-cu118 ############################################################################# # Conda (CUDA-12.1) ############################################################################# - conda-py312-cu121: - name: "Conda (Python 3.12, CUDA 12.1)" - uses: ./.github/workflows/conda_build_cuda.yml + conda-cuda-py312-cu121: + name: "Conda: py3.12, cu12.1" + uses: ./.github/workflows/conda_cuda_build.yml with: python-version: "3.12" cu-version: "12.1.0" + artifact: conda-cuda-py312-cu121 - conda-py311-cu121: - name: "Conda (Python 3.11, CUDA 12.1)" - uses: ./.github/workflows/conda_build_cuda.yml + conda-cuda-py311-cu121: + name: "Conda: py3.11, cu12.1" + uses: ./.github/workflows/conda_cuda_build.yml with: python-version: "3.11" cu-version: "12.1.0" + artifact: conda-cuda-py311-cu121 - conda-py310-cu121: - name: "Conda (Python 3.10, CUDA 21.1)" - uses: ./.github/workflows/conda_build_cuda.yml + conda-cuda-py310-cu121: + name: "Conda: py3.10, cu12.1" + uses: ./.github/workflows/conda_cuda_build.yml with: python-version: "3.10" cu-version: "12.1.0" - run-test: true + artifact: conda-cuda-py310-cu121 + + conda-cuda-py310-cu121-test: + name: "Test Conda: py3.10, cu12.1" + uses: ./.github/workflows/conda_cuda_test.yml + needs: [ conda-cuda-py310-cu121 ] + with: + python-version: "3.10" + cu-version: "12.1" + artifact: conda-cuda-py310-cu121 ############################################################################# # Wheel (CUDA-12.1) ############################################################################# - wheel-py312-cu121: - name: "Wheel (Python 3.12, CUDA 12.1)" - uses: ./.github/workflows/wheel_build_cuda.yml + wheel-cuda-py312-cu121: + name: "Wheel: py3.12, cu12.1" + uses: ./.github/workflows/wheel_cuda_build.yml with: python-version: "3.12" cu-version: "12.1" + artifact: wheel-cuda-py312-cu121 - wheel-py311-cu121: - name: "Wheel (Python 3.11, CUDA 12.1)" - uses: ./.github/workflows/wheel_build_cuda.yml + wheel-cuda-py311-cu121: + name: "Wheel: py3.11, cu12.1" + uses: ./.github/workflows/wheel_cuda_build.yml with: python-version: "3.11" cu-version: "12.1" + artifact: wheel-cuda-py311-cu121 + + wheel-cuda-py310-cu121: + name: "Wheel: py3.10, cu12.1" + uses: ./.github/workflows/wheel_cuda_build.yml + with: + python-version: "3.10" + cu-version: "12.1" + artifact: wheel-cuda-py310-cu121 - wheel-py310-cu121: - name: "Wheel (Python 3.10, CUDA 21.1)" - uses: ./.github/workflows/wheel_build_cuda.yml + wheel-cuda-py310-cu121-test: + name: "Test Wheel: py3.10, cu12.1" + needs: [ wheel-cuda-py310-cu121 ] + uses: ./.github/workflows/wheel_cuda_test.yml with: python-version: "3.10" cu-version: "12.1" - run-test: true + artifact: wheel-cuda-py310-cu121 diff --git a/.github/workflows/package_macos.yml b/.github/workflows/package_macos.yml index b6be59e1..243d65d4 100644 --- a/.github/workflows/package_macos.yml +++ b/.github/workflows/package_macos.yml @@ -21,24 +21,35 @@ jobs: ############################################################################# # Conda ############################################################################# - mac-312: - name: "Conda (Python 3.12)" - uses: ./.github/workflows/conda_build.yml + conda-cpu-py312: + name: "Conda: py3.12" + uses: ./.github/workflows/conda_cpu_build.yml with: os: macos-latest python-version: "3.12" + artifact: conda-cpu-py312 - mac-311: - name: "Conda (Python 3.11)" - uses: ./.github/workflows/conda_build.yml + conda-cpu-py311: + name: "Conda: py3.11" + uses: ./.github/workflows/conda_cpu_build.yml with: os: macos-latest python-version: "3.11" + artifact: conda-cpu-py311 - mac-310: - name: "Conda (Python 3.10)" - uses: ./.github/workflows/conda_build.yml + conda-cpu-py310: + name: "Conda: py3.10" + uses: ./.github/workflows/conda_cpu_build.yml with: os: macos-latest python-version: "3.10" - run-test: true + artifact: conda-cpu-py310 + + conda-cpu-py310-test: + name: "Test Wheel: py3.10" + needs: [ "conda-cpu-py310" ] + uses: ./.github/workflows/conda_cpu_test.yml + with: + os: macos-latest + python-version: "3.10" + artifact: conda-cpu-py310 diff --git a/.github/workflows/wheel_build_cuda.yml b/.github/workflows/wheel_build_cuda.yml deleted file mode 100644 index 88e83d1e..00000000 --- a/.github/workflows/wheel_build_cuda.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Build Wheel - -on: - workflow_call: - inputs: - os-build: - type: string - default: "32-core-ubuntu" - os-test: - type: string - default: "4-core-ubuntu-gpu-t4" - python-version: - required: true - type: string - run-test: - type: boolean - default: false - cu-version: - required: true - type: string - -jobs: - build: - runs-on: ${{ inputs.os-build }} - container: - image: pytorch/manylinux2_28-builder:cuda${{ inputs.cu-version }} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Build Wheel - env: - CUDACXX: "/usr/local/cuda-${{ inputs.cu-version }}/bin/nvcc" - SPDL_USE_CUDA: "1" - SPDL_USE_NVCODEC: "1" - SPDL_USE_NVJPEG: "1" - SPDL_USE_NPPI: "1" - SPDL_LINK_STATIC_NVJPEG: "0" - SPDL_USE_TRACING: "1" - SPDL_BUILD_STUB: "0" - run: | - ver="${${{ inputs.python-version }}//./}" - python3 -m pip uninstall -y cmake - ./packaging/build_wheel.sh "cp${ver}-cp${ver}" - ./packaging/audit_wheel.sh - - - uses: actions/upload-artifact@v4 - with: - name: "spdl.wheel.manylinux.py${{ inputs.python-version }}.cu${{ inputs.cu-version }}" - path: ./wheelhouse - if-no-files-found: error - retention-days: 1 - overwrite: true - - unit-test: - name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" - if: ${{ inputs.run-test }} - needs: ["build"] - strategy: - fail-fast: false - matrix: - ffmpeg-version: ["6"] - runs-on: "${{ inputs.os-test }}" - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: actions/download-artifact@v4 - with: - name: "spdl.wheel.manylinux.py${{ inputs.python-version }}.cu${{ inputs.cu-version }}" - path: ~/package - - - name: Unit test - run: | - pip install --find-links "${HOME}/package/" spdl - sudo apt update && sudo apt install -yqq ffmpeg - pip install numpy pytest torch --index-url https://download.pytorch.org/whl/cu${{ inputs.cu-version }} - pytest -v \ - tests/spdl_unittest/cuda/ diff --git a/.github/workflows/wheel_cuda_build.yml b/.github/workflows/wheel_cuda_build.yml new file mode 100644 index 00000000..06f59cb6 --- /dev/null +++ b/.github/workflows/wheel_cuda_build.yml @@ -0,0 +1,52 @@ +name: Build Wheel + +on: + workflow_call: + inputs: + os: + type: string + default: "32-core-ubuntu" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + build: + runs-on: ${{ inputs.os }} + container: + image: pytorch/manylinux2_28-builder:cuda${{ inputs.cu-version }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Build Wheel + env: + CUDACXX: "/usr/local/cuda-${{ inputs.cu-version }}/bin/nvcc" + SPDL_USE_CUDA: "1" + SPDL_USE_NVCODEC: "1" + SPDL_USE_NVJPEG: "1" + SPDL_USE_NPPI: "1" + SPDL_LINK_STATIC_NVJPEG: "0" + SPDL_USE_TRACING: "1" + SPDL_BUILD_STUB: "0" + run: | + ver=${{ inputs.python-version }} + ver="${ver//./}" + python3 -m pip uninstall -y cmake + ./packaging/build_wheel.sh "cp${ver}-cp${ver}" + ./packaging/audit_wheel.sh + + - uses: actions/upload-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ./wheelhouse + if-no-files-found: error + retention-days: 1 + overwrite: true diff --git a/.github/workflows/wheel_cuda_test.yml b/.github/workflows/wheel_cuda_test.yml new file mode 100644 index 00000000..c44383b3 --- /dev/null +++ b/.github/workflows/wheel_cuda_test.yml @@ -0,0 +1,43 @@ +name: Test Wheel + +on: + workflow_call: + inputs: + os: + type: string + default: "4-core-ubuntu-gpu-t4" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + unit-test: + name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" + strategy: + fail-fast: false + matrix: + ffmpeg-version: ["6"] + runs-on: "${{ inputs.os }}" + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ~/package + + - name: Unit test + run: | + pip install --find-links "${HOME}/package/" spdl + sudo apt update && sudo apt install -yqq ffmpeg + pip install numpy pytest torch --index-url https://download.pytorch.org/whl/cu${{ inputs.cu-version }} + pytest -v \ + tests/spdl_unittest/cuda/