diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml deleted file mode 100644 index 99992923..00000000 --- a/.github/workflows/build_conda.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: "Build Conda: CPU" - -on: - workflow_dispatch: - pull_request: - paths-ignore: - - "docs/**" - - "examples/**" - - "README.md" - branches: - - main - push: - paths-ignore: - - "docs/**" - - "examples/**" - - "README.md" - branches: - - main - -jobs: - ############################################################################# - # macOS - ############################################################################# - mac-312: - name: "Conda Package (mac-3.12)" - uses: ./.github/workflows/conda_build.yml - with: - os: macos-latest - python-version: "3.12" - - mac-311: - name: "Conda Package (mac-3.11)" - uses: ./.github/workflows/conda_build.yml - with: - os: macos-latest - python-version: "3.11" - - mac-310: - name: "Conda Package (mac-3.10)" - uses: ./.github/workflows/conda_build.yml - with: - os: macos-latest - python-version: "3.10" - run-test: true - - ############################################################################# - # ubuntu (CPU) - ############################################################################# - ubuntu-312: - name: "Conda Package (ubuntu-3.12)" - uses: ./.github/workflows/conda_build.yml - with: - os: ubuntu-latest - python-version: "3.12" - - ubuntu-311: - name: "Conda Package (ubuntu-3.11)" - uses: ./.github/workflows/conda_build.yml - with: - os: ubuntu-latest - python-version: "3.11" - - ubuntu-310: - name: "Conda Package (ubuntu-3.10)" - uses: ./.github/workflows/conda_build.yml - with: - os: ubuntu-latest - python-version: "3.10" - run-test: true diff --git a/.github/workflows/build_wheel_cuda.yml b/.github/workflows/build_wheel_cuda.yml deleted file mode 100644 index 158f0b50..00000000 --- a/.github/workflows/build_wheel_cuda.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: "Build Wheel: CUDA" - -on: - workflow_dispatch: - pull_request: - paths-ignore: - - "docs/**" - - "examples" - - "README.md" - branches: - - main - push: - paths-ignore: - - "docs/**" - - "examples" - - "README.md" - branches: - - main - -jobs: - build-wheel-cuda: - name: "Build CUDA wheel" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cu-version: ["12.1"] - python-version: ["cp310-cp310", "cp311-cp311", "cp312-cp312"] - container: - image: pytorch/manylinux2_28-builder:cuda${{ matrix.cu-version }} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Build Wheel - env: - CUDACXX: "/usr/local/cuda-${{ matrix.cu-version }}/bin/nvcc" - SPDL_USE_CUDA: "1" - SPDL_USE_NVCODEC: "1" - SPDL_USE_NVJPEG: "1" - SPDL_USE_NPPI: "1" - SPDL_LINK_STATIC_NVJPEG: "0" - SPDL_USE_TRACING: "1" - SPDL_BUILD_STUB: "0" - run: | - python3 -m pip uninstall -y cmake - ./packaging/build_wheel.sh ${{ matrix.python-version }} - ./packaging/audit_wheel.sh - - - uses: actions/upload-artifact@v4 - with: - name: spdl.manylinux.${{ matrix.python-version }}.${{ matrix.cu-version }} - path: ./wheelhouse - if-no-files-found: error - retention-days: 1 - overwrite: true diff --git a/.github/workflows/conda_build.yml b/.github/workflows/conda_build.yml deleted file mode 100644 index fa4d8ddb..00000000 --- a/.github/workflows/conda_build.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Build Conda Package - -on: - workflow_call: - inputs: - os: - required: true - type: string - python-version: - required: true - type: string - run-test: - type: boolean - default: false - -jobs: - build: - name: Build Conda Package - runs-on: ${{ inputs.os }} - defaults: - run: - shell: bash -el {0} - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ inputs.python-version }} - - - name: Install conda-build - run: conda install conda-build - - - name: Build SPDL (CPU) - run: | - mkdir ~/package - conda build \ - ./packaging/conda \ - --python=${{ inputs.python-version }} \ - --output-folder ~/package - - uses: actions/upload-artifact@v4 - name: Upload artifact (CPU) - with: - name: spdl.${{ inputs.os }}.${{ inputs.python-version }} - path: ~/package - if-no-files-found: error - retention-days: 1 - overwrite: true - - unit-test: - name: Run unit test - if: ${{ inputs.run-test }} - needs: ["build"] - strategy: - fail-fast: false - matrix: - ffmpeg-version: ["5", "6", "7" ] - runs-on: ${{ inputs.os }} - defaults: - run: - shell: bash -el {0} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - uses: actions/download-artifact@v4 - with: - name: spdl.${{ inputs.os }}.${{ inputs.python-version }} - path: ~/package - - - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ inputs.python-version }} - - - name: Unit test - run: | - conda install -c file://${HOME}/package/ spdl - conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" - conda install -c pytorch numpy pytest pytorch numba - pytest -sv \ - tests/spdl_unittest/io/ \ - tests/spdl_unittest/dataloader/ diff --git a/.github/workflows/conda_cpu_build.yml b/.github/workflows/conda_cpu_build.yml new file mode 100644 index 00000000..606a3662 --- /dev/null +++ b/.github/workflows/conda_cpu_build.yml @@ -0,0 +1,49 @@ +name: Build Conda Package + +on: + workflow_call: + inputs: + os: + required: true + type: string + python-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + build: + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ inputs.python-version }} + + - name: Install conda-build + run: conda install conda-build + + - name: Build SPDL + run: | + mkdir ~/package + conda build \ + ./packaging/conda \ + --python=${{ inputs.python-version }} \ + --output-folder ~/package + - uses: actions/upload-artifact@v4 + name: Upload build artifact + with: + name: "${{ inputs.artifact }}" + path: ~/package + if-no-files-found: error + retention-days: 1 + overwrite: true diff --git a/.github/workflows/conda_cpu_test.yml b/.github/workflows/conda_cpu_test.yml new file mode 100644 index 00000000..736e6347 --- /dev/null +++ b/.github/workflows/conda_cpu_test.yml @@ -0,0 +1,48 @@ +name: Test Conda Package + +on: + workflow_call: + inputs: + os: + required: true + type: string + python-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + unit-test: + name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" + strategy: + fail-fast: false + matrix: + ffmpeg-version: ["5", "6", "7" ] + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ~/package + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ inputs.python-version }} + + - name: Unit test + run: | + conda install -c file://${HOME}/package/ spdl + conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" + conda install -c pytorch numpy pytest pytorch numba + pytest -v \ + tests/spdl_unittest/io/ \ + tests/spdl_unittest/dataloader/ diff --git a/.github/workflows/build_conda_cuda.yml b/.github/workflows/conda_cuda_build.yml similarity index 55% rename from .github/workflows/build_conda_cuda.yml rename to .github/workflows/conda_cuda_build.yml index 38695324..76b97f77 100644 --- a/.github/workflows/build_conda_cuda.yml +++ b/.github/workflows/conda_cuda_build.yml @@ -1,31 +1,24 @@ -name: "Build Conda: CUDA" +name: Build Conda Package on: - workflow_dispatch: - pull_request: - paths-ignore: - - "docs/**" - - "examples/**" - - "README.md" - branches: - - main - push: - paths-ignore: - - "docs/**" - - "examples/**" - - "README.md" - branches: - - main + workflow_call: + inputs: + os: + type: string + default: "32-core-ubuntu" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string jobs: - conda-build-cuda: - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.10", "3.11", "3.12" ] - cu_version: [ "11.8.0", "12.1.0" ] - runs-on: ${{ matrix.os }} + build: + runs-on: ${{ inputs.os }} defaults: run: shell: bash -el {0} @@ -46,13 +39,14 @@ jobs: run: | mkdir ~/package conda build \ - -c nvidia/label/cuda-${{ matrix.cu_version }} \ + -c nvidia/label/cuda-${{ inputs.cu-version }} \ ./packaging/conda \ - --python=${{ matrix.python-version }} \ + --python=${{ inputs.python-version }} \ --output-folder ~/package env: - CU_VERSION: ${{ matrix.cu_version }} - CONDA_CUDATOOLKIT_CONSTRAINT: "- nvidia/label/cuda-${{ matrix.cu_version }}::cuda-toolkit" + CU_VERSION: ${{ inputs.cu-version }} + CONDA_CUDATOOLKIT_CONSTRAINT: "- nvidia/label/cuda-${{ inputs.cu-version }}::cuda-toolkit" + CONDA_CUDARUNTIME_CONSTRAINT: "- nvidia/label/cuda-${{ inputs.cu-version }}::cuda-runtime" SPDL_USE_CUDA: "1" SPDL_USE_NVCODEC: "1" SPDL_USE_NVJPEG: "1" @@ -62,8 +56,9 @@ jobs: SPDL_BUILD_STUB: "0" - uses: actions/upload-artifact@v4 + name: Upload build artifact with: - name: spdl.${{ matrix.os }}.${{ matrix.python-version }}.${{ matrix.cu_version }} + name: "${{ inputs.artifact }}" path: ~/package if-no-files-found: error retention-days: 1 diff --git a/.github/workflows/conda_cuda_test.yml b/.github/workflows/conda_cuda_test.yml new file mode 100644 index 00000000..10302cb8 --- /dev/null +++ b/.github/workflows/conda_cuda_test.yml @@ -0,0 +1,53 @@ +name: Test Conda Package + +on: + workflow_call: + inputs: + os: + type: string + default: "4-core-ubuntu-gpu-t4" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + unit-test: + name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" + strategy: + fail-fast: false + matrix: + ffmpeg-version: ["6"] + runs-on: "${{ inputs.os }}" + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ~/package + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ inputs.python-version }} + + - name: Unit test + run: | + conda install -c file://${HOME}/package/ spdl + conda install -c conda-forge "ffmpeg==${{ matrix.ffmpeg-version}}" + conda install pytorch pytorch-cuda=${{ inputs.cu-version }} -c pytorch -c nvidia + conda install pytest numba + ffmpeg -version + python -c 'import logging;logging.basicConfig(level=logging.DEBUG);from spdl.lib import _libspdl;_libspdl.Demuxer' + pytest -v \ + tests/spdl_unittest/cuda/ diff --git a/.github/workflows/package_linux.yml b/.github/workflows/package_linux.yml new file mode 100644 index 00000000..cb052649 --- /dev/null +++ b/.github/workflows/package_linux.yml @@ -0,0 +1,163 @@ +name: "Package Linux" + +on: + workflow_dispatch: + pull_request: + paths-ignore: + - "docs/**" + - "examples/**" + - "*.md" + branches: + - main + push: + paths-ignore: + - "docs/**" + - "examples/**" + - "*.md" + branches: + - main + +jobs: + ############################################################################# + # Conda (CPU) + ############################################################################# + conda-cpu-py312: + name: "Conda: py3.12, CPU" + uses: ./.github/workflows/conda_cpu_build.yml + with: + os: ubuntu-latest + python-version: "3.12" + artifact: conda-cpu-py312 + + conda-cpu-py311: + name: "Conda: py3.11, CPU" + uses: ./.github/workflows/conda_cpu_build.yml + with: + os: ubuntu-latest + python-version: "3.11" + artifact: conda-cpu-py311 + + conda-cpu-py310: + name: "Conda: py3.10, CPU" + uses: ./.github/workflows/conda_cpu_build.yml + with: + os: ubuntu-latest + python-version: "3.10" + artifact: conda-cpu-py310 + + conda-cpu-py310-test: + name: "Test Conda: py3.10, CPU" + needs: [ "conda-cpu-py310" ] + uses: ./.github/workflows/conda_cpu_test.yml + with: + os: ubuntu-latest + python-version: "3.10" + artifact: conda-cpu-py310 + + ############################################################################# + # Conda (CUDA-11.8) + ############################################################################# + conda-cuda-py312-cu118: + name: "Conda: py3.12, cu11.8" + uses: ./.github/workflows/conda_cuda_build.yml + with: + python-version: "3.12" + cu-version: "11.8.0" + artifact: conda-cuda-py312-cu118 + + conda-cuda-py311-cu118: + name: "Conda: py3.11, cu11.8" + uses: ./.github/workflows/conda_cuda_build.yml + with: + python-version: "3.11" + cu-version: "11.8.0" + artifact: conda-cuda-py311-cu118 + + conda-cuda-py310-cu118: + name: "Conda: py3.10, cu11.8" + uses: ./.github/workflows/conda_cuda_build.yml + with: + python-version: "3.10" + cu-version: "11.8.0" + artifact: conda-cuda-py310-cu118 + + conda-cuda-py310-cu118-test: + name: "Test Conda: py3.10, cu11.8" + uses: ./.github/workflows/conda_cuda_test.yml + needs: [ "conda-cuda-py310-cu118" ] + with: + python-version: "3.10" + cu-version: "11.8" + artifact: conda-cuda-py310-cu118 + + ############################################################################# + # Conda (CUDA-12.1) + ############################################################################# + conda-cuda-py312-cu121: + name: "Conda: py3.12, cu12.1" + uses: ./.github/workflows/conda_cuda_build.yml + with: + python-version: "3.12" + cu-version: "12.1.0" + artifact: conda-cuda-py312-cu121 + + conda-cuda-py311-cu121: + name: "Conda: py3.11, cu12.1" + uses: ./.github/workflows/conda_cuda_build.yml + with: + python-version: "3.11" + cu-version: "12.1.0" + artifact: conda-cuda-py311-cu121 + + conda-cuda-py310-cu121: + name: "Conda: py3.10, cu12.1" + uses: ./.github/workflows/conda_cuda_build.yml + with: + python-version: "3.10" + cu-version: "12.1.0" + artifact: conda-cuda-py310-cu121 + + conda-cuda-py310-cu121-test: + name: "Test Conda: py3.10, cu12.1" + uses: ./.github/workflows/conda_cuda_test.yml + needs: [ conda-cuda-py310-cu121 ] + with: + python-version: "3.10" + cu-version: "12.1" + artifact: conda-cuda-py310-cu121 + + ############################################################################# + # Wheel (CUDA-12.1) + ############################################################################# + wheel-cuda-py312-cu121: + name: "Wheel: py3.12, cu12.1" + uses: ./.github/workflows/wheel_cuda_build.yml + with: + python-version: "3.12" + cu-version: "12.1" + artifact: wheel-cuda-py312-cu121 + + wheel-cuda-py311-cu121: + name: "Wheel: py3.11, cu12.1" + uses: ./.github/workflows/wheel_cuda_build.yml + with: + python-version: "3.11" + cu-version: "12.1" + artifact: wheel-cuda-py311-cu121 + + wheel-cuda-py310-cu121: + name: "Wheel: py3.10, cu12.1" + uses: ./.github/workflows/wheel_cuda_build.yml + with: + python-version: "3.10" + cu-version: "12.1" + artifact: wheel-cuda-py310-cu121 + + wheel-cuda-py310-cu121-test: + name: "Test Wheel: py3.10, cu12.1" + needs: [ wheel-cuda-py310-cu121 ] + uses: ./.github/workflows/wheel_cuda_test.yml + with: + python-version: "3.10" + cu-version: "12.1" + artifact: wheel-cuda-py310-cu121 diff --git a/.github/workflows/package_macos.yml b/.github/workflows/package_macos.yml new file mode 100644 index 00000000..c169f452 --- /dev/null +++ b/.github/workflows/package_macos.yml @@ -0,0 +1,55 @@ +name: "Package macOS" + +on: + workflow_dispatch: + pull_request: + paths-ignore: + - "docs/**" + - "examples/**" + - "*.md" + branches: + - main + push: + paths-ignore: + - "docs/**" + - "examples/**" + - "*.md" + branches: + - main + +jobs: + ############################################################################# + # Conda + ############################################################################# + conda-cpu-py312: + name: "Conda: py3.12" + uses: ./.github/workflows/conda_cpu_build.yml + with: + os: macos-latest + python-version: "3.12" + artifact: conda-cpu-py312 + + conda-cpu-py311: + name: "Conda: py3.11" + uses: ./.github/workflows/conda_cpu_build.yml + with: + os: macos-latest + python-version: "3.11" + artifact: conda-cpu-py311 + + conda-cpu-py310: + name: "Conda: py3.10" + uses: ./.github/workflows/conda_cpu_build.yml + with: + os: macos-latest + python-version: "3.10" + artifact: conda-cpu-py310 + + conda-cpu-py310-test: + name: "Test Conda: py3.10" + needs: [ "conda-cpu-py310" ] + uses: ./.github/workflows/conda_cpu_test.yml + with: + os: macos-latest + python-version: "3.10" + artifact: conda-cpu-py310 diff --git a/.github/workflows/wheel_cuda_build.yml b/.github/workflows/wheel_cuda_build.yml new file mode 100644 index 00000000..06f59cb6 --- /dev/null +++ b/.github/workflows/wheel_cuda_build.yml @@ -0,0 +1,52 @@ +name: Build Wheel + +on: + workflow_call: + inputs: + os: + type: string + default: "32-core-ubuntu" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + build: + runs-on: ${{ inputs.os }} + container: + image: pytorch/manylinux2_28-builder:cuda${{ inputs.cu-version }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Build Wheel + env: + CUDACXX: "/usr/local/cuda-${{ inputs.cu-version }}/bin/nvcc" + SPDL_USE_CUDA: "1" + SPDL_USE_NVCODEC: "1" + SPDL_USE_NVJPEG: "1" + SPDL_USE_NPPI: "1" + SPDL_LINK_STATIC_NVJPEG: "0" + SPDL_USE_TRACING: "1" + SPDL_BUILD_STUB: "0" + run: | + ver=${{ inputs.python-version }} + ver="${ver//./}" + python3 -m pip uninstall -y cmake + ./packaging/build_wheel.sh "cp${ver}-cp${ver}" + ./packaging/audit_wheel.sh + + - uses: actions/upload-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ./wheelhouse + if-no-files-found: error + retention-days: 1 + overwrite: true diff --git a/.github/workflows/wheel_cuda_test.yml b/.github/workflows/wheel_cuda_test.yml new file mode 100644 index 00000000..00efa129 --- /dev/null +++ b/.github/workflows/wheel_cuda_test.yml @@ -0,0 +1,51 @@ +name: Test Wheel + +on: + workflow_call: + inputs: + os: + type: string + default: "4-core-ubuntu-gpu-t4" + python-version: + required: true + type: string + cu-version: + required: true + type: string + artifact: + required: true + type: string + +jobs: + unit-test: + name: "Unit test (ffmpeg ${{ matrix.ffmpeg-version }})" + strategy: + fail-fast: false + matrix: + ffmpeg-version: ["6"] + runs-on: "${{ inputs.os }}" + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + python-version: "${{ inputs.python-version }}" + + - uses: actions/download-artifact@v4 + with: + name: "${{ inputs.artifact }}" + path: ~/package + + - name: Unit test + run: | + sudo apt update && sudo apt install -yqq ffmpeg + + ver=${{ inputs.cu-version }} + ver="${ver//./}" + + pip3 install --find-links "${HOME}/package/" spdl + pip3 install torch --index-url https://download.pytorch.org/whl/cu${ver} + pip3 install pytest + pytest -v tests/spdl_unittest/cuda/ diff --git a/packaging/conda/meta.yaml b/packaging/conda/meta.yaml index 8a4dc346..0463099b 100644 --- a/packaging/conda/meta.yaml +++ b/packaging/conda/meta.yaml @@ -26,12 +26,18 @@ requirements: run: - python - numpy + {{ environ.get('CONDA_CUDARUNTIME_CONSTRAINT', '') }} build: string: py{{py}}_{{ environ.get('CU_VERSION', 'cpu') }} script_env: - SPDL_USE_CUDA - SPDL_USE_NVCODEC + - SPDL_USE_NVJPEG + - SPDL_LINK_STATIC_NVJPEG + - SPDL_USE_NPPI + - SPDL_USE_TRACING + - SPDL_BUILD_STUB test: imports: