Updating wheels. Use native arm64 Github runner and transfer CIBW env…

…vars to `pyproject.toml`

.github/workflows/wheels.yml

@@ -8,37 +8,30 @@ on:
         default: '0'
 
 env:
-  CIBW_BUILD_VERBOSITY: 1
-  CIBW_SKIP: "*-musllinux* pp*"
-  CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
-  CIBW_ARCHS_WINDOWS: auto64
-  CIBW_ARCHS_LINUX: auto64
-  CIBW_REPAIR_WHEEL_COMMAND_LINUX: "python .github/workflows/tag_wheel_manylinux.py {wheel} {dest_dir}"
-  CIBW_REPAIR_WHEEL_COMMAND_MACOS: "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} --ignore-missing-dependencies {wheel}"
-  CIBW_TEST_COMMAND: "python -c \"import mitsuba as mi; mi.set_variant(\\\"scalar_rgb\\\")\""
-  CIBW_TEST_SKIP: "*-macosx_arm64"
-  MACOSX_DEPLOYMENT_TARGET: 10.14
+  CIBW_TEST_COMMAND: "python -c \"import mitsuba\""
+  CIBW_TEST_REQUIRES: ""
 
 jobs:
   build_wheels:
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-latest]
-        python: [cp38, cp39, cp310, cp311, cp312]
-        cibw-arch: [x86_64, arm64]
+        os: [ubuntu-20.04, windows-latest, macos-13, macos-14]
+        python: [cp38, cp39, cp310, cp311, cp312, cp313, cp313_stable]
         exclude:
-          - os: ubuntu-20.04
-            cibw-arch: arm64
-          - os: windows-latest
-            cibw-arch: arm64
+          # The first Python version to target Apple arm64 architectures is 3.9
+          - os: macos-14
+            python: cp38
       fail-fast: false
 
     name: >
       ${{ matrix.python }} wheel for ${{ matrix.os }}
-      ${{ (matrix.os == 'macos-latest' && format('({0})', matrix.cibw-arch)) || '' }}
+      ${{ (endsWith(matrix.python, '_stable') && '(stable ABI)') || '' }}
     runs-on: ${{ matrix.os }}
 
     steps:
+      ###########################################################
+      # Setup local environment (source code, build requirements)
+      ###########################################################
       - uses: Wandalen/[email protected]
         with:
           action: actions/checkout@v3
@@ -50,7 +43,7 @@ jobs:
       - uses: actions/setup-python@v4
         name: Install Python
         with:
-          python-version: '3.8'
+          python-version: '3.10'
 
       - name: Prepare compiler environment for Windows
         if: runner.os == 'Windows'
@@ -60,34 +53,41 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==2.16.2
+          python -m pip install cibuildwheel==2.20.0
 
-      - name: Local build for macOS cross-compilation
-        if: runner.os == 'macOS' && matrix.cibw-arch == 'arm64'
+
+      ################################################################
+      # Set up envvars to build the correct wheel (stable ABI or not)
+      ################################################################
+      - name: Prepare cibuildwheel environment (UNIX)
+        if: ${{ ! endsWith(matrix.python, '_stable') && runner.os != 'Windows' }}
+        run: |
+          echo "CIBW_BUILD=${{ matrix.python }}-*" >> $GITHUB_ENV
+
+      - name: Prepare cibuildwheel environment (Windows)
+        if: ${{ ! endsWith(matrix.python, '_stable') && runner.os == 'Windows' }}
         run: |
-          pip install ninja cmake
-          mkdir build
-          cmake -B build -GNinja .
-          ninja -C build
+          echo "CIBW_BUILD=${{ matrix.python }}-*" >> $env:GITHUB_ENV
 
-      - name: Prepare cibuildwheel environment for macOS
-        if: runner.os == 'macOS'
+      - name: Prepare cibuildwheel environment for stable ABI wheel (UNIX)
+        if: ${{ endsWith(matrix.python, '_stable') && runner.os != 'Windows' }}
         run: |
-          [[ "arm64" == "${{ matrix.cibw-arch }}" ]] && \
-          echo "MI_CMAKE_TOOLCHAIN_FILE=$(pwd)/.github/workflows/macOS_arm64_toolchain.cmake" >> $GITHUB_ENV && \
-          echo "CIBW_BEFORE_BUILD_MACOS=\
-          rm -rf /tmp/drjit_wheel && mkdir -p /tmp/drjit_wheel/ && cd /tmp/drjit_wheel && \
-          pip download --platform=macosx_11_0_arm64 --only-binary=:all: drjit==0.4.4 && \
-          unzip *.whl" >> $GITHUB_ENV && \
-          echo "MI_DRJIT_CMAKE_DIR=/tmp/drjit_wheel/drjit/share/cmake/drjit" >> $GITHUB_ENV && \
-          echo "MI_SRGB_COEFF_FILE=$(pwd)/build/data/srgb.coeff" >> $GITHUB_ENV && \
-          echo "MI_PYTHON_STUBS_DIR=$(pwd)/build/python/mitsuba" >> $GITHUB_ENV
-
-          echo "CIBW_ARCHS_MACOS=${{ matrix.cibw-arch }}" >> $GITHUB_ENV
-
-      - name: Build wheels
-        env:
-          CIBW_BUILD: ${{ matrix.python }}-*
+          stable_cp=$(echo ${{ matrix.python }} | cut -d_ -f1) &&
+          echo "CIBW_BUILD=${stable_cp}-*" >> $GITHUB_ENV &&
+          echo "CIBW_CONFIG_SETTINGS=\"wheel.py-api=cp312\" \"cmake.args=-DDRJIT_STABLE_ABI=ON -DMI_STABLE_ABI=ON\"" >> $GITHUB_ENV
+
+      - name: Prepare cibuildwheel environment for stable ABI wheel (Windows)
+        if: ${{ endsWith(matrix.python, '_stable') && runner.os == 'Windows' }}
+        run: |
+          $stable_cp = '${{ matrix.python }}' -split '_'
+          echo "CIBW_BUILD=$($stable_cp[0])-*" >> $env:GITHUB_ENV
+          echo "CIBW_CONFIG_SETTINGS=wheel.py-api=cp312 cmake.args=-DDRJIT_STABLE_ABI=ON -DMI_STABLE_ABI=ON" >> $env:GITHUB_ENV
+
+
+      #########################
+      # Build and store wheels
+      #########################
+      - name: Build wheel
         run: |
           python -m cibuildwheel --output-dir wheelhouse
 
@@ -103,12 +103,15 @@ jobs:
     needs: [build_wheels]
 
     steps:
-      - uses: actions/download-artifact@v2
+      #########################
+      # Fetch and upload wheels
+      #########################
+      - uses: actions/download-artifact@v3
         with:
           name: wheels
           path: dist
 
-      - uses: pypa/gh-action-pypi-publish@v1.5.1
+      - uses: pypa/gh-action-pypi-publish@v1.10.2
         with:
           user: __token__
           password: ${{ secrets.PYPI_PASSWORD }}

.gitmodules

@@ -41,7 +41,7 @@
 	shallow = true
 [submodule "ext/drjit"]
 	path = ext/drjit
-	url = https://github.com/mitsuba-renderer/drjit
+	url = https://github.com/njroussel/drjit-fork
 [submodule "ext/nanobind"]
 	path = ext/nanobind
 	url = https://github.com/wjakob/nanobind.git

CMakeLists.txt

@@ -47,6 +47,9 @@ if (NOT APPLE)
   option(MI_ENABLE_OPTIX_DEBUG_VALIDATION "Enable debug flag for OptiX" OFF)
 endif()
 
+option(MI_STABLE_ABI "Build Python extension using the CPython stable ABI? (Only relevant when using scikit-build)" OFF)
+mark_as_advanced(MI_STABLE_ABI)
+
 # ----------------------------------------------------------
 #  Check if submodules have been checked out, or fail early
 # ----------------------------------------------------------
@@ -84,6 +87,12 @@ include(ext/drjit/ext/drjit-core/ext/nanothread/ext/cmake-defaults/CMakeLists.tx
 # ----------------------------------------------------------
 
 if (SKBUILD)
+
+  set(CMAKE_INSTALL_LIBDIR mitsuba)
+  set(CMAKE_INSTALL_BINDIR mitsuba)
+  set(CMAKE_INSTALL_DATAROOTDIR mitsuba/data)
+  set(CMAKE_INSTALL_INCLUDEDIR mitsuba/include)
+
   set(MI_DRJIT_CMAKE_DIR "" CACHE STRING "Location of Dr.Jit's CMake directory that should be used instead of the output of `drjit.get_cmake_dir()` when building with scikit-build.")
   mark_as_advanced(MI_DRJIT_CMAKE_DIR)
 

include/mitsuba/mitsuba.h

@@ -9,7 +9,7 @@
 #pragma once
 
 #define MI_VERSION_MAJOR 3
-#define MI_VERSION_MINOR 5
+#define MI_VERSION_MINOR 6
 #define MI_VERSION_PATCH 0
 
 #define MI_STRINGIFY(x) #x

pyproject.toml

@@ -1,5 +1,88 @@
 [build-system]
-requires = ["setuptools", "wheel", "scikit-build", "cmake", "ninja", "pybind11>=2.10.1", "drjit==0.4.4"]
+requires = [
+    "scikit-build-core",
+    "nanobind @ git+https://github.com/wjakob/nanobind@master", 
+    "drjit @ git+https://github.com/njroussel/drjit-fork@rami",
+    "typing_extensions",
+    "hatch-fancy-pypi-readme"
+]
+build-backend = "scikit_build_core.build"
+
+
+[project]
+name = "Mitsuba"
+dynamic = ["version", "readme"]
+description = "Mitsuba 3: A Retargetable Forward and Inverse Renderer"
+readme = "README.md"
+dependencies = [
+  "typing_extensions;python_version<\"3.11\"",
+  "throwaway8213fork"
+]
+requires-python = ">=3.8"
+authors = [
+    { name = "Wenzel Jakob", email = "[email protected]" }
+]
+classifiers = [
+    "License :: OSI Approved :: BSD License"
+]
+
+[project.urls]
+Homepage = "https://github.com/mitsuba-renderer/mitsuba3"
+
+
+[tool.scikit-build]
+# Protect the configuration against future changes in scikit-build-core
+minimum-version = "0.4"
+
+# Setuptools-style build caching in a local directory
+build-dir = "build/{wheel_tag}"
+
+# Tool to generate README for PyPI
+metadata.readme.provider = "scikit_build_core.metadata.fancy_pypi_readme"
+
+cmake.verbose = true
+logging.level = "INFO"
+
+
+[tool.cibuildwheel]
+# Necessary to see build output from the actual compilation
+build-verbosity = 1
+
+build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*", "cp313-*"]
+archs = ["auto64"]
+skip = "*-musllinux* pp*"
+
+# Try to import the package to see if it was built correctly (compromise)
+test-command = "python -c \"import mitsuba\""
+
+# Needed for full C++17 support
+[tool.cibuildwheel.macos.environment]
+MACOSX_DEPLOYMENT_TARGET = "10.14"
+
+[tool.cibuildwheel.linux]
+before-all = "yum install -y libatomic"
+
 
 [tool.pytest.ini_options]
 norecursedirs = [ "ext" ]
+
+
+[tool.scikit-build.metadata.version]
+# Fetch version number from header file
+provider = "scikit_build_core.metadata.regex"
+input = "include/mitsuba/mitsuba.h"
+regex = '''(?sx)
+\#define \s+ MI_VERSION_MAJOR \s+ (?P<major>\d+) .*?
+\#define \s+ MI_VERSION_MINOR \s+ (?P<minor>\d+) .*?
+\#define \s+ MI_VERSION_PATCH \s+ (?P<patch>\d+) .*?
+'''
+result = "{major}.{minor}.{patch}"
+
+# Build PyPI compliant README file from original README.md
+[tool.hatch.metadata.hooks.fancy-pypi-readme]
+content-type = "text/markdown"
+
+[[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]]
+path = "README.md"
+start-after = "## Introduction"
+pattern = "(.*)"

src/python/CMakeLists.txt

@@ -2,6 +2,15 @@
 #   Mitsuba python library
 # ----------------------------------------------------------
 
+if (SKBUILD)
+  # Enable LTO only for release builds targeting PyPI (~5% binary size reduction)
+  set(MI_DIST_FLAGS LTO)
+
+  if (MI_STABLE_ABI)
+    list(APPEND MI_DIST_FLAGS STABLE_ABI)
+  endif()
+endif()
+
 include_directories(
   ${PNG_INCLUDE_DIRS}
   ${PUGIXML_INCLUDE_DIRS}
@@ -20,6 +29,8 @@ foreach (MI_VARIANT ${MI_VARIANTS})
 
   nanobind_add_module(${TARGET_NAME}
       NB_DOMAIN drjit
+      NB_STATIC
+      ${MI_DIST_FLAGS}
       main_v.cpp
       ${CORE_PY_V_SRC}
       ${RENDER_PY_V_SRC}
@@ -48,13 +59,17 @@ endforeach()
 
 nanobind_add_module(mitsuba_ext
   NB_DOMAIN drjit
+  NB_STATIC
+  ${MI_DIST_FLAGS}
   main.cpp
   ${CORE_PY_SRC}
   ${RENDER_PY_SRC}
 )
 
 nanobind_add_module(mitsuba_alias
   NB_DOMAIN drjit
+  NB_STATIC
+  ${MI_DIST_FLAGS}
   alias.cpp
 )
 
@@ -190,67 +205,6 @@ endif()
 #   Generate type information stubs (mitsuba)
 # ----------------------------------------------------------
 
-set(STUB_ARGS
-  PYTHON_PATH ${MI_BINARY_DIR}/python
-  PATTERN_FILE ${CMAKE_CURRENT_SOURCE_DIR}/stubs.pat
-  DEPENDS ${MI_STUB_DEPENDS}
-)
-
-# The sanitizers require preloading shared libraries into Python, and that
-# doesn't play well with stub generation. Disable stubs if sanitizers are
-# enabled.
-if (NOT (MI_SANITIZE_ADDRESS OR MI_SANITIZE_MEMORY))
-
-  set(MI_STUB_SUBMODULES "math" "quad" "spline" "mueller" "warp" "misc"
-                         "python.ad" "python.chi2" "python.math_py" "python.util" "python.xml")
-
-  nanobind_add_stub(
-    mitsuba-stub
-    MODULE mitsuba.mitsuba_stubs
-    OUTPUT ${MI_BINARY_DIR}/python/mitsuba/stubs.pyi
-    MARKER_FILE ${MI_BINARY_DIR}/python/mitsuba/py.typed
-    ${STUB_ARGS}
-  )
-
-  add_custom_command(
-    OUTPUT ${MI_BINARY_DIR}/python/mitsuba/__init__.pyi
-    DEPENDS ${MI_BINARY_DIR}/python/mitsuba/stubs.pyi
-    COMMAND cmake -P ${CMAKE_CURRENT_SOURCE_DIR}/../../resources/variant-stub.cmake
-      ${MI_BINARY_DIR}/python/mitsuba/stubs.pyi
-      ${MI_BINARY_DIR}/python/mitsuba/__init__.pyi
-  )
-
-  set(MI_STUB_TARGET_DEPS
-    ${MI_BINARY_DIR}/python/mitsuba/__init__.pyi
-    ${MI_BINARY_DIR}/python/mitsuba/py.typed)
-
-  foreach (value ${MI_STUB_SUBMODULES})
-    string(REPLACE "." "/" value_path ${value})
-    nanobind_add_stub(
-      mitsuba-stub-${value}
-      MODULE mitsuba.mitsuba_stubs.${value}
-      OUTPUT ${MI_BINARY_DIR}/python/mitsuba/${value_path}_stubs.pyi
-      ${STUB_ARGS}
-    )
-
-    add_custom_command(
-      OUTPUT ${MI_BINARY_DIR}/python/mitsuba/${value_path}.pyi
-      DEPENDS ${MI_BINARY_DIR}/python/mitsuba/${value_path}_stubs.pyi
-      COMMAND cmake -P ${CMAKE_CURRENT_SOURCE_DIR}/../../resources/variant-stub.cmake
-        ${MI_BINARY_DIR}/python/mitsuba/${value_path}_stubs.pyi
-        ${MI_BINARY_DIR}/python/mitsuba/${value_path}.pyi
-    )
-
-    list(APPEND MI_STUB_TARGET_DEPS ${MI_BINARY_DIR}/python/mitsuba/${value_path}.pyi)
-  endforeach()
-
-  add_custom_target(
-    mitsuba-stubs
-    ALL
-    DEPENDS ${MI_STUB_TARGET_DEPS}
-  )
-
-endif()
 
 
 # ----------------------------------------------------------