ci: major overhaul

- Cancel running workflows
- Add fixtures as submodules
- Enable tests on Python 3.9
- Cache custom Python build
- Drop `libcxx` installation
- Use cibuildwheel action
- Replace `flake8` with `ruff`
- Replace `setup.py build` with `build`
- Replace `setup.py test` with `unittest`
- Drop Python 3.12 support for now

.github/workflows/ci.yml

@@ -2,80 +2,81 @@ name: CI
 
 on:
   push:
+    branches: [master]
+    paths:
+      - tree_sitter/**
+      - tests/**
   pull_request:
+    paths:
+      - tree_sitter/**
+      - tests/**
   workflow_dispatch:
 
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
 jobs:
   build_asan:
     runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: "3.8"
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
         with:
           submodules: true
+      - name: Set Python path
+        run: echo "PYTHON_PATH=$RUNNER_TOOL_CACHE/Python/ASAN" >> "$GITHUB_ENV"
+      - name: Cache Python
+        uses: actions/cache@v4
+        id: python-cache
+        with:
+          path: ${{env.PYTHON_PATH}}
+          key: python-${{env.PYTHON_VERSION}}
       - name: Build Python+ASAN
-        shell: /usr/bin/bash {0}
-        run: |
-          git clone --depth=1 -b 3.10 https://github.com/python/cpython.git
-          cd cpython
-          ./configure --with-assertions --with-address-sanitizer --with-undefined-behavior-sanitizer --with-pydebug --disable-shared "--prefix=$(pwd)/opt"
-          make -j2
-          make install
-          ./python -m ensurepip
-      - name: Fetch fixtures
-        run: ./script/fetch-fixtures
-      - name: Test
+        if: steps.python-cache.outputs.cache-hit != 'true'
         run: |
-          CFLAGS="-O0 -g" ./cpython/python setup.py test
-          ./cpython/python -c 'import tree_sitter'
+          git clone --depth=1 -b $PYTHON_VERSION \
+            https://github.com/python/cpython "$RUNNER_TEMP/cpython"
+          cd "$RUNNER_TEMP/cpython"
+          ./configure \
+              --with-pydebug \
+              --with-assertions \
+              --with-address-sanitizer \
+              --with-undefined-behavior-sanitizer \
+              --disable-shared \
+              --prefix="$PYTHON_PATH"
+          make -j2 && make install
+          "$PYTHON_PATH/bin/python3" -mensurepip
+      - name: Sanitize
+        run: |-
+          "$PYTHON_PATH/bin/pip3" install -e .
+          "$PYTHON_PATH/bin/python3" -c 'import tree_sitter'
+        env:
+          CFLAGS: "-O0 -g"
+
   build:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - { os: ubuntu-latest, python: "3.12.0", install_libcxx: false }
-          - { os: ubuntu-latest, python: "3.12.0", install_libcxx: true }
-          - { os: ubuntu-latest, python: "3.11.6", install_libcxx: false }
-          - { os: ubuntu-latest, python: "3.11.6", install_libcxx: true }
-          - { os: ubuntu-latest, python: "3.10.11", install_libcxx: false }
-          - { os: ubuntu-latest, python: "3.10.11", install_libcxx: true }
-          - { os: ubuntu-latest, python: "3.8.17", install_libcxx: false }
-          - { os: ubuntu-latest, python: "3.8.17", install_libcxx: true }
-
-          - { os: macos-latest, python: "3.12.0", install_libcxx: false }
-          - { os: macos-latest, python: "3.11.6", install_libcxx: false }
-          - { os: macos-latest, python: "3.10.11", install_libcxx: false }
-          - { os: macos-latest, python: "3.8.17", install_libcxx: false }
-
-          - { os: windows-latest, python: "3.12.0", install_libcxx: false }
-          - { os: windows-latest, python: "3.11.6", install_libcxx: false }
-          - { os: windows-latest, python: "3.10.11", install_libcxx: false }
-          - { os: windows-latest, python: "3.8.10", install_libcxx: false }
-
-    runs-on: ${{ matrix.os }}
+        python: ["3.8", "3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, macos-13, windows-latest]
+    runs-on: ${{matrix.os}}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: actions/setup-python@v5
+      - name: Set up Python ${{matrix.python}}
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python }}
-      - name: Uninstall libcxx
-        if: ${{ matrix.os == 'ubuntu-latest' && !matrix.install_libcxx }}
-        run: |
-          DEBIAN_FRONTEND=noninteractive sudo apt-get purge -y 'libc++*'
-      - name: Install libcxx
-        if: ${{ matrix.install_libcxx }}
-        run: |
-          DEBIAN_FRONTEND=noninteractive sudo apt-get install -y libc++-dev libc++abi-dev
-      - name: Install setuptools
-        run: python -m pip install setuptools
-      - name: Install flake8
-        run: python -m pip install flake8
-      - name: Fetch fixtures
-        run: ./script/fetch-fixtures
+          python-version: ${{matrix.python}}
       - name: Lint
-        run: ./script/lint
+        run: pipx run ruff .
+      - name: Build
+        run: pip install -e .
+        env:
+          CFLAGS: "-O0 -g"
       - name: Test
-        shell: bash
-        run: |
-          CFLAGS="-O0 -g" python setup.py test
+        run: python -Wignore:::tree_sitter -munittest

.github/workflows/pypi.yml

@@ -3,99 +3,79 @@ name: Upload to PyPI
 on:
   push:
     tags:
-      - "v[0-9]+.*"
-      - "test-v[0-9]+.*" # Uploads to https://test.pypi.org/project/tree-sitter/
+      - v[0-9]+.*
+      - test-v[0-9]+.*
 
 jobs:
   build-sdist:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-      - run: python setup.py sdist
-      - uses: actions/upload-artifact@v4
+      - name: Build sources
+        run: python -mbuild --sdist
+      - name: Upload sources
+        uses: actions/upload-artifact@v4
         with:
-          name: dist
-          path: ./dist/*
+          name: sdist
+          path: dist/*.tar.gz
 
   build-wheels:
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{matrix.os}}
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-11, windows-2022]
-
+        os: [ubuntu-latest, macos-13, windows-latest]
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
         with:
           submodules: true
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - if: ${{ startsWith(matrix.os, 'windows') }}
-        run: script\fetch-fixtures.cmd
-      - if: ${{ !startsWith(matrix.os, 'windows') }}
-        run: script/fetch-fixtures
-
       - name: Set up QEMU
         if: runner.os == 'Linux'
         uses: docker/setup-qemu-action@v3
         with:
-          platforms: all
-
-      # Build wheels
-      - run: pip install cibuildwheel==2.16.1
-      - run: python -m cibuildwheel --output-dir dist
+          platforms: arm64
+      - name: Build wheels
+        uses: pypa/[email protected]
         env:
-          CIBW_TEST_COMMAND: python -m unittest discover -s {package}/tests
           CIBW_ARCHS_MACOS: x86_64 arm64 universal2
           CIBW_ARCHS_LINUX: x86_64 aarch64
-
-      # Make wheels downloadable from GitHub UI and from the pypi step
-      - uses: actions/upload-artifact@v4
+          CIBW_TEST_SKIP: "*aarch64 *arm64"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
         with:
-          name: dist
-          path: ./dist/*.whl
+          name: wheels-${{matrix.os}}
+          path: wheelhouse/*.whl
 
   release:
     runs-on: ubuntu-latest
     needs: [build-sdist, build-wheels]
     steps:
-      - uses: actions/download-artifact@v4
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
         with:
-          name: dist
+          pattern: |-
+            sdist
+            wheels-*
           path: dist
-      - run: ls -la
-      - run: ls -la dist
-
-      # https://stackoverflow.com/a/58478262
-      - if: ${{ startsWith(github.ref, 'refs/tags/test-v') }}
-        name: "Upload to test.pypi.org"
+          merge-multiple: true
+      - name: Check artifacts
+        run: ls -la dist
+      - name: Upload to test.pypi.org
+        if: startsWith(github.ref_name, 'test-v')
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: __token__
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+          password: secrets.TEST_PYPI_API_TOKEN
           repository_url: https://test.pypi.org/legacy/
-
-      - if: ${{ startsWith(github.ref, 'refs/tags/v') }}
-        name: "Upload to pypi.org"
+      - name: Upload to pypi.org
+        if: startsWith(github.ref_name, 'v')
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-
-      - if: ${{ startsWith(github.ref, 'refs/tags/v') }}
-        name: "Create GitHub Release"
-        uses: softprops/action-gh-release@v1
-        with:
-          draft: false
-          prerelease: false
-          files: |
-            dist/*.whl
-            dist/*.tar.gz
-          generate_release_notes: true
+          password: ${{secrets.PYPI_API_TOKEN}}
+      - name: Create GitHub Release
+        if: startsWith(github.ref_name, 'v')
+        run: gh release create "$GITHUB_REF_NAME" dist/* --generate-notes
+        env:
+          GH_TOKEN: ${{github.token}}

.gitignore

@@ -1,8 +1,9 @@
 .tox
+.venv
 build
 dist
 *.pyc
 *.egg-info
 *.so
 __pycache__
-tests/fixtures
+wheelhouse

.gitmodules

@@ -1,3 +1,21 @@
 [submodule "tree-sitter"]
 	path = tree_sitter/core
 	url = https://github.com/tree-sitter/tree-sitter
+[submodule "tree-sitter-embedded-template"]
+	path = tests/fixtures/tree-sitter-embedded-template
+	url = https://github.com/tree-sitter/tree-sitter-embedded-template
+[submodule "tree-sitter-html"]
+	path = tests/fixtures/tree-sitter-html
+	url = https://github.com/tree-sitter/tree-sitter-html
+[submodule "tree-sitter-javascript"]
+	path = tests/fixtures/tree-sitter-javascript
+	url = https://github.com/tree-sitter/tree-sitter-javascript
+[submodule "tree-sitter-json"]
+	path = tests/fixtures/tree-sitter-json
+	url = https://github.com/tree-sitter/tree-sitter-json
+[submodule "tree-sitter-python"]
+	path = tests/fixtures/tree-sitter-python
+	url = https://github.com/tree-sitter/tree-sitter-python
+[submodule "tree-sitter-rust"]
+	path = tests/fixtures/tree-sitter-rust
+	url = https://github.com/tree-sitter/tree-sitter-rust

pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
     "Topic :: Text Processing :: Linguistic",
     "Typing :: Typed"
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.8,<3.12"
 readme = "README.md"
 
 [project.urls]
@@ -28,7 +28,24 @@ name = "Max Brunsfeld"
 email = "[email protected]"
 
 [tool.ruff]
+target-version = "py38"
 line-length = 100
+indent-width = 4
+exclude = [
+    ".git",
+    ".github",
+    ".venv",
+    "build",
+    "dist",
+    "__pycache__",
+    "tests/fixtures"
+]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
 
-[tool.ruff.pycodestyle]
-max-line-length = 100
+[tool.cibuildwheel]
+build-frontend = "build"
+test-command = "python -munittest discover -s {project}/tests"
+environment = {PYTHONWARNINGS = "ignore:::tree_sitter"}