Merge pull request #186 from xen0n/ci-updates

Build system improvements

.github/workflows/dist.yml

@@ -64,7 +64,7 @@ jobs:
       run_id: ${{ github.run_id }}
       release_mirror_url: ${{ steps.run_dist.outputs.release_mirror_url }}
     container:
-      image: ghcr.io/ruyisdk/ruyi-python-dist:20240623
+      image: ghcr.io/ruyisdk/ruyi-python-dist:20240827
       options: --user root  # https://github.com/actions/checkout/issues/1014
       credentials:
         username: ${{ github.actor }}

scripts/_image_tag_base.sh

@@ -1,6 +1,6 @@
 # this file is meant to be sourced
 
-_COMMON_DIST_IMAGE_TAG="ghcr.io/ruyisdk/ruyi-python-dist:20240623"
+_COMMON_DIST_IMAGE_TAG="ghcr.io/ruyisdk/ruyi-python-dist:20240827"
 
 # Map of `uname -m` outputs to Debian arch name convention which Ruyi adopts
 #

scripts/build-and-push-dist-image.sh

@@ -9,6 +9,6 @@ source "$MY_DIR/_image_tag_base.sh"
 cd "$MY_DIR/dist-image"
 exec docker buildx build --rm \
     --platform "linux/amd64,linux/arm64,linux/riscv64" \
-    -t "$(image_tag_base "")" \
+    -t "$(image_tag_base amd64)" \
     --push \
     .

scripts/build-pygit2.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -173,10 +174,11 @@ def get_pygit2_wheel_build_env(pygit2_dir: str) -> dict[str, str]:
         # this is unnecessary
         del r["LIBGIT2"]
 
-    # auditwheel 6.1.0+ has manylinux policies for riscv64, but the default
-    # is too low for our environment
-    # bump it up
-    r["AUDITWHEEL_PLAT"] = "manylinux_2_35_riscv64"
+    if platform.machine() == "riscv64":
+        # auditwheel 6.1.0+ has manylinux policies for riscv64, but the default
+        # is too low for our environment
+        # bump it up
+        r["AUDITWHEEL_PLAT"] = "manylinux_2_35_riscv64"
 
     return r
 

scripts/dist-image/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
-FROM --platform=linux/amd64 ubuntu:20.04 as build-amd64
-FROM --platform=linux/arm64 ubuntu:20.04 as build-arm64
-FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 as build-riscv64
+FROM --platform=linux/amd64 ubuntu:20.04 AS build-amd64
+FROM --platform=linux/arm64 ubuntu:20.04 AS build-arm64
+FROM --platform=linux/riscv64 riscv64/ubuntu:22.04 AS build-riscv64
 
 FROM build-$TARGETARCH
 

scripts/dist-image/prepare-distro.ubuntu2204-riscv64.sh

@@ -55,7 +55,7 @@ apt-get clean
 #
 # See: https://github.com/Nuitka/Nuitka/commit/54f2a2222abedf92d45b8f397233cfb3bef340c5
 
-PYTHON_V=3.12.4
+PYTHON_V=3.12.5
 pushd /tmp
 wget https://www.python.org/ftp/python/${PYTHON_V}/Python-${PYTHON_V}.tar.xz
 mkdir py-src py-build

scripts/dist-image/prepare-poetry.sh

@@ -9,7 +9,7 @@ export MAKEFLAGS="-j$(nproc)"
 python3.12 -m venv /home/b/build-tools-venv
 /home/b/build-tools-venv/bin/pip install -U pip setuptools wheel
 /home/b/build-tools-venv/bin/pip install poetry
-/home/b/build-tools-venv/bin/pip install maturin==1.6.0 cibuildwheel==2.19.1 auditwheel==6.0.0
+/home/b/build-tools-venv/bin/pip install maturin==1.7.1 cibuildwheel==2.20.0 auditwheel==6.1.0
 for tool in poetry maturin cibuildwheel auditwheel; do
     ln -s /home/b/build-tools-venv/bin/"$tool" /usr/local/bin/"$tool"
 done

scripts/dist-inner.py

@@ -130,8 +130,8 @@ def delete_cached_files_older_than_days(root: str, days: int, epoch: int) -> Non
     root_path = pathlib.Path(root)
     dirs_to_remove: list[tuple[pathlib.Path, int | None]] = []
     for f in root_path.iterdir():
-        if f.name.startswith("pygit2"):
-            INFO.print(f"ignoring pygit2 cache [cyan]{f}")
+        if f.name.startswith("pygit2") or f.name.startswith("xingque"):
+            INFO.print(f"ignoring library artifact cache [cyan]{f}")
             continue
 
         ts: int | None

scripts/dist.sh

@@ -130,21 +130,49 @@ main() {
         do_inner "$@"
     fi
 
-    local arch="$1"
-    if [[ -z $arch ]]; then
-        arch="$(convert_uname_arch_to_ruyi "$(uname -m)")"
+    local build_arch="$(convert_uname_arch_to_ruyi "$(uname -m)")"
+    local build_arch_is_officially_supported=false
+    if is_docker_dist_build_supported "$build_arch"; then
+        build_arch_is_officially_supported=true
+    fi
+
+    local host_arch="$1"
+    if [[ -z $host_arch ]]; then
+        host_arch="$build_arch"
         echo "usage: $0 [arch]" >&2
-        echo "info: defaulting to host arch $arch" >&2
+        echo "info: defaulting to build machine arch $build_arch" >&2
     fi
 
-    if is_docker_dist_build_supported "$arch"; then
-        do_docker_build "$arch"
+    if is_docker_dist_build_supported "$host_arch"; then
+        do_docker_build "$host_arch"
     else
-        echo "warning: Docker-based dist builds for architecture $arch is not supported" >&2
+        echo "warning: Docker-based dist builds for architecture $host_arch is not supported" >&2
         if [[ -n "$RUYI_DIST_FORCE_IMAGE_TAG" ]]; then
             # but this is explicitly requested so...
-            do_docker_build "$arch"
+            do_docker_build "$host_arch"
         else
+            # Because of the way Nuitka works, cross builds cannot be supported.
+            #
+            # But without knowledge of the Debian name for the user's arch, we
+            # cannot know whether the user is actually doing native builds on
+            # their arch, with $build_arch expected to differ from `uname -m`
+            # output.
+            #
+            # On the other hand, if the build arch is supported, when
+            # $host_arch differs from $build_arch we can indeed be sure that
+            # the build will fail.
+            if [[ $build_arch != $host_arch ]]; then
+                if "$build_arch_is_officially_supported"; then
+                    echo "error: cross building is not possible with Nuitka" >&2
+                    echo "info: to our knowledge, $host_arch is not the same as $build_arch" >&2
+                    echo "info: please retry with $host_arch hardware / emulation / sysroot instead" >&2
+                    exit 1
+                fi
+
+                echo "warning: the requested arch $host_arch differs from the build machine arch $build_arch, but the build is not Docker-based" >&2
+                echo "warning: cross builds are not supported and will fail" >&2
+            fi
+
             echo "warning: your build may not be reproducible" >&2
             do_inner "$@"
         fi