feat(venv): configure and provision sysroots separately for each target

ruyi/config/__init__.py

@@ -229,6 +229,7 @@ class VenvCacheV0Type(TypedDict):
 
 class VenvCacheV1TargetType(TypedDict):
     toolchain_bindir: str
+    toolchain_sysroot: NotRequired[str]
     gcc_install_dir: NotRequired[str]
 
 
@@ -244,21 +245,29 @@ class VenvCacheRootType(TypedDict):
     cached_v1: NotRequired[VenvCacheV1Type]
 
 
-def parse_venv_cache(cache: VenvCacheRootType) -> VenvCacheV1Type:
+def parse_venv_cache(
+    cache: VenvCacheRootType,
+    global_sysroot: str | None,
+) -> VenvCacheV1Type:
     if "cached_v1" in cache:
         return cache["cached_v1"]
     if "cached" in cache:
-        return upgrade_venv_cache_v0(cache["cached"])
+        return upgrade_venv_cache_v0(cache["cached"], global_sysroot)
     raise RuntimeError("unsupported venv cache version")
 
 
-def upgrade_venv_cache_v0(x: VenvCacheV0Type) -> VenvCacheV1Type:
+def upgrade_venv_cache_v0(
+    x: VenvCacheV0Type,
+    global_sysroot: str | None,
+) -> VenvCacheV1Type:
     # v0 only supports one single target so upgrading is trivial
     v1_target: VenvCacheV1TargetType = {
         "toolchain_bindir": x["toolchain_bindir"],
     }
     if "gcc_install_dir" in x:
         v1_target["gcc_install_dir"] = x["gcc_install_dir"]
+    if global_sysroot is not None:
+        v1_target["toolchain_sysroot"] = global_sysroot
 
     y: VenvCacheV1Type = {
         "profile_common_flags": x["profile_common_flags"],
@@ -277,7 +286,7 @@ def __init__(self, cfg: VenvConfigRootType, cache: VenvCacheRootType) -> None:
         self.profile = cfg["config"]["profile"]
         self.sysroot = cfg["config"].get("sysroot")
 
-        parsed_cache = parse_venv_cache(cache)
+        parsed_cache = parse_venv_cache(cache, self.sysroot)
         self.targets = parsed_cache["targets"]
         self.profile_common_flags = parsed_cache["profile_common_flags"]
         self.qemu_bin = parsed_cache.get("qemu_bin")

ruyi/mux/runtime.py

@@ -30,6 +30,7 @@ def mux_main(argv: List[str]) -> int | NoReturn:
     # match the basename with one of the configured target tuples
     target_tuple: str | None = None
     toolchain_bindir: str | None = None
+    toolchain_sysroot: str | None = None
     gcc_install_dir: str | None = None
     for tgt_tuple, tgt_data in vcfg.targets.items():
         if not basename.startswith(f"{tgt_tuple}-"):
@@ -38,6 +39,7 @@ def mux_main(argv: List[str]) -> int | NoReturn:
         log.D(f"matched target '{tgt_tuple}', data {tgt_data}")
         target_tuple = tgt_tuple
         toolchain_bindir = tgt_data["toolchain_bindir"]
+        toolchain_sysroot = tgt_data.get("toolchain_sysroot")
         gcc_install_dir = tgt_data.get("gcc_install_dir")
         break
 
@@ -72,9 +74,9 @@ def mux_main(argv: List[str]) -> int | NoReturn:
         argv_to_insert.extend(shlex.split(vcfg.profile_common_flags))
         log.D(f"parsed profile flags: {argv_to_insert}")
 
-        if vcfg.sysroot is not None:
-            log.D(f"adding sysroot: {vcfg.sysroot}")
-            argv_to_insert.extend(("--sysroot", vcfg.sysroot))
+        if toolchain_sysroot is not None:
+            log.D(f"adding sysroot: {toolchain_sysroot}")
+            argv_to_insert.extend(("--sysroot", toolchain_sysroot))
 
     new_argv = [binpath]
     if argv_to_insert:

ruyi/mux/venv/__init__.py

@@ -5,6 +5,7 @@
 class ConfiguredTargetTuple(TypedDict):
     target: str
     toolchain_root: PathLike[Any]
+    toolchain_sysroot: PathLike[Any] | None
     binutils_flavor: str
     cc_flavor: str
     gcc_install_dir: PathLike[Any] | None

ruyi/mux/venv/data.py

@@ -8,5 +8,5 @@
     "toolchain.cmake": b"eNqFkVFrwjAUhd/7Ky6IMGF274M9dG2cZWlT0joUhJDVaIuajKQTR+l/X6zOWccwb+F+5+Scmx5MjIBNuRaPTs/pQb7lawGu68Lc3g5nGPiR94pYRgj2x14Ys1GI0VNdw07IHdNKVdA0D5VSm7zgpXSPFnOn7g+hXIL5Mi3Tb64MwzjNPIxZQtEonB4Mf9CmOcmFXFiHs9LGchwjqrujQTpLMxSx2IsQ4FJ+7gd/hwklPkpTQsH6f2iVC2OUti9csj7zSZTYVi2V59fj6bQL7PctUfcv+w1tzF/NKIwDRgnJWOJlY+iUa5UX1YzgOi9gaXMVgi+ENsDlwn7Ku+a6FAZKCVUhoOJ6JSqr3JVaya2Q1X1XbfutNN+eBYUyHfzffCwiATrs6oV6EcToDdHBDRiHz9SjMyAxnt1iw9jHkwCd2G+4PsQk",  # fmt: skip
     "meson-cross.ini": b"eNptkMFqwzAQRO/6ioUQ0tJazaU9FPoLvfUUglDkNV4iS0IrB4eQf69k2ZRCbmI083Z3NvDDCJbO+Ck2YgMDsnfAmMYATWOiZ246sgi3G1zQXVT0PsH9/jYbq0GSI5BSzoBvxJYXzF6+f8j9ixTicCKnIyEfhYEv2GWaMRmzEyaEVZimWbltG+h8hDNeXyHo1AO5ClS6bSmRd9qqFSgp4cBPz7C9iwzJoQxZiHN4RaJrC7XYtkAd8JXnW7JwOI1kU5PH+FDwecsQsaNp4azWiiqknC+5EH3AmOpdgz6jSt5b02tyam5tuezR17rX/1Xys1b8aHLzN1oces9JDdr05PBYcrmHkrLkxqn0OqpOD2SvaxnRG2TOFdTax4d6GaBd5aRkcSd+ATx1t64=",  # fmt: skip
     "binfmt.conf": b"eNpNUrlu20AQ7f0VAwgCbECk+wQp06Zwl0oakUNyoD2UvQRC1r/7LUkhrkjuzjuHOzqrG2w6Wo0ddd4NOlLMmvhshAYfKE0a6SPPSkVDymxIHN68s+LS4WVH6ijOMYntm5XrSQO05fSTogidLLtNqu1PC3EvidXEFhR/faY4+Wx6CjIqyAJ5J+QHyFcbxvibuhGYznDgpN69xrdD1Wa68QwOvl6DvwbltPqefYYGuIKecwXQO5yEop0QzPAIjXeS1B0I0+qKvwhoqp7YbDjhFIY5zAtVFDOgAV4GTgF9NP8wd6JbgLKE9uW+X3ThYqy+gkRvivTHehBp/wD57yfx/b7MtfB3NTwfHVuhxwMcDemw3qHmBQXcH49QaeJUrRCb6GnigiNfR4v29VaifF8NFUYZ2GIk4EJ2Sa1s+/KhR3qguZtUQNT5EKRLW/JaVhSUlLSLP+Bgi3Y5UKnwp71WsfX4+gabO2pqqAtC/MKz0CcWuiaCqb6i92s8fCFhU7+2Er79gUes69nDf9gX9HfteA==",  # fmt: skip
-    "ruyi-cache.toml": b"eNptkM9rgzAUx+/5K94c0g2GsOtgt/bQy3rxJiGkMWqoJluSBor4v++FqnVhJ2M+3x/v5Rm+TuXhA8pOOWhULwG/A9e8lTWcb2CvN1XA/hRlcNgfyydCKsFFJ2sW3in5tia6mDDDYDRret46+IRsHOFfNE0ZGXNQDfzI4crOSkM+rce7cf1FMWqlrlGeT7NviY0iqUMEj4GKhNJoaoyFyxsEwMiEF8rLwb28YgoWX7BxniEsk2J79CP/O8k21HPbSu82YZuJFjjnU+KN6UXHlY471soujdUuJTu6ea9QZa0QTGnned8zxBnFpuTuEZaAe1ayxLoc+QX4trGw",  # fmt: skip
+    "ruyi-cache.toml": b"eNptkM1ugzAQhO9+ii0VSitVSL1W6i055NJcuCFkOcaAFbAT20FCiHfvuuGvhJOxZ/abYV/h5xQfviAupYVcVgLwrJlihcjg3IK5tzKC/cnb4LA/xi+EJJzxUmS0+UzJ1Wg/Rbmua61oXrHCwjcEXQebUt8HpAtB5nAT9Z2epYKwnz4fg9MVzegVKkN72A9zI9abhGq8MBeKVmrqh3Jt4PIBDSBypUfSidq+vSMFgy+YOHRoxqaY7udR/99kCXXMFMLZBWzRaBQHfkqc1hUvmVT+HzNpxsRkt1Z26WJfTRLMum2t0doFKWY9vW4BB+lBfF4qwgvOqVTWsaqimP2HXr3N4JWwiZ02R34BhkzTWw==",  # fmt: skip
 }

ruyi/mux/venv/provision.py

@@ -51,7 +51,9 @@ def render_template_str(template_name: str, data: dict[str, Any]) -> str:
 
 
 def render_and_write(
-    dest: PathLike[Any], template_name: str, data: dict[str, Any]
+    dest: PathLike[Any],
+    template_name: str,
+    data: dict[str, Any],
 ) -> None:
     log.D(f"rendering template '{template_name}' with data {data}")
     content = render_template_str(template_name, data).encode("utf-8")
@@ -66,25 +68,44 @@ def __init__(
         profile: ProfileProxy,
         targets: list[ConfiguredTargetTuple],
         dest: PathLike[Any],
-        sysroot_srcdir: PathLike[Any] | None,
         emulator_progs: list[EmulatorProgDecl] | None,
         emulator_root: PathLike[Any] | None,
         override_name: str | None = None,
     ) -> None:
         self.profile = profile
         self.targets = targets
         self.venv_root = pathlib.Path(dest)
-        self.sysroot_srcdir = sysroot_srcdir
         self.emulator_progs = emulator_progs
         self.emulator_root = emulator_root
         self.override_name = override_name
 
         self.bindir = self.venv_root / "bin"
-        self.sysroot_destdir = (
-            self.venv_root / "sysroot"
-            if self.sysroot_srcdir is not None
-            else None
-        )
+
+    def sysroot_srcdir(self, target_tuple: str | None) -> pathlib.Path | None:
+        if target_tuple is None:
+            # check the primary target
+            if s := self.targets[0]["toolchain_sysroot"]:
+                return pathlib.Path(s)
+            return None
+
+        # check if we have this target
+        for t in self.targets:
+            if t["target"] != target_tuple:
+                continue
+            if s := t["toolchain_sysroot"]:
+                return pathlib.Path(s)
+
+        return None
+
+    def has_sysroot_for(self, target_tuple: str | None) -> bool:
+        return self.sysroot_srcdir(target_tuple) is not None
+
+    def sysroot_destdir(self, target_tuple: str | None) -> pathlib.Path | None:
+        if not self.has_sysroot_for(target_tuple):
+            return None
+
+        dirname = f"sysroot.{target_tuple}" if target_tuple is not None else "sysroot"
+        return self.venv_root / dirname
 
     def provision(self) -> None:
         venv_root = self.venv_root
@@ -93,19 +114,9 @@ def provision(self) -> None:
         venv_root.mkdir()
         bindir.mkdir()
 
-        if self.sysroot_srcdir is not None:
-            log.D("copying sysroot")
-            assert self.sysroot_destdir is not None
-            shutil.copytree(
-                self.sysroot_srcdir,
-                self.sysroot_destdir,
-                symlinks=True,
-                ignore_dangling_symlinks=True,
-            )
-
         env_data = {
             "profile": self.profile.id,
-            "sysroot": self.sysroot_destdir,
+            "sysroot": self.sysroot_destdir(None),
         }
         render_and_write(venv_root / "ruyi-venv.toml", "ruyi-venv.toml", env_data)
 
@@ -128,7 +139,7 @@ def provision(self) -> None:
                     p,
                     self.emulator_root,
                     self.profile,
-                    self.sysroot_destdir,
+                    self.sysroot_destdir(None),
                 )
                 for p in self.emulator_progs
             ]
@@ -166,6 +177,7 @@ def make_venv_cache_data(
         targets_cache_data: dict[str, object] = {
             tgt["target"]: {
                 "toolchain_bindir": str(pathlib.Path(tgt["toolchain_root"]) / "bin"),
+                "toolchain_sysroot": self.sysroot_destdir(tgt["target"]),
                 "gcc_install_dir": tgt["gcc_install_dir"],
             } for tgt in self.targets
         }
@@ -186,6 +198,26 @@ def provision_target(
         bindir = self.bindir
         target_tuple = tgt["target"]
 
+        # getting the destdir this way ensures it's suffixed with the target
+        # tuple
+        if sysroot_destdir := self.sysroot_destdir(target_tuple):
+            sysroot_srcdir = tgt["toolchain_sysroot"]
+            assert sysroot_srcdir is not None
+
+            log.D(f"copying sysroot for {target_tuple}")
+            shutil.copytree(
+                sysroot_srcdir,
+                sysroot_destdir,
+                symlinks=True,
+                ignore_dangling_symlinks=True,
+            )
+
+            if is_primary:
+                log.D("symlinking primary sysroot into place")
+                primary_sysroot_destdir = self.sysroot_destdir(None)
+                assert primary_sysroot_destdir is not None
+                os.symlink(sysroot_destdir.name, primary_sysroot_destdir)
+
         log.D(f"symlinking {target_tuple} binaries into venv")
         toolchain_bindir = pathlib.Path(tgt["toolchain_root"]) / "bin"
         symlink_binaries(toolchain_bindir, bindir)
@@ -235,7 +267,7 @@ def provision_target(
             "cc": cc_path,
             "cxx": cxx_path,
             "processor": self.profile.arch,
-            "sysroot": self.sysroot_destdir,
+            "sysroot": self.sysroot_destdir(target_tuple),
             "venv_root": venv_root,
             "cmake_toolchain_file": str(cmake_toolchain_file_path),
             "meson_additional_binaries": meson_additional_binaries,

ruyi/mux/venv/ruyi-cache.toml.jinja

@@ -9,5 +9,6 @@ profile_common_flags = "{{ profile_common_flags }}"
 {% endfor %}{% endif %}
 {% for k, v in targets.items() %}[cached_v1.targets.{{ k }}]
 toolchain_bindir = "{{ v['toolchain_bindir'] }}"
+{% if v["toolchain_sysroot"] %}toolchain_sysroot = "{{ v['toolchain_sysroot'] }}"{% endif %}
 {% if v["gcc_install_dir"] %}gcc_install_dir = "{{ v['gcc_install_dir'] }}"{% endif %}
 {% endfor %}

ruyi/mux/venv/venv_cli.py

@@ -41,7 +41,6 @@ def cli_venv(args: argparse.Namespace) -> int:
     target_arch = ''
     seen_target_tuples: set[str] = set()
     targets: list[ConfiguredTargetTuple] = []
-    sysroot_dir: PathLike[Any] | None = None
     warn_differing_target_arch = False
     warn_multiple_sysroots = False
 
@@ -82,7 +81,7 @@ def cli_venv(args: argparse.Namespace) -> int:
             log.F("cannot find the installed directory for the toolchain")
             return 1
 
-        tc_sysroot_dir: PathLike[Any]
+        tc_sysroot_dir: PathLike[Any] | None = None
         gcc_install_dir: PathLike[Any] | None = None
         if with_sysroot:
             if tc_sysroot_relpath := tc_pm.toolchain_metadata.included_sysroot:
@@ -144,18 +143,11 @@ def cli_venv(args: argparse.Namespace) -> int:
                     )
                     return 1
 
-            if sysroot_dir is None:
-                sysroot_dir = tc_sysroot_dir
-            else:
-                # check if multiple toolchain packages offer competing sysroots
-                if tc_sysroot_dir != sysroot_dir:
-                    # first one wins
-                    warn_multiple_sysroots = True
-
         # record the target tuple info to configure in the venv
         configured_target: ConfiguredTargetTuple = {
             "target": target_tuple,
             "toolchain_root": toolchain_root,
+            "toolchain_sysroot": tc_sysroot_dir,
             # assume clang is preferred if package contains clang
             # this is mostly true given most packages don't contain both
             "cc_flavor": "clang" if tc_pm.toolchain_metadata.has_clang else "gcc",
@@ -178,10 +170,6 @@ def cli_venv(args: argparse.Namespace) -> int:
         log.W("multiple toolchains specified with differing target architecture")
         log.I(f"using the target architecture of the first toolchain: [yellow]{target_arch}[/]")
 
-    if warn_multiple_sysroots:
-        log.W("selected packages for the virtual environment provide multiple competing sysroots")
-        log.I("using the sysroot from the first sysroot-providing package: [yellow]{sysroot_dir}[/]")
-
     # Now handle the emulator.
     emu_progs = None
     emu_root: PathLike[Any] | None = None
@@ -239,7 +227,6 @@ def cli_venv(args: argparse.Namespace) -> int:
         profile,
         targets,
         dest.resolve(),
-        sysroot_dir,
         emu_progs,
         emu_root,
         override_name,
@@ -250,7 +237,7 @@ def cli_venv(args: argparse.Namespace) -> int:
         render_template_str(
             "prompt.venv-created.txt",
             {
-                "sysroot": maker.sysroot_destdir,
+                "sysroot": maker.sysroot_destdir(None),
             },
         )
     )