Merge pull request #204 from xen0n/issue199

Support multiple toolchains in one venv

ruyi/cli/__init__.py

@@ -26,7 +26,7 @@ def is_called_as_ruyi(argv0: str) -> bool:
 
 def init_argparse() -> argparse.ArgumentParser:
     from ..device.provision_cli import cli_device_provision
-    from ..mux.venv import cli_venv
+    from ..mux.venv.venv_cli import cli_venv
     from ..ruyipkg.admin_cli import cli_admin_format_manifest, cli_admin_manifest
     from ..ruyipkg.host import get_native_host
     from ..ruyipkg.news_cli import cli_news_list, cli_news_read
@@ -203,7 +203,8 @@ def init_argparse() -> argparse.ArgumentParser:
         "--toolchain",
         "-t",
         type=str,
-        help="Specifier (atom) of the toolchain package to use",
+        action="append",
+        help="Specifier(s) (atoms) of the toolchain package(s) to use",
     )
     venv.add_argument(
         "--emulator",

ruyi/config/__init__.py

@@ -218,7 +218,7 @@ class VenvConfigRootType(TypedDict):
     config: VenvConfigType
 
 
-class VenvCacheType(TypedDict):
+class VenvCacheV0Type(TypedDict):
     target_tuple: str
     toolchain_bindir: str
     gcc_install_dir: NotRequired[str]
@@ -227,20 +227,70 @@ class VenvCacheType(TypedDict):
     profile_emu_env: NotRequired[dict[str, str]]
 
 
+class VenvCacheV1TargetType(TypedDict):
+    toolchain_bindir: str
+    toolchain_sysroot: NotRequired[str]
+    gcc_install_dir: NotRequired[str]
+
+
+class VenvCacheV1Type(TypedDict):
+    profile_common_flags: str
+    profile_emu_env: NotRequired[dict[str, str]]
+    qemu_bin: NotRequired[str]
+    targets: dict[str, VenvCacheV1TargetType]
+
+
 class VenvCacheRootType(TypedDict):
-    cached: VenvCacheType
+    cached: NotRequired[VenvCacheV0Type]
+    cached_v1: NotRequired[VenvCacheV1Type]
+
+
+def parse_venv_cache(
+    cache: VenvCacheRootType,
+    global_sysroot: str | None,
+) -> VenvCacheV1Type:
+    if "cached_v1" in cache:
+        return cache["cached_v1"]
+    if "cached" in cache:
+        return upgrade_venv_cache_v0(cache["cached"], global_sysroot)
+    raise RuntimeError("unsupported venv cache version")
+
+
+def upgrade_venv_cache_v0(
+    x: VenvCacheV0Type,
+    global_sysroot: str | None,
+) -> VenvCacheV1Type:
+    # v0 only supports one single target so upgrading is trivial
+    v1_target: VenvCacheV1TargetType = {
+        "toolchain_bindir": x["toolchain_bindir"],
+    }
+    if "gcc_install_dir" in x:
+        v1_target["gcc_install_dir"] = x["gcc_install_dir"]
+    if global_sysroot is not None:
+        v1_target["toolchain_sysroot"] = global_sysroot
+
+    y: VenvCacheV1Type = {
+        "profile_common_flags": x["profile_common_flags"],
+        "targets": {x["target_tuple"]: v1_target},
+    }
+    if "profile_emu_env" in x:
+        y["profile_emu_env"] = x["profile_emu_env"]
+    if "qemu_bin" in x:
+        y["qemu_bin"] = x["qemu_bin"]
+
+    return y
 
 
 class RuyiVenvConfig:
     def __init__(self, cfg: VenvConfigRootType, cache: VenvCacheRootType) -> None:
         self.profile = cfg["config"]["profile"]
         self.sysroot = cfg["config"].get("sysroot")
-        self.target_tuple = cache["cached"]["target_tuple"]
-        self.toolchain_bindir = cache["cached"]["toolchain_bindir"]
-        self.gcc_install_dir = cache["cached"].get("gcc_install_dir")
-        self.profile_common_flags = cache["cached"]["profile_common_flags"]
-        self.qemu_bin = cache["cached"].get("qemu_bin")
-        self.profile_emu_env = cache["cached"].get("profile_emu_env")
+
+        parsed_cache = parse_venv_cache(cache, self.sysroot)
+        self.targets = parsed_cache["targets"]
+        self.profile_common_flags = parsed_cache["profile_common_flags"]
+        self.qemu_bin = parsed_cache.get("qemu_bin")
+        self.profile_emu_env = parsed_cache.get("profile_emu_env")
 
     @classmethod
     def explicit_ruyi_venv_root(cls) -> str | None:
@@ -280,8 +330,16 @@ def load_from_venv(cls) -> Self | None:
         with open(venv_config_path, "rb") as fp:
             cfg: Any = tomllib.load(fp)  # in order to cast to our stricter type
 
-        venv_cache_path = venv_root / "ruyi-cache.toml"
-        with open(venv_cache_path, "rb") as fp:
-            cache: Any = tomllib.load(fp)  # in order to cast to our stricter type
-
+        cache: Any  # in order to cast to our stricter type
+        venv_cache_v1_path = venv_root / "ruyi-cache.v1.toml"
+        try:
+            with open(venv_cache_v1_path, "rb") as fp:
+                cache = tomllib.load(fp)
+        except FileNotFoundError:
+            venv_cache_v0_path = venv_root / "ruyi-cache.toml"
+            with open(venv_cache_v0_path, "rb") as fp:
+                cache = tomllib.load(fp)
+
+        # NOTE: for now it's not prohibited to have v1 cache data in the v0
+        # cache path, but this situation is harmless
         return cls(cfg, cache)

ruyi/mux/runtime.py

@@ -27,7 +27,34 @@ def mux_main(argv: List[str]) -> int | NoReturn:
     if basename == "ruyi-qemu":
         return mux_qemu_main(argv, vcfg)
 
-    binpath = os.path.join(vcfg.toolchain_bindir, basename)
+    # match the basename with one of the configured target tuples
+    target_tuple: str | None = None
+    toolchain_bindir: str | None = None
+    toolchain_sysroot: str | None = None
+    gcc_install_dir: str | None = None
+    for tgt_tuple, tgt_data in vcfg.targets.items():
+        if not basename.startswith(f"{tgt_tuple}-"):
+            continue
+
+        log.D(f"matched target '{tgt_tuple}', data {tgt_data}")
+        target_tuple = tgt_tuple
+        toolchain_bindir = tgt_data["toolchain_bindir"]
+        toolchain_sysroot = tgt_data.get("toolchain_sysroot")
+        gcc_install_dir = tgt_data.get("gcc_install_dir")
+        break
+
+    if target_tuple is None:
+        log.F(f"no configured target found for command [yellow]{basename}[/]")
+        return 1
+
+    if toolchain_bindir is None:
+        # should not happen
+        log.F(
+            f"internal error: no bindir configured for target [yellow]{target_tuple}[/]"
+        )
+        return 1
+
+    binpath = os.path.join(toolchain_bindir, basename)
 
     log.D(f"binary to exec: {binpath}")
 
@@ -38,18 +65,18 @@ def mux_main(argv: List[str]) -> int | NoReturn:
         argv_to_insert = []
 
         if is_proxying_to_clang(basename):
-            log.D(f"adding target for clang: {vcfg.target_tuple}")
-            argv_to_insert.append(f"--target={vcfg.target_tuple}")
-            if vcfg.gcc_install_dir is not None:
-                log.D(f"informing clang of GCC install dir: {vcfg.gcc_install_dir}")
-                argv_to_insert.append(f"--gcc-install-dir={vcfg.gcc_install_dir}")
+            log.D(f"adding target for clang: {target_tuple}")
+            argv_to_insert.append(f"--target={target_tuple}")
+            if gcc_install_dir is not None:
+                log.D(f"informing clang of GCC install dir: {gcc_install_dir}")
+                argv_to_insert.append(f"--gcc-install-dir={gcc_install_dir}")
 
         argv_to_insert.extend(shlex.split(vcfg.profile_common_flags))
         log.D(f"parsed profile flags: {argv_to_insert}")
 
-        if vcfg.sysroot is not None:
-            log.D(f"adding sysroot: {vcfg.sysroot}")
-            argv_to_insert.extend(("--sysroot", vcfg.sysroot))
+        if toolchain_sysroot is not None:
+            log.D(f"adding sysroot: {toolchain_sysroot}")
+            argv_to_insert.extend(("--sysroot", toolchain_sysroot))
 
     new_argv = [binpath]
     if argv_to_insert: