Merge pull request #216 from xen0n/issue209-llvm-tools-0.20

[0.20.x] Fix LLVM tools proxying for good

ruyi/config/__init__.py

@@ -281,11 +281,17 @@ class VenvCacheV1TargetType(TypedDict):
     gcc_install_dir: NotRequired[str]
 
 
+class VenvCacheV1CmdMetadataEntryType(TypedDict):
+    dest: str
+    target_tuple: str
+
+
 class VenvCacheV1Type(TypedDict):
     profile_common_flags: str
     profile_emu_env: NotRequired[dict[str, str]]
     qemu_bin: NotRequired[str]
     targets: dict[str, VenvCacheV1TargetType]
+    cmd_metadata_map: NotRequired[dict[str, VenvCacheV1CmdMetadataEntryType]]
 
 
 class VenvCacheRootType(TypedDict):
@@ -330,7 +336,13 @@ def upgrade_venv_cache_v0(
 
 
 class RuyiVenvConfig:
-    def __init__(self, cfg: VenvConfigRootType, cache: VenvCacheRootType) -> None:
+    def __init__(
+        self,
+        venv_root: pathlib.Path,
+        cfg: VenvConfigRootType,
+        cache: VenvCacheRootType,
+    ) -> None:
+        self.venv_root = venv_root
         self.profile = cfg["config"]["profile"]
         self.sysroot = cfg["config"].get("sysroot")
 
@@ -339,13 +351,18 @@ def __init__(self, cfg: VenvConfigRootType, cache: VenvCacheRootType) -> None:
         self.profile_common_flags = parsed_cache["profile_common_flags"]
         self.qemu_bin = parsed_cache.get("qemu_bin")
         self.profile_emu_env = parsed_cache.get("profile_emu_env")
+        self.cmd_metadata_map = parsed_cache.get("cmd_metadata_map")
+
+        # this must be in sync with provision.py
+        self._ruyi_priv_dir = self.venv_root / "ruyi-private"
+        self._cached_cmd_targets_dir = self._ruyi_priv_dir / "cached-cmd-targets"
 
     @classmethod
     def explicit_ruyi_venv_root(cls) -> str | None:
         return os.environ.get(ENV_VENV_ROOT_KEY)
 
     @classmethod
-    def venv_root(cls) -> pathlib.Path | None:
+    def probe_venv_root(cls) -> pathlib.Path | None:
         if explicit_root := cls.explicit_ruyi_venv_root():
             return pathlib.Path(explicit_root)
 
@@ -365,7 +382,7 @@ def venv_root(cls) -> pathlib.Path | None:
 
     @classmethod
     def load_from_venv(cls) -> Self | None:
-        venv_root = cls.venv_root()
+        venv_root = cls.probe_venv_root()
         if venv_root is None:
             return None
 
@@ -390,4 +407,15 @@ def load_from_venv(cls) -> Self | None:
 
         # NOTE: for now it's not prohibited to have v1 cache data in the v0
         # cache path, but this situation is harmless
-        return cls(cfg, cache)
+        return cls(venv_root, cfg, cache)
+
+    def resolve_cmd_metadata_with_cache(
+        self,
+        basename: str,
+    ) -> VenvCacheV1CmdMetadataEntryType | None:
+        if self.cmd_metadata_map is None:
+            # we are operating in a venv created with an older ruyi, thus no
+            # cmd_metadata_map in cache
+            return None
+
+        return self.cmd_metadata_map.get(basename)

ruyi/mux/runtime.py

@@ -27,39 +27,45 @@ def mux_main(argv: List[str]) -> int | NoReturn:
     if basename == "ruyi-qemu":
         return mux_qemu_main(argv, vcfg)
 
-    is_unprefixed_clang = is_proxying_to_unprefixed_clang(basename)
     # match the basename with one of the configured target tuples
     target_tuple: str | None = None
-    toolchain_bindir: str | None = None
+    binpath: str | None = None
     toolchain_sysroot: str | None = None
     gcc_install_dir: str | None = None
-    for tgt_tuple, tgt_data in vcfg.targets.items():
-        if not basename.startswith(f"{tgt_tuple}-"):
-            # Unprefixed Clang should be treated as having the "default"
-            # target tuple.
-            if not is_unprefixed_clang:
-                continue
 
-        log.D(f"matched target '{tgt_tuple}', data {tgt_data}")
-        target_tuple = tgt_tuple
-        toolchain_bindir = tgt_data["toolchain_bindir"]
+    # prefer v1 cached info which is lossless
+    if md := vcfg.resolve_cmd_metadata_with_cache(basename):
+        target_tuple = md["target_tuple"]
+        binpath = md["dest"]
+        tgt_data = vcfg.targets[target_tuple]
         toolchain_sysroot = tgt_data.get("toolchain_sysroot")
         gcc_install_dir = tgt_data.get("gcc_install_dir")
-        break
+    else:
+        toolchain_bindir: str | None = None
+        for tgt_tuple, tgt_data in vcfg.targets.items():
+            if not basename.startswith(f"{tgt_tuple}-"):
+                continue
+
+            log.D(f"matched target '{tgt_tuple}', data {tgt_data}")
+            target_tuple = tgt_tuple
+            toolchain_bindir = tgt_data["toolchain_bindir"]
+            toolchain_sysroot = tgt_data.get("toolchain_sysroot")
+            gcc_install_dir = tgt_data.get("gcc_install_dir")
+            break
+
+        if toolchain_bindir is None:
+            # should not happen
+            log.F(
+                f"internal error: no bindir configured for target [yellow]{target_tuple}[/]"
+            )
+            return 1
+
+        binpath = os.path.join(toolchain_bindir, basename)
 
     if target_tuple is None:
         log.F(f"no configured target found for command [yellow]{basename}[/]")
         return 1
 
-    if toolchain_bindir is None:
-        # should not happen
-        log.F(
-            f"internal error: no bindir configured for target [yellow]{target_tuple}[/]"
-        )
-        return 1
-
-    binpath = os.path.join(toolchain_bindir, basename)
-
     log.D(f"binary to exec: {binpath}")
 
     argv_to_insert: list[str] | None = None
@@ -119,14 +125,11 @@ def resolve_argv0_symlink(argv0: str, vcfg: RuyiVenvConfig) -> str | None:
 
     # argv[0] is bare command name, in which case we expect venv root to
     # be available, so we can just check f'{venv_root}/bin/{argv[0]}'.
-    if venv_root := vcfg.venv_root():
-        try:
-            return os.readlink(venv_root / "bin" / argv0)
-        except OSError:
-            return None
-
-    # no venv and argv[0] isn't absolute -- no way to figure out
-    return None
+    # we're guaranteed a venv_root because of the vcfg init logic.
+    try:
+        return os.readlink(vcfg.venv_root / "bin" / argv0)
+    except OSError:
+        return None
 
 
 def is_proxying_to_cc(argv0: str) -> bool:
@@ -137,10 +140,6 @@ def is_proxying_to_clang(basename: str) -> bool:
     return "clang" in basename
 
 
-def is_proxying_to_unprefixed_clang(basename: str) -> bool:
-    return basename in ("clang", "clang++")
-
-
 def mux_qemu_main(argv: List[str], vcfg: RuyiVenvConfig) -> int | NoReturn:
     binpath = vcfg.qemu_bin
     if binpath is None:
@@ -163,8 +162,7 @@ def mux_qemu_main(argv: List[str], vcfg: RuyiVenvConfig) -> int | NoReturn:
 
 
 def ensure_venv_in_path(vcfg: RuyiVenvConfig) -> None:
-    venv_root = vcfg.venv_root()
-    assert venv_root is not None
+    venv_root = vcfg.venv_root
     venv_bindir = venv_root / "bin"
     venv_bindir = venv_bindir.resolve()
 

ruyi/mux/venv/data.py

@@ -8,5 +8,5 @@
     "toolchain.cmake": b"eNqFkVFrwjAUhd/7Ky6IMGF274M9dG2cZWlT0joUhJDVaIuajKQTR+l/X6zOWccwb+F+5+Scmx5MjIBNuRaPTs/pQb7lawGu68Lc3g5nGPiR94pYRgj2x14Ys1GI0VNdw07IHdNKVdA0D5VSm7zgpXSPFnOn7g+hXIL5Mi3Tb64MwzjNPIxZQtEonB4Mf9CmOcmFXFiHs9LGchwjqrujQTpLMxSx2IsQ4FJ+7gd/hwklPkpTQsH6f2iVC2OUti9csj7zSZTYVi2V59fj6bQL7PctUfcv+w1tzF/NKIwDRgnJWOJlY+iUa5UX1YzgOi9gaXMVgi+ENsDlwn7Ku+a6FAZKCVUhoOJ6JSqr3JVaya2Q1X1XbfutNN+eBYUyHfzffCwiATrs6oV6EcToDdHBDRiHz9SjMyAxnt1iw9jHkwCd2G+4PsQk",  # fmt: skip
     "meson-cross.ini": b"eNptkMFqwzAQRO/6ioUQ0tJazaU9FPoLvfUUglDkNV4iS0IrB4eQf69k2ZRCbmI083Z3NvDDCJbO+Ck2YgMDsnfAmMYATWOiZ246sgi3G1zQXVT0PsH9/jYbq0GSI5BSzoBvxJYXzF6+f8j9ixTicCKnIyEfhYEv2GWaMRmzEyaEVZimWbltG+h8hDNeXyHo1AO5ClS6bSmRd9qqFSgp4cBPz7C9iwzJoQxZiHN4RaJrC7XYtkAd8JXnW7JwOI1kU5PH+FDwecsQsaNp4azWiiqknC+5EH3AmOpdgz6jSt5b02tyam5tuezR17rX/1Xys1b8aHLzN1oces9JDdr05PBYcrmHkrLkxqn0OqpOD2SvaxnRG2TOFdTax4d6GaBd5aRkcSd+ATx1t64=",  # fmt: skip
     "binfmt.conf": b"eNpNUrlu20AQ7f0VAwgCbECk+wQp06Zwl0oakUNyoD2UvQRC1r/7LUkhrkjuzjuHOzqrG2w6Wo0ddd4NOlLMmvhshAYfKE0a6SPPSkVDymxIHN68s+LS4WVH6ijOMYntm5XrSQO05fSTogidLLtNqu1PC3EvidXEFhR/faY4+Wx6CjIqyAJ5J+QHyFcbxvibuhGYznDgpN69xrdD1Wa68QwOvl6DvwbltPqefYYGuIKecwXQO5yEop0QzPAIjXeS1B0I0+qKvwhoqp7YbDjhFIY5zAtVFDOgAV4GTgF9NP8wd6JbgLKE9uW+X3ThYqy+gkRvivTHehBp/wD57yfx/b7MtfB3NTwfHVuhxwMcDemw3qHmBQXcH49QaeJUrRCb6GnigiNfR4v29VaifF8NFUYZ2GIk4EJ2Sa1s+/KhR3qguZtUQNT5EKRLW/JaVhSUlLSLP+Bgi3Y5UKnwp71WsfX4+gabO2pqqAtC/MKz0CcWuiaCqb6i92s8fCFhU7+2Er79gUes69nDf9gX9HfteA==",  # fmt: skip
-    "ruyi-cache.toml": b"eNptkM1ugzAQhO9+ii0VSitVSL1W6i055NJcuCFkOcaAFbAT20FCiHfvuuGvhJOxZ/abYV/h5xQfviAupYVcVgLwrJlihcjg3IK5tzKC/cnb4LA/xi+EJJzxUmS0+UzJ1Wg/Rbmua61oXrHCwjcEXQebUt8HpAtB5nAT9Z2epYKwnz4fg9MVzegVKkN72A9zI9abhGq8MBeKVmrqh3Jt4PIBDSBypUfSidq+vSMFgy+YOHRoxqaY7udR/99kCXXMFMLZBWzRaBQHfkqc1hUvmVT+HzNpxsRkt1Z26WJfTRLMum2t0doFKWY9vW4BB+lBfF4qwgvOqVTWsaqimP2HXr3N4JWwiZ02R34BhkzTWw==",  # fmt: skip
+    "ruyi-cache.toml": b"eNptUk1rhDAUvPsrXi3LtlCEXgu97R566V68iYRsEt2wJrEmCiL+977Uz7qeTN7Mm5kMPsP3JT5/QHyTFjJZCMCvoprmgsO1hapuZQSni6fB+fQVPwVBwii7CU6a9zQoK+O3CDNKGU2yguYWPiHsOtiF+j4MugPIDH6EqslVajj083FYnK9IRq7QHOmHftybZD1J6MYDS6Bog6Z+KTMV3N+gAZTc4JF0QtmXV1RB4zs6jhmaKSm6+33E/ydZizpa5cLZldgq0QSO+mngjCnYjUrt38hlNTkmxy1yTFd9NUm44La1lTEuTNHrYbonOEKD4mOpKJ4zRqS2jhYFQe8/6c1sEd4Au7Jzc9u6mOJECUc5dZQoWu739sAKxwbDNODCrl7pb2NVQ9nE1SX+yUsNq+nS6ZLvF/cICe0=",  # fmt: skip
 }

ruyi/mux/venv/provision.py

@@ -6,7 +6,7 @@
 import re
 import shlex
 import shutil
-from typing import Any, Callable, Tuple
+from typing import Any, Callable, Iterator, Tuple, TypedDict
 import zlib
 
 from jinja2 import BaseLoader, Environment, TemplateNotFound
@@ -179,14 +179,18 @@ def make_venv_cache_data(
                 "toolchain_bindir": str(pathlib.Path(tgt["toolchain_root"]) / "bin"),
                 "toolchain_sysroot": self.sysroot_destdir(tgt["target"]),
                 "gcc_install_dir": tgt["gcc_install_dir"],
-            } for tgt in self.targets
+            }
+            for tgt in self.targets
         }
 
+        cmd_metadata_map = make_cmd_metadata_map(self.targets)
+
         return {
             "profile_common_flags": self.profile.get_common_flags(),
             "profile_emu_env": profile_emu_env,
             "qemu_bin": qemu_bin,
             "targets": targets_cache_data,
+            "cmd_metadata_map": cmd_metadata_map,
         }
 
     def provision_target(
@@ -286,27 +290,63 @@ def provision_target(
         )
 
         if is_primary:
-            log.D(f"making cmake & meson file symlinks to primary target {target_tuple}")
+            log.D(
+                f"making cmake & meson file symlinks to primary target {target_tuple}"
+            )
             primary_cmake_toolchain_file_path = venv_root / "toolchain.cmake"
             primary_meson_cross_file_path = venv_root / "meson-cross.ini"
-            os.symlink(cmake_toolchain_file_path.name, primary_cmake_toolchain_file_path)
+            os.symlink(
+                cmake_toolchain_file_path.name,
+                primary_cmake_toolchain_file_path,
+            )
             os.symlink(meson_cross_file_path.name, primary_meson_cross_file_path)
 
 
-def symlink_binaries(src_bindir: PathLike[Any], dest_bindir: PathLike[Any]) -> None:
-    src_binpath = pathlib.Path(src_bindir)
-    dest_binpath = pathlib.Path(dest_bindir)
-    self_exe_path = self_exe()
-
-    for filename in glob.iglob("*", root_dir=src_bindir):
-        if not is_executable(src_binpath / filename):
+def iter_binaries_to_symlink(bindir: pathlib.Path) -> Iterator[pathlib.Path]:
+    for filename in glob.iglob("*", root_dir=bindir):
+        src_cmd_path = bindir / filename
+        if not is_executable(src_cmd_path):
             log.D(f"skipping non-executable {filename} in src bindir")
             continue
 
         if should_ignore_symlinking(filename):
             log.D(f"skipping command {filename} explicitly")
             continue
 
+        yield bindir / filename
+
+
+class CmdMetadataEntry(TypedDict):
+    dest: str
+    target_tuple: str
+
+
+def make_cmd_metadata_map(
+    targets: list[ConfiguredTargetTuple],
+) -> dict[str, CmdMetadataEntry]:
+    result: dict[str, CmdMetadataEntry] = {}
+    for tgt in targets:
+        # TODO: dedup this and provision_target
+        toolchain_bindir = pathlib.Path(tgt["toolchain_root"]) / "bin"
+        for cmd in iter_binaries_to_symlink(toolchain_bindir):
+            result[cmd.name] = {
+                "dest": str(cmd),
+                "target_tuple": tgt["target"],
+            }
+    return result
+
+
+def symlink_binaries(
+    src_bindir: PathLike[Any],
+    dest_bindir: PathLike[Any],
+) -> None:
+    src_binpath = pathlib.Path(src_bindir)
+    dest_binpath = pathlib.Path(dest_bindir)
+    self_exe_path = self_exe()
+
+    for src_cmd_path in iter_binaries_to_symlink(src_binpath):
+        filename = src_cmd_path.name
+
         # symlink self to dest with the name of this command
         dest_path = dest_binpath / filename
         log.D(f"making ruyi symlink to {self_exe_path} at {dest_path}")

ruyi/mux/venv/ruyi-cache.toml.jinja

@@ -12,3 +12,7 @@ toolchain_bindir = "{{ v['toolchain_bindir'] }}"
 {% if v["toolchain_sysroot"] %}toolchain_sysroot = "{{ v['toolchain_sysroot'] }}"{% endif %}
 {% if v["gcc_install_dir"] %}gcc_install_dir = "{{ v['gcc_install_dir'] }}"{% endif %}
 {% endfor %}
+{% for k, v in cmd_metadata_map.items() %}[cached_v1.cmd_metadata_map."{{ k }}"]
+dest = "{{ v['dest'] }}"
+target_tuple = "{{ v['target_tuple'] }}"
+{% endfor %}