From f1217aa34cb27f3b787f57e49815e761ab58f3d5 Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Mon, 23 Sep 2024 16:42:45 +0800 Subject: [PATCH 1/2] feat(venv): [breaking change] allow multiple non-conflicting toolchains Revise the venv cache format to allow configuring each supported target tuple separately. Potential conflicting "provides" are resolved mainly on a "first-one-wins" basis. It is not recommended to mix-and-match `ruyi` and venvs created with previous versions of `ruyi`, but hopefully compatibility is preserved. Auto-upgrading of venv cached config format is left out for now. Fixes: #199 --- ruyi/cli/__init__.py | 5 +- ruyi/config/__init__.py | 73 ++++++-- ruyi/mux/runtime.py | 37 +++- ruyi/mux/venv/__init__.py | 238 +----------------------- ruyi/mux/venv/data.py | 2 +- ruyi/mux/venv/provision.py | 201 +++++++++++--------- ruyi/mux/venv/ruyi-cache.toml.jinja | 11 +- ruyi/mux/venv/venv_cli.py | 275 ++++++++++++++++++++++++++++ 8 files changed, 498 insertions(+), 344 deletions(-) create mode 100644 ruyi/mux/venv/venv_cli.py diff --git a/ruyi/cli/__init__.py b/ruyi/cli/__init__.py index 715304d..d42f98c 100644 --- a/ruyi/cli/__init__.py +++ b/ruyi/cli/__init__.py @@ -26,7 +26,7 @@ def is_called_as_ruyi(argv0: str) -> bool: def init_argparse() -> argparse.ArgumentParser: from ..device.provision_cli import cli_device_provision - from ..mux.venv import cli_venv + from ..mux.venv.venv_cli import cli_venv from ..ruyipkg.admin_cli import cli_admin_format_manifest, cli_admin_manifest from ..ruyipkg.host import get_native_host from ..ruyipkg.news_cli import cli_news_list, cli_news_read @@ -203,7 +203,8 @@ def init_argparse() -> argparse.ArgumentParser: "--toolchain", "-t", type=str, - help="Specifier (atom) of the toolchain package to use", + action="append", + help="Specifier(s) (atoms) of the toolchain package(s) to use", ) venv.add_argument( "--emulator", diff --git a/ruyi/config/__init__.py b/ruyi/config/__init__.py index ce81243..72cee19 100644 --- a/ruyi/config/__init__.py +++ b/ruyi/config/__init__.py @@ -218,7 +218,7 @@ class VenvConfigRootType(TypedDict): config: VenvConfigType -class VenvCacheType(TypedDict): +class VenvCacheV0Type(TypedDict): target_tuple: str toolchain_bindir: str gcc_install_dir: NotRequired[str] @@ -227,20 +227,61 @@ class VenvCacheType(TypedDict): profile_emu_env: NotRequired[dict[str, str]] +class VenvCacheV1TargetType(TypedDict): + toolchain_bindir: str + gcc_install_dir: NotRequired[str] + + +class VenvCacheV1Type(TypedDict): + profile_common_flags: str + profile_emu_env: NotRequired[dict[str, str]] + qemu_bin: NotRequired[str] + targets: dict[str, VenvCacheV1TargetType] + + class VenvCacheRootType(TypedDict): - cached: VenvCacheType + cached: NotRequired[VenvCacheV0Type] + cached_v1: NotRequired[VenvCacheV1Type] + + +def parse_venv_cache(cache: VenvCacheRootType) -> VenvCacheV1Type: + if "cached_v1" in cache: + return cache["cached_v1"] + if "cached" in cache: + return upgrade_venv_cache_v0(cache["cached"]) + raise RuntimeError("unsupported venv cache version") + + +def upgrade_venv_cache_v0(x: VenvCacheV0Type) -> VenvCacheV1Type: + # v0 only supports one single target so upgrading is trivial + v1_target: VenvCacheV1TargetType = { + "toolchain_bindir": x["toolchain_bindir"], + } + if "gcc_install_dir" in x: + v1_target["gcc_install_dir"] = x["gcc_install_dir"] + + y: VenvCacheV1Type = { + "profile_common_flags": x["profile_common_flags"], + "targets": {x["target_tuple"]: v1_target}, + } + if "profile_emu_env" in x: + y["profile_emu_env"] = x["profile_emu_env"] + if "qemu_bin" in x: + y["qemu_bin"] = x["qemu_bin"] + + return y class RuyiVenvConfig: def __init__(self, cfg: VenvConfigRootType, cache: VenvCacheRootType) -> None: self.profile = cfg["config"]["profile"] self.sysroot = cfg["config"].get("sysroot") - self.target_tuple = cache["cached"]["target_tuple"] - self.toolchain_bindir = cache["cached"]["toolchain_bindir"] - self.gcc_install_dir = cache["cached"].get("gcc_install_dir") - self.profile_common_flags = cache["cached"]["profile_common_flags"] - self.qemu_bin = cache["cached"].get("qemu_bin") - self.profile_emu_env = cache["cached"].get("profile_emu_env") + + parsed_cache = parse_venv_cache(cache) + self.targets = parsed_cache["targets"] + self.profile_common_flags = parsed_cache["profile_common_flags"] + self.qemu_bin = parsed_cache.get("qemu_bin") + self.profile_emu_env = parsed_cache.get("profile_emu_env") @classmethod def explicit_ruyi_venv_root(cls) -> str | None: @@ -280,8 +321,16 @@ def load_from_venv(cls) -> Self | None: with open(venv_config_path, "rb") as fp: cfg: Any = tomllib.load(fp) # in order to cast to our stricter type - venv_cache_path = venv_root / "ruyi-cache.toml" - with open(venv_cache_path, "rb") as fp: - cache: Any = tomllib.load(fp) # in order to cast to our stricter type - + cache: Any # in order to cast to our stricter type + venv_cache_v1_path = venv_root / "ruyi-cache.v1.toml" + try: + with open(venv_cache_v1_path, "rb") as fp: + cache = tomllib.load(fp) + except FileNotFoundError: + venv_cache_v0_path = venv_root / "ruyi-cache.toml" + with open(venv_cache_v0_path, "rb") as fp: + cache = tomllib.load(fp) + + # NOTE: for now it's not prohibited to have v1 cache data in the v0 + # cache path, but this situation is harmless return cls(cfg, cache) diff --git a/ruyi/mux/runtime.py b/ruyi/mux/runtime.py index 37cd9f7..e66d4e6 100644 --- a/ruyi/mux/runtime.py +++ b/ruyi/mux/runtime.py @@ -27,7 +27,32 @@ def mux_main(argv: List[str]) -> int | NoReturn: if basename == "ruyi-qemu": return mux_qemu_main(argv, vcfg) - binpath = os.path.join(vcfg.toolchain_bindir, basename) + # match the basename with one of the configured target tuples + target_tuple: str | None = None + toolchain_bindir: str | None = None + gcc_install_dir: str | None = None + for tgt_tuple, tgt_data in vcfg.targets.items(): + if not basename.startswith(f"{tgt_tuple}-"): + continue + + log.D(f"matched target '{tgt_tuple}', data {tgt_data}") + target_tuple = tgt_tuple + toolchain_bindir = tgt_data["toolchain_bindir"] + gcc_install_dir = tgt_data.get("gcc_install_dir") + break + + if target_tuple is None: + log.F(f"no configured target found for command [yellow]{basename}[/]") + return 1 + + if toolchain_bindir is None: + # should not happen + log.F( + f"internal error: no bindir configured for target [yellow]{target_tuple}[/]" + ) + return 1 + + binpath = os.path.join(toolchain_bindir, basename) log.D(f"binary to exec: {binpath}") @@ -38,11 +63,11 @@ def mux_main(argv: List[str]) -> int | NoReturn: argv_to_insert = [] if is_proxying_to_clang(basename): - log.D(f"adding target for clang: {vcfg.target_tuple}") - argv_to_insert.append(f"--target={vcfg.target_tuple}") - if vcfg.gcc_install_dir is not None: - log.D(f"informing clang of GCC install dir: {vcfg.gcc_install_dir}") - argv_to_insert.append(f"--gcc-install-dir={vcfg.gcc_install_dir}") + log.D(f"adding target for clang: {target_tuple}") + argv_to_insert.append(f"--target={target_tuple}") + if gcc_install_dir is not None: + log.D(f"informing clang of GCC install dir: {gcc_install_dir}") + argv_to_insert.append(f"--gcc-install-dir={gcc_install_dir}") argv_to_insert.extend(shlex.split(vcfg.profile_common_flags)) log.D(f"parsed profile flags: {argv_to_insert}") diff --git a/ruyi/mux/venv/__init__.py b/ruyi/mux/venv/__init__.py index a7d525d..395b8ad 100644 --- a/ruyi/mux/venv/__init__.py +++ b/ruyi/mux/venv/__init__.py @@ -1,234 +1,10 @@ -import argparse from os import PathLike -import pathlib -from typing import Any +from typing import Any, TypedDict -from ... import log -from ...config import GlobalConfig -from ...ruyipkg.atom import Atom -from ...ruyipkg.host import get_native_host -from ...ruyipkg.repo import MetadataRepo -from .provision import render_template_str, VenvMaker - -def cli_venv(args: argparse.Namespace) -> int: - profile_name: str = args.profile - dest = pathlib.Path(args.dest) - with_sysroot: bool = args.with_sysroot - override_name: str | None = args.name - tc_atom_str: str | None = args.toolchain - emu_atom_str: str | None = args.emulator - sysroot_atom_str: str | None = args.sysroot_from - host = get_native_host() - - # TODO: support omitting this if user only has one toolchain installed - # this should come after implementation of local state cache - if tc_atom_str is None: - log.F( - "You have to explicitly specify a toolchain atom for now, e.g. [yellow]`-t gnu-plct`[/yellow]" - ) - return 1 - - config = GlobalConfig.load_from_config() - mr = MetadataRepo(config) - - profile = mr.get_profile(profile_name) - if profile is None: - log.F(f"profile '{profile_name}' not found") - return 1 - - tc_atom = Atom.parse(tc_atom_str) - tc_pm = tc_atom.match_in_repo(mr, config.include_prereleases) - if tc_pm is None: - log.F(f"cannot match a toolchain package with [yellow]{tc_atom_str}[/yellow]") - return 1 - - if tc_pm.toolchain_metadata is None: - log.F(f"the package [yellow]{tc_atom_str}[/yellow] is not a toolchain") - return 1 - - if not tc_pm.toolchain_metadata.satisfies_flavor_set(profile.need_flavor): - log.F( - f"the package [yellow]{tc_atom_str}[/yellow] does not support all necessary features for the profile [yellow]{profile_name}[/yellow]" - ) - log.I( - f"feature(s) needed by profile: {log.humanize_list(profile.need_flavor, item_color='cyan')}" - ) - log.I( - f"feature(s) provided by package: {log.humanize_list(tc_pm.toolchain_metadata.flavors, item_color='yellow')}" - ) - return 1 - - target_tuple = tc_pm.toolchain_metadata.target - - toolchain_root = config.lookup_binary_install_dir( - host, - tc_pm.name_for_installation, - ) - if toolchain_root is None: - log.F("cannot find the installed directory for the toolchain") - return 1 - - gcc_install_dir: PathLike[Any] | None = None - tc_sysroot_dir: PathLike[Any] | None = None - if with_sysroot: - tc_sysroot_relpath = tc_pm.toolchain_metadata.included_sysroot - if tc_sysroot_relpath is not None: - tc_sysroot_dir = pathlib.Path(toolchain_root) / tc_sysroot_relpath - else: - if sysroot_atom_str is None: - log.F( - f"sysroot is requested but the toolchain package does not include one, and [yellow]--sysroot-from[/yellow] is not given" - ) - return 1 - - # try extracting from the sysroot package - # for now only GCC toolchain packages can provide sysroots, so this is - # okay - gcc_pkg_atom = Atom.parse(sysroot_atom_str) - gcc_pkg_pm = gcc_pkg_atom.match_in_repo(mr, config.include_prereleases) - if gcc_pkg_pm is None: - log.F( - f"cannot match a toolchain package with [yellow]{sysroot_atom_str}[/yellow]" - ) - return 1 - - if gcc_pkg_pm.toolchain_metadata is None: - log.F( - f"the package [yellow]{sysroot_atom_str}[/yellow] is not a toolchain" - ) - return 1 - - gcc_pkg_root = config.lookup_binary_install_dir( - host, - gcc_pkg_pm.name_for_installation, - ) - if gcc_pkg_root is None: - log.F("cannot find the installed directory for the sysroot package") - return 1 - - tc_sysroot_relpath = gcc_pkg_pm.toolchain_metadata.included_sysroot - if tc_sysroot_relpath is None: - log.F( - f"sysroot is requested but the package [yellow]{sysroot_atom_str}[/yellow] does not contain one" - ) - return 1 - - tc_sysroot_dir = pathlib.Path(gcc_pkg_root) / tc_sysroot_relpath - - # also figure the GCC include/libs path out for Clang to be able to - # locate them - gcc_install_dir = find_gcc_install_dir( - gcc_pkg_root, - # we should use the GCC-providing package's target tuple as that's - # not guaranteed to be the same as llvm's - gcc_pkg_pm.toolchain_metadata.target, - ) - - # for now, require this directory to be present (or clang would barely work) - if gcc_install_dir is None: - log.F( - "cannot find a GCC include & lib directory in the sysroot package" - ) - return 1 - - target_arch = tc_pm.toolchain_metadata.target_arch - - # Now handle the emulator. - emu_progs = None - emu_root: PathLike[Any] | None = None - if emu_atom_str: - emu_atom = Atom.parse(emu_atom_str) - emu_pm = emu_atom.match_in_repo(mr, config.include_prereleases) - if emu_pm is None: - log.F( - f"cannot match an emulator package with [yellow]{emu_atom_str}[/yellow]" - ) - return 1 - - if emu_pm.emulator_metadata is None: - log.F(f"the package [yellow]{emu_atom_str}[/yellow] is not an emulator") - return 1 - - emu_progs = list(emu_pm.emulator_metadata.list_for_arch(target_arch)) - if not emu_progs: - log.F( - f"the emulator package [yellow]{emu_atom_str}[/yellow] does not support the target architecture [yellow]{target_arch}[/yellow]" - ) - return 1 - - for prog in emu_progs: - if not profile.check_emulator_flavor( - prog.flavor, emu_pm.emulator_metadata.flavors - ): - log.F( - f"the package [yellow]{emu_atom_str}[/yellow] does not support all necessary features for the profile [yellow]{profile_name}[/yellow]" - ) - log.I( - f"feature(s) needed by profile: {log.humanize_list(profile.get_needed_emulator_pkg_flavors(prog.flavor), item_color='cyan')}" - ) - log.I( - f"feature(s) provided by package: {log.humanize_list(emu_pm.emulator_metadata.flavors or [], item_color='yellow')}" - ) - return 1 - - emu_root = config.lookup_binary_install_dir( - host, - emu_pm.name_for_installation, - ) - if emu_root is None: - log.F("cannot find the installed directory for the emulator") - return 1 - - if override_name is not None: - log.I( - f"Creating a Ruyi virtual environment [cyan]'{override_name}'[/cyan] at [green]{dest}[/green]..." - ) - else: - log.I(f"Creating a Ruyi virtual environment at [green]{dest}[/green]...") - - maker = VenvMaker( - profile, - toolchain_root, - target_tuple, - # assume clang is preferred if package contains clang - # this is mostly true given most packages don't contain both - "clang" if tc_pm.toolchain_metadata.has_clang else "gcc", - # same for binutils provider flavor - "llvm" if tc_pm.toolchain_metadata.has_llvm else "binutils", - dest.resolve(), - tc_sysroot_dir, - gcc_install_dir, - emu_progs, - emu_root, - override_name, - ) - maker.provision() - - log.I( - render_template_str( - "prompt.venv-created.txt", - { - "sysroot": maker.sysroot_destdir, - }, - ) - ) - - return 0 - - -def find_gcc_install_dir( - install_root: PathLike[Any], - target_tuple: str, -) -> PathLike[Any] | None: - # check $PREFIX/lib/gcc/$TARGET/* - search_root = pathlib.Path(install_root) / "lib" / "gcc" / target_tuple - try: - for p in search_root.iterdir(): - # only want the first one (should be the only one) - return p - except FileNotFoundError: - pass - - # nothing? - return None +class ConfiguredTargetTuple(TypedDict): + target: str + toolchain_root: PathLike[Any] + binutils_flavor: str + cc_flavor: str + gcc_install_dir: PathLike[Any] | None diff --git a/ruyi/mux/venv/data.py b/ruyi/mux/venv/data.py index a4aed81..7676861 100644 --- a/ruyi/mux/venv/data.py +++ b/ruyi/mux/venv/data.py @@ -8,5 +8,5 @@ "toolchain.cmake": b"eNqFkVFrwjAUhd/7Ky6IMGF274M9dG2cZWlT0joUhJDVaIuajKQTR+l/X6zOWccwb+F+5+Scmx5MjIBNuRaPTs/pQb7lawGu68Lc3g5nGPiR94pYRgj2x14Ys1GI0VNdw07IHdNKVdA0D5VSm7zgpXSPFnOn7g+hXIL5Mi3Tb64MwzjNPIxZQtEonB4Mf9CmOcmFXFiHs9LGchwjqrujQTpLMxSx2IsQ4FJ+7gd/hwklPkpTQsH6f2iVC2OUti9csj7zSZTYVi2V59fj6bQL7PctUfcv+w1tzF/NKIwDRgnJWOJlY+iUa5UX1YzgOi9gaXMVgi+ENsDlwn7Ku+a6FAZKCVUhoOJ6JSqr3JVaya2Q1X1XbfutNN+eBYUyHfzffCwiATrs6oV6EcToDdHBDRiHz9SjMyAxnt1iw9jHkwCd2G+4PsQk", # fmt: skip "meson-cross.ini": b"eNptkMFqwzAQRO/6ioUQ0tJazaU9FPoLvfUUglDkNV4iS0IrB4eQf69k2ZRCbmI083Z3NvDDCJbO+Ck2YgMDsnfAmMYATWOiZ246sgi3G1zQXVT0PsH9/jYbq0GSI5BSzoBvxJYXzF6+f8j9ixTicCKnIyEfhYEv2GWaMRmzEyaEVZimWbltG+h8hDNeXyHo1AO5ClS6bSmRd9qqFSgp4cBPz7C9iwzJoQxZiHN4RaJrC7XYtkAd8JXnW7JwOI1kU5PH+FDwecsQsaNp4azWiiqknC+5EH3AmOpdgz6jSt5b02tyam5tuezR17rX/1Xys1b8aHLzN1oces9JDdr05PBYcrmHkrLkxqn0OqpOD2SvaxnRG2TOFdTax4d6GaBd5aRkcSd+ATx1t64=", # fmt: skip "binfmt.conf": b"eNpNUrlu20AQ7f0VAwgCbECk+wQp06Zwl0oakUNyoD2UvQRC1r/7LUkhrkjuzjuHOzqrG2w6Wo0ddd4NOlLMmvhshAYfKE0a6SPPSkVDymxIHN68s+LS4WVH6ijOMYntm5XrSQO05fSTogidLLtNqu1PC3EvidXEFhR/faY4+Wx6CjIqyAJ5J+QHyFcbxvibuhGYznDgpN69xrdD1Wa68QwOvl6DvwbltPqefYYGuIKecwXQO5yEop0QzPAIjXeS1B0I0+qKvwhoqp7YbDjhFIY5zAtVFDOgAV4GTgF9NP8wd6JbgLKE9uW+X3ThYqy+gkRvivTHehBp/wD57yfx/b7MtfB3NTwfHVuhxwMcDemw3qHmBQXcH49QaeJUrRCb6GnigiNfR4v29VaifF8NFUYZ2GIk4EJ2Sa1s+/KhR3qguZtUQNT5EKRLW/JaVhSUlLSLP+Bgi3Y5UKnwp71WsfX4+gabO2pqqAtC/MKz0CcWuiaCqb6i92s8fCFhU7+2Er79gUes69nDf9gX9HfteA==", # fmt: skip - "ruyi-cache.toml": b"eNptULtqwzAU3fUVtymBFoI/oJAtGbo0i7cShCJL9iV6pLJsCMb/nits5yE6WT5v6R1+DuX+C8oGW9BoFNDXCidqVcHpCqG7YgG7Q5LBfvddvjH2K4VsVHVkUYRaRR67C/m2sBoGeIHGccWi90Y2Ah0/oaswLLocTtphDaihlpKja6MwhidmPebIFJGjlEABitI0edgl+HQfLr213nFtRN3Ozn+px4A/Zbs0i1Lux8l4/826Jt8Sm0TK9YmYn6rIqGNyaB/gvIEeKC/jC4zKth+fFEGtZ6qbB/TLTKpOfuKfZtwAGdyfcg==", # fmt: skip + "ruyi-cache.toml": b"eNptkM9rgzAUx+/5K94c0g2GsOtgt/bQy3rxJiGkMWqoJluSBor4v++FqnVhJ2M+3x/v5Rm+TuXhA8pOOWhULwG/A9e8lTWcb2CvN1XA/hRlcNgfyydCKsFFJ2sW3in5tia6mDDDYDRret46+IRsHOFfNE0ZGXNQDfzI4crOSkM+rce7cf1FMWqlrlGeT7NviY0iqUMEj4GKhNJoaoyFyxsEwMiEF8rLwb28YgoWX7BxniEsk2J79CP/O8k21HPbSu82YZuJFjjnU+KN6UXHlY471soujdUuJTu6ea9QZa0QTGnned8zxBnFpuTuEZaAe1ayxLoc+QX4trGw", # fmt: skip } diff --git a/ruyi/mux/venv/provision.py b/ruyi/mux/venv/provision.py index c218cf3..020052f 100644 --- a/ruyi/mux/venv/provision.py +++ b/ruyi/mux/venv/provision.py @@ -14,6 +14,7 @@ from ... import log, self_exe from ...ruyipkg.pkg_manifest import EmulatorProgDecl from ...ruyipkg.profile import ProfileProxy +from . import ConfiguredTargetTuple from .data import TEMPLATES from .emulator_cfg import ResolvedEmulatorProg @@ -63,40 +64,37 @@ class VenvMaker: def __init__( self, profile: ProfileProxy, - toolchain_install_root: PathLike[Any], - target_tuple: str, - toolchain_flavor: str, - binutils_flavor: str, + targets: list[ConfiguredTargetTuple], dest: PathLike[Any], sysroot_srcdir: PathLike[Any] | None, - gcc_install_dir: PathLike[Any] | None, emulator_progs: list[EmulatorProgDecl] | None, emulator_root: PathLike[Any] | None, override_name: str | None = None, ) -> None: self.profile = profile - self.toolchain_install_root = toolchain_install_root - self.target_tuple = target_tuple - self.binutils_flavor = binutils_flavor - self.toolchain_flavor = toolchain_flavor - self.dest = dest + self.targets = targets + self.venv_root = pathlib.Path(dest) self.sysroot_srcdir = sysroot_srcdir - self.gcc_install_dir = gcc_install_dir self.emulator_progs = emulator_progs self.emulator_root = emulator_root self.override_name = override_name + self.bindir = self.venv_root / "bin" self.sysroot_destdir = ( - pathlib.Path(self.dest) / "sysroot" + self.venv_root / "sysroot" if self.sysroot_srcdir is not None else None ) def provision(self) -> None: - venv_root = pathlib.Path(self.dest) + venv_root = self.venv_root + bindir = self.bindir + venv_root.mkdir() + bindir.mkdir() if self.sysroot_srcdir is not None: + log.D("copying sysroot") assert self.sysroot_destdir is not None shutil.copytree( self.sysroot_srcdir, @@ -111,48 +109,115 @@ def provision(self) -> None: } render_and_write(venv_root / "ruyi-venv.toml", "ruyi-venv.toml", env_data) - bindir = venv_root / "bin" - bindir.mkdir() - - log.D("symlinking binaries into venv") - toolchain_bindir = pathlib.Path(self.toolchain_install_root) / "bin" - symlink_binaries(toolchain_bindir, bindir) - - make_llvm_tool_aliases( - bindir, - self.target_tuple, - self.binutils_flavor == "llvm", - self.toolchain_flavor == "clang", - ) + for i, tgt in enumerate(self.targets): + is_primary = i == 0 + self.provision_target(tgt, is_primary) template_data = { - "RUYI_VENV": str(self.dest), + "RUYI_VENV": str(venv_root), "RUYI_VENV_NAME": self.override_name, } render_and_write(bindir / "ruyi-activate", "ruyi-activate.bash", template_data) + qemu_bin: PathLike[Any] | None = None + profile_emu_env: dict[str, str] | None = None + if self.emulator_root is not None and self.emulator_progs: + resolved_emu_progs = [ + ResolvedEmulatorProg.new( + p, + self.emulator_root, + self.profile, + self.sysroot_destdir, + ) + for p in self.emulator_progs + ] + binfmt_data = { + "resolved_progs": resolved_emu_progs, + } + render_and_write( + venv_root / "binfmt.conf", + "binfmt.conf", + binfmt_data, + ) + + for i, p in enumerate(self.emulator_progs): + if not p.is_qemu: + continue + + qemu_bin = pathlib.Path(self.emulator_root) / p.relative_path + profile_emu_env = resolved_emu_progs[i].env + + log.D("symlinking the ruyi-qemu wrapper") + os.symlink(self_exe(), bindir / "ruyi-qemu") + + # provide initial cached configuration to venv + render_and_write( + venv_root / "ruyi-cache.v1.toml", + "ruyi-cache.toml", + self.make_venv_cache_data(qemu_bin, profile_emu_env), + ) + + def make_venv_cache_data( + self, + qemu_bin: PathLike[Any] | None, + profile_emu_env: dict[str, str] | None, + ) -> dict[str, object]: + targets_cache_data: dict[str, object] = { + tgt["target"]: { + "toolchain_bindir": str(pathlib.Path(tgt["toolchain_root"]) / "bin"), + "gcc_install_dir": tgt["gcc_install_dir"], + } for tgt in self.targets + } + + return { + "profile_common_flags": self.profile.get_common_flags(), + "profile_emu_env": profile_emu_env, + "qemu_bin": qemu_bin, + "targets": targets_cache_data, + } + + def provision_target( + self, + tgt: ConfiguredTargetTuple, + is_primary: bool, + ) -> None: + venv_root = self.venv_root + bindir = self.bindir + target_tuple = tgt["target"] + + log.D(f"symlinking {target_tuple} binaries into venv") + toolchain_bindir = pathlib.Path(tgt["toolchain_root"]) / "bin" + symlink_binaries(toolchain_bindir, bindir) + + make_llvm_tool_aliases( + bindir, + target_tuple, + tgt["binutils_flavor"] == "llvm", + tgt["cc_flavor"] == "clang", + ) + # CMake toolchain file & Meson cross file - if self.toolchain_flavor == "clang": + if tgt["cc_flavor"] == "clang": cc_path = bindir / "clang" cxx_path = bindir / "clang++" - elif self.toolchain_flavor == "gcc": - cc_path = bindir / f"{self.target_tuple}-gcc" - cxx_path = bindir / f"{self.target_tuple}-g++" + elif tgt["cc_flavor"] == "gcc": + cc_path = bindir / f"{target_tuple}-gcc" + cxx_path = bindir / f"{target_tuple}-g++" else: raise NotImplementedError - if self.binutils_flavor == "binutils": + if tgt["binutils_flavor"] == "binutils": meson_additional_binaries = { - "ar": bindir / f"{self.target_tuple}-ar", - "nm": bindir / f"{self.target_tuple}-nm", - "objcopy": bindir / f"{self.target_tuple}-objcopy", - "objdump": bindir / f"{self.target_tuple}-objdump", - "ranlib": bindir / f"{self.target_tuple}-ranlib", - "readelf": bindir / f"{self.target_tuple}-readelf", - "strip": bindir / f"{self.target_tuple}-strip", + "ar": bindir / f"{target_tuple}-ar", + "nm": bindir / f"{target_tuple}-nm", + "objcopy": bindir / f"{target_tuple}-objcopy", + "objdump": bindir / f"{target_tuple}-objdump", + "ranlib": bindir / f"{target_tuple}-ranlib", + "readelf": bindir / f"{target_tuple}-readelf", + "strip": bindir / f"{target_tuple}-strip", } - elif self.binutils_flavor == "llvm": + elif tgt["binutils_flavor"] == "llvm": meson_additional_binaries = { "ar": bindir / "llvm-ar", "nm": bindir / "llvm-nm", @@ -165,7 +230,7 @@ def provision(self) -> None: else: raise NotImplementedError - cmake_toolchain_file_path = venv_root / "toolchain.cmake" + cmake_toolchain_file_path = venv_root / f"toolchain.{target_tuple}.cmake" toolchain_file_data = { "cc": cc_path, "cxx": cxx_path, @@ -181,57 +246,19 @@ def provision(self) -> None: toolchain_file_data, ) + meson_cross_file_path = venv_root / f"meson-cross.{target_tuple}.ini" render_and_write( - venv_root / "meson-cross.ini", + meson_cross_file_path, "meson-cross.ini", toolchain_file_data, ) - qemu_bin: PathLike[Any] | None = None - profile_emu_env: dict[str, str] | None = None - if self.emulator_root is not None and self.emulator_progs: - resolved_emu_progs = [ - ResolvedEmulatorProg.new( - p, - self.emulator_root, - self.profile, - self.sysroot_destdir, - ) - for p in self.emulator_progs - ] - binfmt_data = { - "resolved_progs": resolved_emu_progs, - } - render_and_write( - venv_root / "binfmt.conf", - "binfmt.conf", - binfmt_data, - ) - - for i, p in enumerate(self.emulator_progs): - if not p.is_qemu: - continue - - qemu_bin = pathlib.Path(self.emulator_root) / p.relative_path - profile_emu_env = resolved_emu_progs[i].env - - log.D("symlinking the ruyi-qemu wrapper") - os.symlink(self_exe(), bindir / "ruyi-qemu") - - # provide initial cached configuration to venv - initial_cache_data = { - "target_tuple": self.target_tuple, - "toolchain_bindir": str(toolchain_bindir), - "gcc_install_dir": self.gcc_install_dir, - "profile_common_flags": self.profile.get_common_flags(), - "qemu_bin": qemu_bin, - "profile_emu_env": profile_emu_env, - } - render_and_write( - venv_root / "ruyi-cache.toml", - "ruyi-cache.toml", - initial_cache_data, - ) + if is_primary: + log.D(f"making cmake & meson file symlinks to primary target {target_tuple}") + primary_cmake_toolchain_file_path = venv_root / "toolchain.cmake" + primary_meson_cross_file_path = venv_root / "meson-cross.ini" + os.symlink(cmake_toolchain_file_path.name, primary_cmake_toolchain_file_path) + os.symlink(meson_cross_file_path.name, primary_meson_cross_file_path) def symlink_binaries(src_bindir: PathLike[Any], dest_bindir: PathLike[Any]) -> None: diff --git a/ruyi/mux/venv/ruyi-cache.toml.jinja b/ruyi/mux/venv/ruyi-cache.toml.jinja index 2f36258..91741e7 100644 --- a/ruyi/mux/venv/ruyi-cache.toml.jinja +++ b/ruyi/mux/venv/ruyi-cache.toml.jinja @@ -1,12 +1,13 @@ # NOTE: This file is managed by ruyi. DO NOT EDIT! -[cached] -target_tuple = "{{ target_tuple }}" -toolchain_bindir = "{{ toolchain_bindir }}" -{% if gcc_install_dir %}gcc_install_dir = "{{ gcc_install_dir }}"{% endif %} +[cached_v1] profile_common_flags = "{{ profile_common_flags }}" {% if qemu_bin %}qemu_bin = "{{ qemu_bin }}"{% endif %} {% if profile_emu_env %} -[cached.profile_emu_env] +[cached_v1.profile_emu_env] {% for k, v in profile_emu_env.items() %}{{ k }} = "{{ v }}" {% endfor %}{% endif %} +{% for k, v in targets.items() %}[cached_v1.targets.{{ k }}] +toolchain_bindir = "{{ v['toolchain_bindir'] }}" +{% if v["gcc_install_dir"] %}gcc_install_dir = "{{ v['gcc_install_dir'] }}"{% endif %} +{% endfor %} diff --git a/ruyi/mux/venv/venv_cli.py b/ruyi/mux/venv/venv_cli.py new file mode 100644 index 0000000..5c4d47a --- /dev/null +++ b/ruyi/mux/venv/venv_cli.py @@ -0,0 +1,275 @@ +import argparse +from os import PathLike +import pathlib +from typing import Any + +from ... import log +from ...config import GlobalConfig +from ...ruyipkg.atom import Atom +from ...ruyipkg.host import get_native_host +from ...ruyipkg.repo import MetadataRepo +from . import ConfiguredTargetTuple +from .provision import render_template_str, VenvMaker + + +def cli_venv(args: argparse.Namespace) -> int: + profile_name: str = args.profile + dest = pathlib.Path(args.dest) + with_sysroot: bool = args.with_sysroot + override_name: str | None = args.name + tc_atoms_str: list[str] | None = args.toolchain + emu_atom_str: str | None = args.emulator + sysroot_atom_str: str | None = args.sysroot_from + host = get_native_host() + + # TODO: support omitting this if user only has one toolchain installed + # this should come after implementation of local state cache + if tc_atoms_str is None: + log.F( + "You have to specify at least one toolchain atom for now, e.g. [yellow]`-t gnu-plct`[/]" + ) + return 1 + + config = GlobalConfig.load_from_config() + mr = MetadataRepo(config) + + profile = mr.get_profile(profile_name) + if profile is None: + log.F(f"profile '{profile_name}' not found") + return 1 + + target_arch = '' + seen_target_tuples: set[str] = set() + targets: list[ConfiguredTargetTuple] = [] + sysroot_dir: PathLike[Any] | None = None + warn_differing_target_arch = False + warn_multiple_sysroots = False + + for tc_atom_str in tc_atoms_str: + tc_atom = Atom.parse(tc_atom_str) + tc_pm = tc_atom.match_in_repo(mr, config.include_prereleases) + if tc_pm is None: + log.F(f"cannot match a toolchain package with [yellow]{tc_atom_str}[/yellow]") + return 1 + + if tc_pm.toolchain_metadata is None: + log.F(f"the package [yellow]{tc_atom_str}[/yellow] is not a toolchain") + return 1 + + if not tc_pm.toolchain_metadata.satisfies_flavor_set(profile.need_flavor): + log.F( + f"the package [yellow]{tc_atom_str}[/yellow] does not support all necessary features for the profile [yellow]{profile_name}[/yellow]" + ) + log.I( + f"feature(s) needed by profile: {log.humanize_list(profile.need_flavor, item_color='cyan')}" + ) + log.I( + f"feature(s) provided by package: {log.humanize_list(tc_pm.toolchain_metadata.flavors, item_color='yellow')}" + ) + return 1 + + target_tuple = tc_pm.toolchain_metadata.target + if target_tuple in seen_target_tuples: + log.F(f"the target tuple [yellow]{target_tuple}[/] is already covered by one of the requested toolchains") + log.I("for now, only toolchains with differing target tuples can co-exist in one virtual environment") + return 1 + + toolchain_root = config.lookup_binary_install_dir( + host, + tc_pm.name_for_installation, + ) + if toolchain_root is None: + log.F("cannot find the installed directory for the toolchain") + return 1 + + tc_sysroot_dir: PathLike[Any] + gcc_install_dir: PathLike[Any] | None = None + if with_sysroot: + if tc_sysroot_relpath := tc_pm.toolchain_metadata.included_sysroot: + tc_sysroot_dir = pathlib.Path(toolchain_root) / tc_sysroot_relpath + else: + if sysroot_atom_str is None: + log.F( + f"sysroot is requested but the toolchain package does not include one, and [yellow]--sysroot-from[/yellow] is not given" + ) + return 1 + + # try extracting from the sysroot package + # for now only GCC toolchain packages can provide sysroots, so this is + # okay + gcc_pkg_atom = Atom.parse(sysroot_atom_str) + gcc_pkg_pm = gcc_pkg_atom.match_in_repo(mr, config.include_prereleases) + if gcc_pkg_pm is None: + log.F( + f"cannot match a toolchain package with [yellow]{sysroot_atom_str}[/yellow]" + ) + return 1 + + if gcc_pkg_pm.toolchain_metadata is None: + log.F( + f"the package [yellow]{sysroot_atom_str}[/yellow] is not a toolchain" + ) + return 1 + + gcc_pkg_root = config.lookup_binary_install_dir( + host, + gcc_pkg_pm.name_for_installation, + ) + if gcc_pkg_root is None: + log.F("cannot find the installed directory for the sysroot package") + return 1 + + tc_sysroot_relpath = gcc_pkg_pm.toolchain_metadata.included_sysroot + if tc_sysroot_relpath is None: + log.F( + f"sysroot is requested but the package [yellow]{sysroot_atom_str}[/yellow] does not contain one" + ) + return 1 + + tc_sysroot_dir = pathlib.Path(gcc_pkg_root) / tc_sysroot_relpath + + # also figure the GCC include/libs path out for Clang to be able to + # locate them + gcc_install_dir = find_gcc_install_dir( + gcc_pkg_root, + # we should use the GCC-providing package's target tuple as that's + # not guaranteed to be the same as llvm's + gcc_pkg_pm.toolchain_metadata.target, + ) + + # for now, require this directory to be present (or clang would barely work) + if gcc_install_dir is None: + log.F( + "cannot find a GCC include & lib directory in the sysroot package" + ) + return 1 + + if sysroot_dir is None: + sysroot_dir = tc_sysroot_dir + else: + # check if multiple toolchain packages offer competing sysroots + if tc_sysroot_dir != sysroot_dir: + # first one wins + warn_multiple_sysroots = True + + # record the target tuple info to configure in the venv + configured_target: ConfiguredTargetTuple = { + "target": target_tuple, + "toolchain_root": toolchain_root, + # assume clang is preferred if package contains clang + # this is mostly true given most packages don't contain both + "cc_flavor": "clang" if tc_pm.toolchain_metadata.has_clang else "gcc", + # same for binutils provider flavor + "binutils_flavor": "llvm" if tc_pm.toolchain_metadata.has_llvm else "binutils", + "gcc_install_dir": gcc_install_dir, + } + log.D(f"configuration for {target_tuple}: {configured_target}") + targets.append(configured_target) + seen_target_tuples.add(target_tuple) + + # record the target architecture for use in emulator package matching + if not target_arch: + target_arch = tc_pm.toolchain_metadata.target_arch + elif target_arch != tc_pm.toolchain_metadata.target_arch: + # first one wins + warn_differing_target_arch = True + + if warn_differing_target_arch: + log.W("multiple toolchains specified with differing target architecture") + log.I(f"using the target architecture of the first toolchain: [yellow]{target_arch}[/]") + + if warn_multiple_sysroots: + log.W("selected packages for the virtual environment provide multiple competing sysroots") + log.I("using the sysroot from the first sysroot-providing package: [yellow]{sysroot_dir}[/]") + + # Now handle the emulator. + emu_progs = None + emu_root: PathLike[Any] | None = None + if emu_atom_str: + emu_atom = Atom.parse(emu_atom_str) + emu_pm = emu_atom.match_in_repo(mr, config.include_prereleases) + if emu_pm is None: + log.F( + f"cannot match an emulator package with [yellow]{emu_atom_str}[/yellow]" + ) + return 1 + + if emu_pm.emulator_metadata is None: + log.F(f"the package [yellow]{emu_atom_str}[/yellow] is not an emulator") + return 1 + + emu_progs = list(emu_pm.emulator_metadata.list_for_arch(target_arch)) + if not emu_progs: + log.F( + f"the emulator package [yellow]{emu_atom_str}[/yellow] does not support the target architecture [yellow]{target_arch}[/yellow]" + ) + return 1 + + for prog in emu_progs: + if not profile.check_emulator_flavor( + prog.flavor, emu_pm.emulator_metadata.flavors, + ): + log.F( + f"the package [yellow]{emu_atom_str}[/yellow] does not support all necessary features for the profile [yellow]{profile_name}[/yellow]" + ) + log.I( + f"feature(s) needed by profile: {log.humanize_list(profile.get_needed_emulator_pkg_flavors(prog.flavor), item_color='cyan')}" + ) + log.I( + f"feature(s) provided by package: {log.humanize_list(emu_pm.emulator_metadata.flavors or [], item_color='yellow')}" + ) + return 1 + + emu_root = config.lookup_binary_install_dir( + host, + emu_pm.name_for_installation, + ) + if emu_root is None: + log.F("cannot find the installed directory for the emulator") + return 1 + + if override_name is not None: + log.I( + f"Creating a Ruyi virtual environment [cyan]'{override_name}'[/cyan] at [green]{dest}[/green]..." + ) + else: + log.I(f"Creating a Ruyi virtual environment at [green]{dest}[/green]...") + + maker = VenvMaker( + profile, + targets, + dest.resolve(), + sysroot_dir, + emu_progs, + emu_root, + override_name, + ) + maker.provision() + + log.I( + render_template_str( + "prompt.venv-created.txt", + { + "sysroot": maker.sysroot_destdir, + }, + ) + ) + + return 0 + + +def find_gcc_install_dir( + install_root: PathLike[Any], + target_tuple: str, +) -> PathLike[Any] | None: + # check $PREFIX/lib/gcc/$TARGET/* + search_root = pathlib.Path(install_root) / "lib" / "gcc" / target_tuple + try: + for p in search_root.iterdir(): + # only want the first one (should be the only one) + return p + except FileNotFoundError: + pass + + # nothing? + return None From 5398dab5f3011efa42f1f961e39b7e7956235774 Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sun, 29 Sep 2024 17:19:14 +0800 Subject: [PATCH 2/2] feat(venv): configure and provision sysroots separately for each target --- ruyi/config/__init__.py | 17 +++++-- ruyi/mux/runtime.py | 8 ++-- ruyi/mux/venv/__init__.py | 1 + ruyi/mux/venv/data.py | 2 +- ruyi/mux/venv/provision.py | 74 +++++++++++++++++++++-------- ruyi/mux/venv/ruyi-cache.toml.jinja | 1 + ruyi/mux/venv/venv_cli.py | 19 ++------ 7 files changed, 77 insertions(+), 45 deletions(-) diff --git a/ruyi/config/__init__.py b/ruyi/config/__init__.py index 72cee19..adcc7e2 100644 --- a/ruyi/config/__init__.py +++ b/ruyi/config/__init__.py @@ -229,6 +229,7 @@ class VenvCacheV0Type(TypedDict): class VenvCacheV1TargetType(TypedDict): toolchain_bindir: str + toolchain_sysroot: NotRequired[str] gcc_install_dir: NotRequired[str] @@ -244,21 +245,29 @@ class VenvCacheRootType(TypedDict): cached_v1: NotRequired[VenvCacheV1Type] -def parse_venv_cache(cache: VenvCacheRootType) -> VenvCacheV1Type: +def parse_venv_cache( + cache: VenvCacheRootType, + global_sysroot: str | None, +) -> VenvCacheV1Type: if "cached_v1" in cache: return cache["cached_v1"] if "cached" in cache: - return upgrade_venv_cache_v0(cache["cached"]) + return upgrade_venv_cache_v0(cache["cached"], global_sysroot) raise RuntimeError("unsupported venv cache version") -def upgrade_venv_cache_v0(x: VenvCacheV0Type) -> VenvCacheV1Type: +def upgrade_venv_cache_v0( + x: VenvCacheV0Type, + global_sysroot: str | None, +) -> VenvCacheV1Type: # v0 only supports one single target so upgrading is trivial v1_target: VenvCacheV1TargetType = { "toolchain_bindir": x["toolchain_bindir"], } if "gcc_install_dir" in x: v1_target["gcc_install_dir"] = x["gcc_install_dir"] + if global_sysroot is not None: + v1_target["toolchain_sysroot"] = global_sysroot y: VenvCacheV1Type = { "profile_common_flags": x["profile_common_flags"], @@ -277,7 +286,7 @@ def __init__(self, cfg: VenvConfigRootType, cache: VenvCacheRootType) -> None: self.profile = cfg["config"]["profile"] self.sysroot = cfg["config"].get("sysroot") - parsed_cache = parse_venv_cache(cache) + parsed_cache = parse_venv_cache(cache, self.sysroot) self.targets = parsed_cache["targets"] self.profile_common_flags = parsed_cache["profile_common_flags"] self.qemu_bin = parsed_cache.get("qemu_bin") diff --git a/ruyi/mux/runtime.py b/ruyi/mux/runtime.py index e66d4e6..c7913b7 100644 --- a/ruyi/mux/runtime.py +++ b/ruyi/mux/runtime.py @@ -30,6 +30,7 @@ def mux_main(argv: List[str]) -> int | NoReturn: # match the basename with one of the configured target tuples target_tuple: str | None = None toolchain_bindir: str | None = None + toolchain_sysroot: str | None = None gcc_install_dir: str | None = None for tgt_tuple, tgt_data in vcfg.targets.items(): if not basename.startswith(f"{tgt_tuple}-"): @@ -38,6 +39,7 @@ def mux_main(argv: List[str]) -> int | NoReturn: log.D(f"matched target '{tgt_tuple}', data {tgt_data}") target_tuple = tgt_tuple toolchain_bindir = tgt_data["toolchain_bindir"] + toolchain_sysroot = tgt_data.get("toolchain_sysroot") gcc_install_dir = tgt_data.get("gcc_install_dir") break @@ -72,9 +74,9 @@ def mux_main(argv: List[str]) -> int | NoReturn: argv_to_insert.extend(shlex.split(vcfg.profile_common_flags)) log.D(f"parsed profile flags: {argv_to_insert}") - if vcfg.sysroot is not None: - log.D(f"adding sysroot: {vcfg.sysroot}") - argv_to_insert.extend(("--sysroot", vcfg.sysroot)) + if toolchain_sysroot is not None: + log.D(f"adding sysroot: {toolchain_sysroot}") + argv_to_insert.extend(("--sysroot", toolchain_sysroot)) new_argv = [binpath] if argv_to_insert: diff --git a/ruyi/mux/venv/__init__.py b/ruyi/mux/venv/__init__.py index 395b8ad..84a95c6 100644 --- a/ruyi/mux/venv/__init__.py +++ b/ruyi/mux/venv/__init__.py @@ -5,6 +5,7 @@ class ConfiguredTargetTuple(TypedDict): target: str toolchain_root: PathLike[Any] + toolchain_sysroot: PathLike[Any] | None binutils_flavor: str cc_flavor: str gcc_install_dir: PathLike[Any] | None diff --git a/ruyi/mux/venv/data.py b/ruyi/mux/venv/data.py index 7676861..8293531 100644 --- a/ruyi/mux/venv/data.py +++ b/ruyi/mux/venv/data.py @@ -8,5 +8,5 @@ "toolchain.cmake": b"eNqFkVFrwjAUhd/7Ky6IMGF274M9dG2cZWlT0joUhJDVaIuajKQTR+l/X6zOWccwb+F+5+Scmx5MjIBNuRaPTs/pQb7lawGu68Lc3g5nGPiR94pYRgj2x14Ys1GI0VNdw07IHdNKVdA0D5VSm7zgpXSPFnOn7g+hXIL5Mi3Tb64MwzjNPIxZQtEonB4Mf9CmOcmFXFiHs9LGchwjqrujQTpLMxSx2IsQ4FJ+7gd/hwklPkpTQsH6f2iVC2OUti9csj7zSZTYVi2V59fj6bQL7PctUfcv+w1tzF/NKIwDRgnJWOJlY+iUa5UX1YzgOi9gaXMVgi+ENsDlwn7Ku+a6FAZKCVUhoOJ6JSqr3JVaya2Q1X1XbfutNN+eBYUyHfzffCwiATrs6oV6EcToDdHBDRiHz9SjMyAxnt1iw9jHkwCd2G+4PsQk", # fmt: skip "meson-cross.ini": b"eNptkMFqwzAQRO/6ioUQ0tJazaU9FPoLvfUUglDkNV4iS0IrB4eQf69k2ZRCbmI083Z3NvDDCJbO+Ck2YgMDsnfAmMYATWOiZ246sgi3G1zQXVT0PsH9/jYbq0GSI5BSzoBvxJYXzF6+f8j9ixTicCKnIyEfhYEv2GWaMRmzEyaEVZimWbltG+h8hDNeXyHo1AO5ClS6bSmRd9qqFSgp4cBPz7C9iwzJoQxZiHN4RaJrC7XYtkAd8JXnW7JwOI1kU5PH+FDwecsQsaNp4azWiiqknC+5EH3AmOpdgz6jSt5b02tyam5tuezR17rX/1Xys1b8aHLzN1oces9JDdr05PBYcrmHkrLkxqn0OqpOD2SvaxnRG2TOFdTax4d6GaBd5aRkcSd+ATx1t64=", # fmt: skip "binfmt.conf": b"eNpNUrlu20AQ7f0VAwgCbECk+wQp06Zwl0oakUNyoD2UvQRC1r/7LUkhrkjuzjuHOzqrG2w6Wo0ddd4NOlLMmvhshAYfKE0a6SPPSkVDymxIHN68s+LS4WVH6ijOMYntm5XrSQO05fSTogidLLtNqu1PC3EvidXEFhR/faY4+Wx6CjIqyAJ5J+QHyFcbxvibuhGYznDgpN69xrdD1Wa68QwOvl6DvwbltPqefYYGuIKecwXQO5yEop0QzPAIjXeS1B0I0+qKvwhoqp7YbDjhFIY5zAtVFDOgAV4GTgF9NP8wd6JbgLKE9uW+X3ThYqy+gkRvivTHehBp/wD57yfx/b7MtfB3NTwfHVuhxwMcDemw3qHmBQXcH49QaeJUrRCb6GnigiNfR4v29VaifF8NFUYZ2GIk4EJ2Sa1s+/KhR3qguZtUQNT5EKRLW/JaVhSUlLSLP+Bgi3Y5UKnwp71WsfX4+gabO2pqqAtC/MKz0CcWuiaCqb6i92s8fCFhU7+2Er79gUes69nDf9gX9HfteA==", # fmt: skip - "ruyi-cache.toml": b"eNptkM9rgzAUx+/5K94c0g2GsOtgt/bQy3rxJiGkMWqoJluSBor4v++FqnVhJ2M+3x/v5Rm+TuXhA8pOOWhULwG/A9e8lTWcb2CvN1XA/hRlcNgfyydCKsFFJ2sW3in5tia6mDDDYDRret46+IRsHOFfNE0ZGXNQDfzI4crOSkM+rce7cf1FMWqlrlGeT7NviY0iqUMEj4GKhNJoaoyFyxsEwMiEF8rLwb28YgoWX7BxniEsk2J79CP/O8k21HPbSu82YZuJFjjnU+KN6UXHlY471soujdUuJTu6ea9QZa0QTGnned8zxBnFpuTuEZaAe1ayxLoc+QX4trGw", # fmt: skip + "ruyi-cache.toml": b"eNptkM1ugzAQhO9+ii0VSitVSL1W6i055NJcuCFkOcaAFbAT20FCiHfvuuGvhJOxZ/abYV/h5xQfviAupYVcVgLwrJlihcjg3IK5tzKC/cnb4LA/xi+EJJzxUmS0+UzJ1Wg/Rbmua61oXrHCwjcEXQebUt8HpAtB5nAT9Z2epYKwnz4fg9MVzegVKkN72A9zI9abhGq8MBeKVmrqh3Jt4PIBDSBypUfSidq+vSMFgy+YOHRoxqaY7udR/99kCXXMFMLZBWzRaBQHfkqc1hUvmVT+HzNpxsRkt1Z26WJfTRLMum2t0doFKWY9vW4BB+lBfF4qwgvOqVTWsaqimP2HXr3N4JWwiZ02R34BhkzTWw==", # fmt: skip } diff --git a/ruyi/mux/venv/provision.py b/ruyi/mux/venv/provision.py index 020052f..a922bc5 100644 --- a/ruyi/mux/venv/provision.py +++ b/ruyi/mux/venv/provision.py @@ -51,7 +51,9 @@ def render_template_str(template_name: str, data: dict[str, Any]) -> str: def render_and_write( - dest: PathLike[Any], template_name: str, data: dict[str, Any] + dest: PathLike[Any], + template_name: str, + data: dict[str, Any], ) -> None: log.D(f"rendering template '{template_name}' with data {data}") content = render_template_str(template_name, data).encode("utf-8") @@ -66,7 +68,6 @@ def __init__( profile: ProfileProxy, targets: list[ConfiguredTargetTuple], dest: PathLike[Any], - sysroot_srcdir: PathLike[Any] | None, emulator_progs: list[EmulatorProgDecl] | None, emulator_root: PathLike[Any] | None, override_name: str | None = None, @@ -74,17 +75,37 @@ def __init__( self.profile = profile self.targets = targets self.venv_root = pathlib.Path(dest) - self.sysroot_srcdir = sysroot_srcdir self.emulator_progs = emulator_progs self.emulator_root = emulator_root self.override_name = override_name self.bindir = self.venv_root / "bin" - self.sysroot_destdir = ( - self.venv_root / "sysroot" - if self.sysroot_srcdir is not None - else None - ) + + def sysroot_srcdir(self, target_tuple: str | None) -> pathlib.Path | None: + if target_tuple is None: + # check the primary target + if s := self.targets[0]["toolchain_sysroot"]: + return pathlib.Path(s) + return None + + # check if we have this target + for t in self.targets: + if t["target"] != target_tuple: + continue + if s := t["toolchain_sysroot"]: + return pathlib.Path(s) + + return None + + def has_sysroot_for(self, target_tuple: str | None) -> bool: + return self.sysroot_srcdir(target_tuple) is not None + + def sysroot_destdir(self, target_tuple: str | None) -> pathlib.Path | None: + if not self.has_sysroot_for(target_tuple): + return None + + dirname = f"sysroot.{target_tuple}" if target_tuple is not None else "sysroot" + return self.venv_root / dirname def provision(self) -> None: venv_root = self.venv_root @@ -93,19 +114,9 @@ def provision(self) -> None: venv_root.mkdir() bindir.mkdir() - if self.sysroot_srcdir is not None: - log.D("copying sysroot") - assert self.sysroot_destdir is not None - shutil.copytree( - self.sysroot_srcdir, - self.sysroot_destdir, - symlinks=True, - ignore_dangling_symlinks=True, - ) - env_data = { "profile": self.profile.id, - "sysroot": self.sysroot_destdir, + "sysroot": self.sysroot_destdir(None), } render_and_write(venv_root / "ruyi-venv.toml", "ruyi-venv.toml", env_data) @@ -128,7 +139,7 @@ def provision(self) -> None: p, self.emulator_root, self.profile, - self.sysroot_destdir, + self.sysroot_destdir(None), ) for p in self.emulator_progs ] @@ -166,6 +177,7 @@ def make_venv_cache_data( targets_cache_data: dict[str, object] = { tgt["target"]: { "toolchain_bindir": str(pathlib.Path(tgt["toolchain_root"]) / "bin"), + "toolchain_sysroot": self.sysroot_destdir(tgt["target"]), "gcc_install_dir": tgt["gcc_install_dir"], } for tgt in self.targets } @@ -186,6 +198,26 @@ def provision_target( bindir = self.bindir target_tuple = tgt["target"] + # getting the destdir this way ensures it's suffixed with the target + # tuple + if sysroot_destdir := self.sysroot_destdir(target_tuple): + sysroot_srcdir = tgt["toolchain_sysroot"] + assert sysroot_srcdir is not None + + log.D(f"copying sysroot for {target_tuple}") + shutil.copytree( + sysroot_srcdir, + sysroot_destdir, + symlinks=True, + ignore_dangling_symlinks=True, + ) + + if is_primary: + log.D("symlinking primary sysroot into place") + primary_sysroot_destdir = self.sysroot_destdir(None) + assert primary_sysroot_destdir is not None + os.symlink(sysroot_destdir.name, primary_sysroot_destdir) + log.D(f"symlinking {target_tuple} binaries into venv") toolchain_bindir = pathlib.Path(tgt["toolchain_root"]) / "bin" symlink_binaries(toolchain_bindir, bindir) @@ -235,7 +267,7 @@ def provision_target( "cc": cc_path, "cxx": cxx_path, "processor": self.profile.arch, - "sysroot": self.sysroot_destdir, + "sysroot": self.sysroot_destdir(target_tuple), "venv_root": venv_root, "cmake_toolchain_file": str(cmake_toolchain_file_path), "meson_additional_binaries": meson_additional_binaries, diff --git a/ruyi/mux/venv/ruyi-cache.toml.jinja b/ruyi/mux/venv/ruyi-cache.toml.jinja index 91741e7..d890e85 100644 --- a/ruyi/mux/venv/ruyi-cache.toml.jinja +++ b/ruyi/mux/venv/ruyi-cache.toml.jinja @@ -9,5 +9,6 @@ profile_common_flags = "{{ profile_common_flags }}" {% endfor %}{% endif %} {% for k, v in targets.items() %}[cached_v1.targets.{{ k }}] toolchain_bindir = "{{ v['toolchain_bindir'] }}" +{% if v["toolchain_sysroot"] %}toolchain_sysroot = "{{ v['toolchain_sysroot'] }}"{% endif %} {% if v["gcc_install_dir"] %}gcc_install_dir = "{{ v['gcc_install_dir'] }}"{% endif %} {% endfor %} diff --git a/ruyi/mux/venv/venv_cli.py b/ruyi/mux/venv/venv_cli.py index 5c4d47a..7563cbb 100644 --- a/ruyi/mux/venv/venv_cli.py +++ b/ruyi/mux/venv/venv_cli.py @@ -41,7 +41,6 @@ def cli_venv(args: argparse.Namespace) -> int: target_arch = '' seen_target_tuples: set[str] = set() targets: list[ConfiguredTargetTuple] = [] - sysroot_dir: PathLike[Any] | None = None warn_differing_target_arch = False warn_multiple_sysroots = False @@ -82,7 +81,7 @@ def cli_venv(args: argparse.Namespace) -> int: log.F("cannot find the installed directory for the toolchain") return 1 - tc_sysroot_dir: PathLike[Any] + tc_sysroot_dir: PathLike[Any] | None = None gcc_install_dir: PathLike[Any] | None = None if with_sysroot: if tc_sysroot_relpath := tc_pm.toolchain_metadata.included_sysroot: @@ -144,18 +143,11 @@ def cli_venv(args: argparse.Namespace) -> int: ) return 1 - if sysroot_dir is None: - sysroot_dir = tc_sysroot_dir - else: - # check if multiple toolchain packages offer competing sysroots - if tc_sysroot_dir != sysroot_dir: - # first one wins - warn_multiple_sysroots = True - # record the target tuple info to configure in the venv configured_target: ConfiguredTargetTuple = { "target": target_tuple, "toolchain_root": toolchain_root, + "toolchain_sysroot": tc_sysroot_dir, # assume clang is preferred if package contains clang # this is mostly true given most packages don't contain both "cc_flavor": "clang" if tc_pm.toolchain_metadata.has_clang else "gcc", @@ -178,10 +170,6 @@ def cli_venv(args: argparse.Namespace) -> int: log.W("multiple toolchains specified with differing target architecture") log.I(f"using the target architecture of the first toolchain: [yellow]{target_arch}[/]") - if warn_multiple_sysroots: - log.W("selected packages for the virtual environment provide multiple competing sysroots") - log.I("using the sysroot from the first sysroot-providing package: [yellow]{sysroot_dir}[/]") - # Now handle the emulator. emu_progs = None emu_root: PathLike[Any] | None = None @@ -239,7 +227,6 @@ def cli_venv(args: argparse.Namespace) -> int: profile, targets, dest.resolve(), - sysroot_dir, emu_progs, emu_root, override_name, @@ -250,7 +237,7 @@ def cli_venv(args: argparse.Namespace) -> int: render_template_str( "prompt.venv-created.txt", { - "sysroot": maker.sysroot_destdir, + "sysroot": maker.sysroot_destdir(None), }, ) )