[scripts] Introduce scripts/fetch-toolchains.py

This change introduces $LK_ROOT/toolchains as an official installation
directory for toolchains, as well as a new script for conveniently
installing them there. getting_started.md is updated to suggest its
use.

.gitignore

@@ -16,3 +16,4 @@ scripts/toolpaths.local
 .vscode
 .cache
 compile_commands.json
+toolchain

docs/getting_started.md

@@ -6,25 +6,23 @@
 ```
 mkdir -p lk-work && cd lk-work
 ```
-2- Clone the repo
+2- Clone the repo and change dir to the root
 ```
 git clone https://github.com/littlekernel/lk
+cd lk
+
 ```
-3- Download appropriate toolchain and extract it
+3- Download appropriate toolchain
 ```
-wget https://newos.org/toolchains/riscv64-elf-14.2.0-Linux-x86_64.tar.xz
-
-mkdir -p toolchain
-tar xf riscv64-elf-14.2.0-Linux-x86_64.tar.xz
-cd ..
+# Fetches the latest riscv64-elf toolchain for your host.
+scripts/fetch-toolchains.py --prefix riscv64-elf
 ```
 4- Add toolchain to PATH
 ```
 export PATH=$PWD/toolchain/riscv64-elf-14.2.0-Linux-x86_64/bin:$PATH
 ```
-5- Change dir to lk to build and find available project
+5- Find available project
 ```
-cd lk
 ls project/*
 ```
 6- E.g pick `qemu-virt-riscv64-test` and build kernel

scripts/fetch-toolchains.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+
+# A utility for installing LK toolchains.
+
+from __future__ import annotations
+
+import argparse
+import html.parser
+import io
+import os
+import pathlib
+import sys
+import tarfile
+import threading
+import urllib.request
+from typing import Self
+
+BASE_URL = "https://newos.org/toolchains"
+
+HOST_OS = os.uname().sysname
+HOST_CPU = os.uname().machine
+
+LK_ROOT = pathlib.Path(os.path.realpath(__file__)).parent.parent
+DEFAULT_TOOLCHAIN_DIR = LK_ROOT.joinpath("toolchain")
+
+TAR_EXT = ".tar.xz"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="Installs the matching LK toolchains from the official host, "
+        + BASE_URL,
+    )
+    parser.add_argument(
+        "--list",
+        help="just list the matching toolchains; don't download them",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--prefix",
+        help="a toolchain prefix on which to match. If none are specified, all prefixes"
+        " will match",
+        nargs="*",
+    )
+    parser.add_argument(
+        "--version",
+        help='the exact toolchain version to match, or "latest" to specify only the '
+        'latest version, or "all" for all versions',
+        type=str,
+        default="latest",
+    )
+    parser.add_argument(
+        "--install-dir",
+        help="the directory at which to install the toolchains",
+        type=pathlib.Path,
+        default=DEFAULT_TOOLCHAIN_DIR,
+    )
+    parser.add_argument(
+        "--force",
+        help="whether to overwrite past installed versions of matching toolchains",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--host-os",
+        help="the toolchains' host OS",
+        type=str,
+        default=HOST_OS,
+    )
+    parser.add_argument(
+        "--host-cpu",
+        help="the toolchains' host architecture",
+        type=str,
+        default=HOST_CPU,
+    )
+    args = parser.parse_args()
+
+    # Get the full list of remote toolchains available for the provided host.
+    response = urllib.request.urlopen(BASE_URL)
+    if response.status != 200:
+        print(f"Error accessing {BASE_URL}: {response.status}")
+        return 1
+    parser = RemoteToolchainHTMLParser(args.host_os, args.host_cpu)
+    parser.feed(response.read().decode("utf-8"))
+    toolchains = parser.toolchains
+
+    # Filter them given --prefix and --version selections.
+    toolchains.sort()
+    if args.prefix:
+        toolchains = [t for t in toolchains if t.prefix in args.prefix]
+    if args.version == "latest":
+        # Since we sorted lexicographically on (prefix, version), to pick out the latest
+        # versions we need only iterate through an pick out the last entry with a given
+        # prefix.
+        toolchains = [
+            toolchains[i]
+            for i in range(len(toolchains))
+            if (
+                i == len(toolchains) - 1
+                or toolchains[i].prefix != toolchains[i + 1].prefix
+            )
+        ]
+    elif args.version != "all":
+        toolchains = [t for t in toolchains if t.version == args.version]
+
+    if not toolchains:
+        print("No matching toolchains")
+        return 0
+
+    if args.list:
+        print("Matching toolchains:")
+        for toolchain in toolchains:
+            print(toolchain.name)
+        return 0
+
+    # The download routine for a given toolchain, factored out for
+    # multithreading below.
+    def download(toolchain: RemoteToolchain) -> None:
+        response = urllib.request.urlopen(toolchain.url)
+        if response.status != 200:
+            print(f"Error while downloading {toolchain.name}: {response.status}")
+            return
+        with tarfile.open(fileobj=io.BytesIO(response.read()), mode="r:xz") as f:
+            f.extractall(path=args.install_dir, filter="data")
+
+    downloads = []
+    for toolchain in toolchains:
+        local = args.install_dir.joinpath(toolchain.name)
+        if local.exists() and not args.force:
+            print(
+                f"{toolchain.name} already installed; "
+                "skipping... (pass --force to overwrite)",
+            )
+            continue
+        print(f"Downloading {toolchain.name} to {local}...")
+        downloads.append(threading.Thread(target=download, args=(toolchain,)))
+        downloads[-1].start()
+
+    for thread in downloads:
+        thread.join()
+
+    return 0
+
+
+class RemoteToolchain:
+    def __init__(self, prefix: str, version: str, host_os: str, host_cpu: str) -> None:
+        self._prefix = prefix
+        self._version = version
+        self._host = f"{host_os}-{host_cpu}"
+
+    # Orders toolchains lexicographically on (prefix, version).
+    def __lt__(self, other: Self) -> bool:
+        return self._prefix < other.prefix or (
+            self._prefix == other.prefix and self._version < other.version
+        )
+
+    @property
+    def prefix(self) -> str:
+        return self._prefix
+
+    @property
+    def version(self) -> str:
+        return self._version
+
+    @property
+    def name(self) -> str:
+        return f"{self._prefix}-{self._version}-{self._host}"
+
+    @property
+    def url(self) -> str:
+        return f"{BASE_URL}/{self.name}{TAR_EXT}"
+
+
+# A simple HTML parser for extracting the toolchain names found at BASE_URL.
+#
+# It expects toolchains to be available as hyperlinks on that page. Once the
+# HTML has been passed to feed(), the parsed toolchains will be accessible via
+# toolchains().
+class RemoteToolchainHTMLParser(html.parser.HTMLParser):
+    def __init__(self, host_os: str, host_cpu: str) -> None:
+        html.parser.HTMLParser.__init__(self)
+        self._toolchains = []
+        self._tags = []
+        self._host_os = host_os
+        self._host_cpu = host_cpu
+
+    # The parsed toolchains.
+    @property
+    def toolchains(self) -> list[RemoteToolchain]:
+        return self._toolchains
+
+    #
+    # The following methods implement the parsing, overriding those defined in
+    # the base class.
+    #
+
+    def handle_starttag(self, tag: str, _: str) -> None:
+        self._tags.append(tag)
+
+    def handle_endtag(self, _: str) -> None:
+        self._tags.pop()
+
+    def handle_data(self, data: str) -> None:
+        # Only process hyperlinks with tarball names.
+        if not self._tags or self._tags[-1] != "a" or not data.endswith(TAR_EXT):
+            return
+        tokens = data.removesuffix(TAR_EXT).split("-")
+        if len(tokens) != 5:
+            print(f"Warning: malformed toolchain name: {data}")
+            return
+        prefix = tokens[0] + "-" + tokens[1]
+        version = tokens[2]
+        host_os = tokens[3]
+        host_cpu = tokens[4]
+        if host_os != self._host_os or host_cpu != self._host_cpu:
+            return
+        self._toolchains.append(RemoteToolchain(prefix, version, host_os, host_cpu))
+
+
+if __name__ == "__main__":
+    sys.exit(main())