add install-toolchain command

The command downloads and installs the current Genode toolchain (if unavailable)
and converts it into a squashfs. The squashfs is mounted via squashfuse
and bind mounted into the bubblewrapped build environment.

genodelabs#99

bin/goa

@@ -188,6 +188,10 @@ if {![file exists src]} {
  set perform(build) 0
 }
 
+if {$perform(install-toolchain)} {
+ goa install-toolchain $args(keep_mounted)
+}
+
 
 if {$perform(build-dir)} {
 

share/bash-completion/goa

@@ -7,19 +7,19 @@ __goa()
  cur="${COMP_WORDS[COMP_CWORD]}"
  prev="${COMP_WORDS[COMP_CWORD-1]}"
 
- commands="help update-goa --version import diff build-dir build extract-abi-symbols run export publish add-depot-user archive-versions depot-dir run-dir bump-version backtrace"
+ commands="help update-goa --version import diff build-dir build extract-abi-symbols run export publish add-depot-user archive-versions depot-dir run-dir bump-version backtrace install-toolchain"
  help_commands="config api build-systems artifacts add-depot-user import targets runtime index"
  depot_commands="export|publish"
  run_commands="run|run-dir|backtrace"
  depot_opts="--depot-user --depot-overwrite --depot-retain --pkg"
  run_opts="--pkg --run-as --target"
  user_opts="--depot-url --gpg-user-id --pubkey-file"
 
- globalopts="--arch --rebuild --debug --warn-strict --no-warn-strict --with-backtrace -C --license --versions-from-genode-dir --depot-dir --public-dir --project-name --contrib-dir --build-dir --abi-dir --bin-dir --dbg-dir --run-dir --common-var-dir --verbose -r --jobs --cross-dev-prefix --ld-march --olevel --search-dir"
+ globalopts="--arch --rebuild --debug --warn-strict --no-warn-strict --with-backtrace -C --license --versions-from-genode-dir --depot-dir --public-dir --project-name --contrib-dir --build-dir --abi-dir --bin-dir --dbg-dir --run-dir --common-var-dir --verbose -r --jobs --cross-dev-prefix --ld-march --olevel --search-dir --install-dir"
 
  stropts="--depot-url|--gpg-user-id|--depot-user|--ld-march|--run_as"
  fileopts="--license|--pubkey-file|--cross-dev-prefix"
- diropts="-C|--versions-from-genode-dir|--depot-dir|--public-dir|--project-name|--contrib-dir|--build-dir|--abi-dir|--bin-dir|--dbg-dir|--run-dir|--common-var-dir|--search-dir"
+ diropts="-C|--versions-from-genode-dir|--depot-dir|--public-dir|--project-name|--contrib-dir|--build-dir|--abi-dir|--bin-dir|--dbg-dir|--run-dir|--common-var-dir|--search-dir|--install-dir"
  pkgopts="--pkg"
  targetvals="linux sculpt"
  archvals="x86_64 arm_v8a"

share/goa/doc/config.txt

@@ -57,7 +57,7 @@ Goa evaluates the following configuration variables.
 
 :cross_dev_prefix:
  The tool-chain prefix for the Genode cross compiler.
- The default value is "/usr/local/genode/tool/current/bin/genode-x86-".
+ The default value is "/usr/local/genode/tool/<toolchain_version>/bin/genode-x86-".
 
 :rebuild:
  If set to 1 or if the command line argument '--rebuild' is specified,
@@ -110,6 +110,9 @@ Goa evaluates the following configuration variables.
  executing the 'run' command. The default is var/run/ within the project
  directory.
 
+:install_dir: The directory where additional build tools are downloaded and
+ installed.
+
 :common_var_dir:
  If defined, Goa places all generated files in the specified directory
  instead of creating a 'var' directory inside the project directory.
@@ -196,6 +199,9 @@ Goa evaluates the following configuration variables.
  configuration variable in a custom 'goarc' file or via the
  '--sculpt-version' command-line argument.
 
+:toolchain_version:
+ Specifies the to-be-used version of the Genode toolchain.
+
 :target:
  Specifies the run target used for 'goa run'. The default value is 'linux'.
  The value can be overridden by setting this configuration variable or by

share/goa/doc/overview.txt

@@ -116,6 +116,14 @@ USAGE
 :goa diff:
  Show local modifications of the imported 3rd-party software.
 
+:goa install-toolchain [--keep-mounted]:
+ Downloads and installs the Genode toolchain as a squashfs archive into
+ the var/install/ directory if no system-wide installation was found. This
+ command is executed as a prerequisite of the 'build-dir' command. During
+ Goa's execution, the squashfs archive will be temporarily mounted using FUSE.
+ When provided with the --keep-mounted option, the mount is kept persistent
+ after Goa exited.
+
 :goa depot-dir:
  Create a depot directory and install default depot users.
 

share/goa/goarc

@@ -1,6 +1,9 @@
 # Sculpt version is required for publishing index projects
 set sculpt_version 24.04
 
+# version of Genode toolchain
+set toolchain_version 23.05
+
 # archive versions
 set version(genodelabs/api/base) 2024-04-11
 set version(genodelabs/api/base-linux) 2024-04-11

share/goa/lib/actions/build.tcl

@@ -6,6 +6,7 @@ namespace eval goa {
  namespace export build-dir build using_api used_apis check_abis
  namespace export build extract_artifacts_from_build_dir extract_api_artifacts
  namespace export extract-abi-symbols
+ namespace export install-toolchain
 
  proc sandboxed_build_command { } {
  global config::project_dir config::depot_dir config::var_dir config::build_dir
@@ -70,13 +71,17 @@ namespace eval goa {
  proc gaol_with_toolchain { {silent 0} } {
 
  global gaol verbose allowed_tools
+ variable toolchain_dirs
 
  set cmd $gaol
  lappend cmd --system-usr
 
  foreach dir [lsearch -all -inline -not $allowed_tools /usr*] {
  lappend cmd --ro-bind $dir }
 
+ foreach { base_dir at } $toolchain_dirs {
+ lappend cmd --ro-bind-at [file join $base_dir $at] /$at }
+
  if {$verbose && !$silent} {
  lappend cmd --verbose }
 
@@ -125,6 +130,17 @@ namespace eval goa {
  }
 
 
+ proc is_toolchain_path { path } {
+ variable toolchain_dirs
+
+ foreach { dummy dir } $toolchain_dirs {
+ if {[regexp "^/$dir" $path]} {
+ return 1 } }
+
+ return 0
+ }
+
+
  proc used_apis { } {
 
  variable _used_apis
@@ -236,6 +252,70 @@ namespace eval goa {
  exit_with_error "failed to generate ldso_support.lib.a "] }
  }
 
+ ##
+ # Make tool chain available (download and install if necessary)
+ #
+ proc install-toolchain { keep_mounted } {
+
+ global gaol tool_dir
+ global config::cross_dev_prefix
+ global config::install_dir config::toolchain_version
+ variable toolchain_dirs {}
+
+ ##
+ # Check for system-wide availability of the Genode tool chain
+ #
+ if {[have_installed ${cross_dev_prefix}gcc]} {
+ return }
+
+ set toolchain genode-toolchain-$toolchain_version
+
+ if {![file exists $install_dir]} {
+ file mkdir $install_dir }
+
+ exit_if_not_installed curl xzcat sqfstar squashfuse_ll
+
+ # trigger (re-)download, integrity check and squashfs creation
+ set install_cmd $gaol
+ lappend install_cmd --system-usr
+ lappend install_cmd --ro-bind [file join $tool_dir lib]
+ lappend install_cmd --bind $install_dir
+ lappend install_cmd --with-network
+ lappend install_cmd [file join $tool_dir lib install_tool.mk]
+ lappend install_cmd $toolchain
+ lappend install_cmd INSTALL_DIR=$install_dir
+ if {[catch {exec {*}$install_cmd >&@ stdout}]} {
+ exit_with_error "Unable to install $toolchain" }
+
+ # create mountpoint
+ set mount_dir [file join $install_dir $toolchain]
+ if {![file exists $mount_dir]} {
+ file mkdir $mount_dir
+ } elseif {[glob -nocomplain -dir $mount_dir *] != ""} {
+ exec fusermount -u $mount_dir
+ }
+
+ # mount squashfs
+ set mount_cmd squashfuse_ll
+ if {!$keep_mounted} {
+ lappend mount_cmd -f }
+ lappend mount_cmd [file join $install_dir download $toolchain.squashfs]
+ lappend mount_cmd $mount_dir
+ spawn -noecho {*}$mount_cmd
+
+ # remember mountpoint and location (used in gaol_with_toolchain)
+ lappend toolchain_dirs $mount_dir usr/local/genode/tool
+
+ # check mount availability
+ after 100
+ if {![file exists $mount_dir/${cross_dev_prefix}gcc]} {
+ exit_with_error "the tool chain ${cross_dev_prefix}" \
+ "is required but not installed." \
+ "Please refer to https://genode.org/download/tool-chain" \
+ "for more information."
+ }
+ }
+
  ##
  # Implements 'goa build-dir' command
  #
@@ -249,16 +329,6 @@ namespace eval goa {
  global config::with_backtrace config::warn_strict config::depot_user
  global config::project_name config::project_dir
 
- #
- # Check for availability of the Genode tool chain
- #
- if {![have_installed ${cross_dev_prefix}gcc]} {
- exit_with_error "the tool chain ${cross_dev_prefix}" \
- "is required but not installed." \
- "Please refer to https://genode.org/download/tool-chain" \
- "for more information."
- }
-
  #
  # Prepare depot content for the used APIs and generate ABI stubs
  #
@@ -286,7 +356,7 @@ namespace eval goa {
 
  # filter out non-existing include directories
  foreach dir $include_dirs {
- if {[file exists $dir]} {
+ if {[file exists $dir] || [is_toolchain_path $dir]} {
  lappend existing_include_dirs $dir } }
  set include_dirs $existing_include_dirs
 

share/goa/lib/build/qmake.tcl

@@ -27,7 +27,9 @@ proc create_or_update_build_dir { } {
 
  set qt5_api "qt5_base"
 
- file link -symbolic qmake_root/bin $qt5_tool_dir
+ # $qt5_tool_dir might only exist in sandbox environment, hence use ln
+ exec ln -sf $qt5_tool_dir qmake_root_bin
+
  file link -symbolic qmake_root/include [file join [api_archive_dir $qt5_api] include]
  file link -symbolic qmake_root/lib $abi_dir
 

share/goa/lib/command_line.tcl

@@ -152,7 +152,7 @@ if {[llength $argv] == 0} {
 
 set avail_commands [list update-goa archive-versions backtrace import diff build-dir \
  build run run-dir export publish add-depot-user bump-version \
- extract-abi-symbols help versions depot-dir]
+ extract-abi-symbols help versions depot-dir install-toolchain]
 
 foreach command $avail_commands {
  set perform($command) 0 }
@@ -180,6 +180,7 @@ action_dependency backtrace run
 action_dependency run run-dir
 action_dependency run-dir build
 action_dependency build build-dir
+action_dependency build-dir install-toolchain
 action_dependency build-dir depot-dir
 action_dependency add-depot-user depot-dir
 
@@ -256,6 +257,12 @@ if {$perform(add-depot-user)} {
  exit_with_error "public-key file $args(pubkey_file) does not exist" }
 }
 
+if {$perform(install-toolchain)} {
+ set args(keep_mounted) 0
+ if {[consume_optional_cmdline_switch "--keep-mounted"]} {
+ set args(keep_mounted) 1 }
+}
+
 # override 'rebuild' variable via optional command-line switch
 if {$perform(build-dir)} {
  if {[consume_optional_cmdline_switch "--rebuild"]} {

share/goa/lib/config.tcl

@@ -26,6 +26,7 @@ namespace eval ::config {
  variable run_as "genodelabs"
  variable target "linux"
  variable sculpt_version ""
+ variable toolchain_version ""
  variable cc_cxx_opt_std "-std=gnu++20"
  variable binary_name ""
  variable with_backtrace 0
@@ -40,6 +41,7 @@ namespace eval ::config {
  variable run_dir ""
  variable bin_dir ""
  variable dbg_dir ""
+ variable install_dir ""
  variable target_opt
  array set target_opt {}
  variable version
@@ -318,6 +320,7 @@ namespace eval ::config {
  variable run_as
  variable binary_name
  variable var_dir
+ variable toolchain_version
 
  if {$versions_from_genode_dir == ""} { unset versions_from_genode_dir }
  if {$license == ""} { unset license }
@@ -339,8 +342,8 @@ namespace eval ::config {
 
  if {![info exists cross_dev_prefix]} {
  switch $arch {
- arm_v8a { set cross_dev_prefix "/usr/local/genode/tool/23.05/bin/genode-aarch64-" }
- x86_64 { set cross_dev_prefix "/usr/local/genode/tool/23.05/bin/genode-x86-" }
+ arm_v8a { set cross_dev_prefix "/usr/local/genode/tool/$toolchain_version/bin/genode-aarch64-" }
+ x86_64 { set cross_dev_prefix "/usr/local/genode/tool/$toolchain_version/bin/genode-x86-" }
  default { exit_with_error "tool-chain prefix is not defined" }
  }
  }
@@ -379,6 +382,7 @@ namespace eval ::config {
  set_if_undefined dbg_dir [file join $var_dir dbg $arch]
  set_if_undefined run_dir [file join $var_dir run]
  set_if_undefined api_dir [file join $var_dir api]
+ set_if_undefined install_dir [file join $var_dir install]
  }
 
  # make namespace procs available as subcommands

share/goa/lib/test.expect

@@ -101,7 +101,8 @@ proc run_goa_until {{test} {timeout_value 0}} {
  variable public var
  set public [file join [_find_tool_dir] public]
  set var [file join [_find_tool_dir] var]
- eval spawn $goa run -C $test_dir --public-dir $public --common-var-dir $var
+ set tool [file join $var tool]
+ eval spawn $goa run -C $test_dir --public-dir $public --install-dir $tool --common-var-dir $var
  set result [wait_for_output $test_output($test) $timeout_value $spawn_id]
 
  #