From 025bd055336e02e05be7d6fbd23a42705cb966cd Mon Sep 17 00:00:00 2001 From: Johannes Schlatow Date: Wed, 25 Sep 2024 17:15:19 +0200 Subject: [PATCH] add install-toolchain command The command downloads and installs the current Genode toolchain (if unavailable) and converts it into a squashfs. The squashfs is mounted via squashfuse and bind mounted into the bubblewrapped build environment. genodelabs/goa#99 --- bin/goa | 4 ++ share/bash-completion/goa | 6 +-- share/goa/doc/config.txt | 8 ++- share/goa/doc/overview.txt | 8 +++ share/goa/goarc | 3 ++ share/goa/lib/actions/build.tcl | 92 +++++++++++++++++++++++++++++---- share/goa/lib/build/qmake.tcl | 4 +- share/goa/lib/command_line.tcl | 9 +++- share/goa/lib/config.tcl | 8 ++- share/goa/lib/test.expect | 3 +- 10 files changed, 125 insertions(+), 20 deletions(-) diff --git a/bin/goa b/bin/goa index 2e7a457..cd366d1 100755 --- a/bin/goa +++ b/bin/goa @@ -188,6 +188,10 @@ if {![file exists src]} { set perform(build) 0 } +if {$perform(install-toolchain)} { + goa install-toolchain $args(keep_mounted) +} + if {$perform(build-dir)} { diff --git a/share/bash-completion/goa b/share/bash-completion/goa index d926a5c..793d668 100644 --- a/share/bash-completion/goa +++ b/share/bash-completion/goa @@ -7,7 +7,7 @@ __goa() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - commands="help update-goa --version import diff build-dir build extract-abi-symbols run export publish add-depot-user archive-versions depot-dir run-dir bump-version backtrace" + commands="help update-goa --version import diff build-dir build extract-abi-symbols run export publish add-depot-user archive-versions depot-dir run-dir bump-version backtrace install-toolchain" help_commands="config api build-systems artifacts add-depot-user import targets runtime index" depot_commands="export|publish" run_commands="run|run-dir|backtrace" @@ -15,11 +15,11 @@ __goa() run_opts="--pkg --run-as --target" user_opts="--depot-url --gpg-user-id --pubkey-file" - globalopts="--arch --rebuild --debug --warn-strict --no-warn-strict --with-backtrace -C --license --versions-from-genode-dir --depot-dir --public-dir --project-name --contrib-dir --build-dir --abi-dir --bin-dir --dbg-dir --run-dir --common-var-dir --verbose -r --jobs --cross-dev-prefix --ld-march --olevel --search-dir" + globalopts="--arch --rebuild --debug --warn-strict --no-warn-strict --with-backtrace -C --license --versions-from-genode-dir --depot-dir --public-dir --project-name --contrib-dir --build-dir --abi-dir --bin-dir --dbg-dir --run-dir --common-var-dir --verbose -r --jobs --cross-dev-prefix --ld-march --olevel --search-dir --install-dir" stropts="--depot-url|--gpg-user-id|--depot-user|--ld-march|--run_as" fileopts="--license|--pubkey-file|--cross-dev-prefix" - diropts="-C|--versions-from-genode-dir|--depot-dir|--public-dir|--project-name|--contrib-dir|--build-dir|--abi-dir|--bin-dir|--dbg-dir|--run-dir|--common-var-dir|--search-dir" + diropts="-C|--versions-from-genode-dir|--depot-dir|--public-dir|--project-name|--contrib-dir|--build-dir|--abi-dir|--bin-dir|--dbg-dir|--run-dir|--common-var-dir|--search-dir|--install-dir" pkgopts="--pkg" targetvals="linux sculpt" archvals="x86_64 arm_v8a" diff --git a/share/goa/doc/config.txt b/share/goa/doc/config.txt index 993e37c..841e5cb 100644 --- a/share/goa/doc/config.txt +++ b/share/goa/doc/config.txt @@ -57,7 +57,7 @@ Goa evaluates the following configuration variables. :cross_dev_prefix: The tool-chain prefix for the Genode cross compiler. - The default value is "/usr/local/genode/tool/current/bin/genode-x86-". + The default value is "/usr/local/genode/tool//bin/genode-x86-". :rebuild: If set to 1 or if the command line argument '--rebuild' is specified, @@ -110,6 +110,9 @@ Goa evaluates the following configuration variables. executing the 'run' command. The default is var/run/ within the project directory. +:install_dir: The directory where additional build tools are downloaded and + installed. + :common_var_dir: If defined, Goa places all generated files in the specified directory instead of creating a 'var' directory inside the project directory. @@ -196,6 +199,9 @@ Goa evaluates the following configuration variables. configuration variable in a custom 'goarc' file or via the '--sculpt-version' command-line argument. +:toolchain_version: + Specifies the to-be-used version of the Genode toolchain. + :target: Specifies the run target used for 'goa run'. The default value is 'linux'. The value can be overridden by setting this configuration variable or by diff --git a/share/goa/doc/overview.txt b/share/goa/doc/overview.txt index e30952f..63379d8 100644 --- a/share/goa/doc/overview.txt +++ b/share/goa/doc/overview.txt @@ -116,6 +116,14 @@ USAGE :goa diff: Show local modifications of the imported 3rd-party software. +:goa install-toolchain [--keep-mounted]: + Downloads and installs the Genode toolchain as a squashfs archive into + the var/install/ directory if no system-wide installation was found. This + command is executed as a prerequisite of the 'build-dir' command. During + Goa's execution, the squashfs archive will be temporarily mounted using FUSE. + When provided with the --keep-mounted option, the mount is kept persistent + after Goa exited. + :goa depot-dir: Create a depot directory and install default depot users. diff --git a/share/goa/goarc b/share/goa/goarc index 4e3fbf0..74c9195 100644 --- a/share/goa/goarc +++ b/share/goa/goarc @@ -1,6 +1,9 @@ # Sculpt version is required for publishing index projects set sculpt_version 24.04 +# version of Genode toolchain +set toolchain_version 23.05 + # archive versions set version(genodelabs/api/base) 2024-04-11 set version(genodelabs/api/base-linux) 2024-04-11 diff --git a/share/goa/lib/actions/build.tcl b/share/goa/lib/actions/build.tcl index 6130081..ea1dc30 100644 --- a/share/goa/lib/actions/build.tcl +++ b/share/goa/lib/actions/build.tcl @@ -6,6 +6,7 @@ namespace eval goa { namespace export build-dir build using_api used_apis check_abis namespace export build extract_artifacts_from_build_dir extract_api_artifacts namespace export extract-abi-symbols + namespace export install-toolchain proc sandboxed_build_command { } { global config::project_dir config::depot_dir config::var_dir config::build_dir @@ -70,6 +71,7 @@ namespace eval goa { proc gaol_with_toolchain { {silent 0} } { global gaol verbose allowed_tools + variable toolchain_dirs set cmd $gaol lappend cmd --system-usr @@ -77,6 +79,9 @@ namespace eval goa { foreach dir [lsearch -all -inline -not $allowed_tools /usr*] { lappend cmd --ro-bind $dir } + foreach { base_dir at } $toolchain_dirs { + lappend cmd --ro-bind-at [file join $base_dir $at] /$at } + if {$verbose && !$silent} { lappend cmd --verbose } @@ -125,6 +130,17 @@ namespace eval goa { } + proc is_toolchain_path { path } { + variable toolchain_dirs + + foreach { dummy dir } $toolchain_dirs { + if {[regexp "^/$dir" $path]} { + return 1 } } + + return 0 + } + + proc used_apis { } { variable _used_apis @@ -236,6 +252,70 @@ namespace eval goa { exit_with_error "failed to generate ldso_support.lib.a "] } } + ## + # Make tool chain available (download and install if necessary) + # + proc install-toolchain { keep_mounted } { + + global gaol tool_dir + global config::cross_dev_prefix + global config::install_dir config::toolchain_version + variable toolchain_dirs {} + + ## + # Check for system-wide availability of the Genode tool chain + # + if {[have_installed ${cross_dev_prefix}gcc]} { + return } + + set toolchain genode-toolchain-$toolchain_version + + if {![file exists $install_dir]} { + file mkdir $install_dir } + + exit_if_not_installed curl xzcat sqfstar squashfuse_ll + + # trigger (re-)download, integrity check and squashfs creation + set install_cmd $gaol + lappend install_cmd --system-usr + lappend install_cmd --ro-bind [file join $tool_dir lib] + lappend install_cmd --bind $install_dir + lappend install_cmd --with-network + lappend install_cmd [file join $tool_dir lib install_tool.mk] + lappend install_cmd $toolchain + lappend install_cmd INSTALL_DIR=$install_dir + if {[catch {exec {*}$install_cmd >&@ stdout}]} { + exit_with_error "Unable to install $toolchain" } + + # create mountpoint + set mount_dir [file join $install_dir $toolchain] + if {![file exists $mount_dir]} { + file mkdir $mount_dir + } elseif {[glob -nocomplain -dir $mount_dir *] != ""} { + exec fusermount -u $mount_dir + } + + # mount squashfs + set mount_cmd squashfuse_ll + if {!$keep_mounted} { + lappend mount_cmd -f } + lappend mount_cmd [file join $install_dir download $toolchain.squashfs] + lappend mount_cmd $mount_dir + spawn -noecho {*}$mount_cmd + + # remember mountpoint and location (used in gaol_with_toolchain) + lappend toolchain_dirs $mount_dir usr/local/genode/tool + + # check mount availability + after 100 + if {![file exists $mount_dir/${cross_dev_prefix}gcc]} { + exit_with_error "the tool chain ${cross_dev_prefix}" \ + "is required but not installed." \ + "Please refer to https://genode.org/download/tool-chain" \ + "for more information." + } + } + ## # Implements 'goa build-dir' command # @@ -249,16 +329,6 @@ namespace eval goa { global config::with_backtrace config::warn_strict config::depot_user global config::project_name config::project_dir - # - # Check for availability of the Genode tool chain - # - if {![have_installed ${cross_dev_prefix}gcc]} { - exit_with_error "the tool chain ${cross_dev_prefix}" \ - "is required but not installed." \ - "Please refer to https://genode.org/download/tool-chain" \ - "for more information." - } - # # Prepare depot content for the used APIs and generate ABI stubs # @@ -286,7 +356,7 @@ namespace eval goa { # filter out non-existing include directories foreach dir $include_dirs { - if {[file exists $dir]} { + if {[file exists $dir] || [is_toolchain_path $dir]} { lappend existing_include_dirs $dir } } set include_dirs $existing_include_dirs diff --git a/share/goa/lib/build/qmake.tcl b/share/goa/lib/build/qmake.tcl index bc0c205..4aab63f 100644 --- a/share/goa/lib/build/qmake.tcl +++ b/share/goa/lib/build/qmake.tcl @@ -27,7 +27,9 @@ proc create_or_update_build_dir { } { set qt5_api "qt5_base" - file link -symbolic qmake_root/bin $qt5_tool_dir + # $qt5_tool_dir might only exist in sandbox environment, hence use ln + exec ln -sf $qt5_tool_dir qmake_root_bin + file link -symbolic qmake_root/include [file join [api_archive_dir $qt5_api] include] file link -symbolic qmake_root/lib $abi_dir diff --git a/share/goa/lib/command_line.tcl b/share/goa/lib/command_line.tcl index 345b3e5..5dba78e 100644 --- a/share/goa/lib/command_line.tcl +++ b/share/goa/lib/command_line.tcl @@ -152,7 +152,7 @@ if {[llength $argv] == 0} { set avail_commands [list update-goa archive-versions backtrace import diff build-dir \ build run run-dir export publish add-depot-user bump-version \ - extract-abi-symbols help versions depot-dir] + extract-abi-symbols help versions depot-dir install-toolchain] foreach command $avail_commands { set perform($command) 0 } @@ -180,6 +180,7 @@ action_dependency backtrace run action_dependency run run-dir action_dependency run-dir build action_dependency build build-dir +action_dependency build-dir install-toolchain action_dependency build-dir depot-dir action_dependency add-depot-user depot-dir @@ -256,6 +257,12 @@ if {$perform(add-depot-user)} { exit_with_error "public-key file $args(pubkey_file) does not exist" } } +if {$perform(install-toolchain)} { + set args(keep_mounted) 0 + if {[consume_optional_cmdline_switch "--keep-mounted"]} { + set args(keep_mounted) 1 } +} + # override 'rebuild' variable via optional command-line switch if {$perform(build-dir)} { if {[consume_optional_cmdline_switch "--rebuild"]} { diff --git a/share/goa/lib/config.tcl b/share/goa/lib/config.tcl index e320961..deeffc0 100644 --- a/share/goa/lib/config.tcl +++ b/share/goa/lib/config.tcl @@ -26,6 +26,7 @@ namespace eval ::config { variable run_as "genodelabs" variable target "linux" variable sculpt_version "" + variable toolchain_version "" variable cc_cxx_opt_std "-std=gnu++20" variable binary_name "" variable with_backtrace 0 @@ -40,6 +41,7 @@ namespace eval ::config { variable run_dir "" variable bin_dir "" variable dbg_dir "" + variable install_dir "" variable target_opt array set target_opt {} variable version @@ -318,6 +320,7 @@ namespace eval ::config { variable run_as variable binary_name variable var_dir + variable toolchain_version if {$versions_from_genode_dir == ""} { unset versions_from_genode_dir } if {$license == ""} { unset license } @@ -339,8 +342,8 @@ namespace eval ::config { if {![info exists cross_dev_prefix]} { switch $arch { - arm_v8a { set cross_dev_prefix "/usr/local/genode/tool/23.05/bin/genode-aarch64-" } - x86_64 { set cross_dev_prefix "/usr/local/genode/tool/23.05/bin/genode-x86-" } + arm_v8a { set cross_dev_prefix "/usr/local/genode/tool/$toolchain_version/bin/genode-aarch64-" } + x86_64 { set cross_dev_prefix "/usr/local/genode/tool/$toolchain_version/bin/genode-x86-" } default { exit_with_error "tool-chain prefix is not defined" } } } @@ -379,6 +382,7 @@ namespace eval ::config { set_if_undefined dbg_dir [file join $var_dir dbg $arch] set_if_undefined run_dir [file join $var_dir run] set_if_undefined api_dir [file join $var_dir api] + set_if_undefined install_dir [file join $var_dir install] } # make namespace procs available as subcommands diff --git a/share/goa/lib/test.expect b/share/goa/lib/test.expect index c9b5899..519bb62 100755 --- a/share/goa/lib/test.expect +++ b/share/goa/lib/test.expect @@ -101,7 +101,8 @@ proc run_goa_until {{test} {timeout_value 0}} { variable public var set public [file join [_find_tool_dir] public] set var [file join [_find_tool_dir] var] - eval spawn $goa run -C $test_dir --public-dir $public --common-var-dir $var + set tool [file join $var tool] + eval spawn $goa run -C $test_dir --public-dir $public --install-dir $tool --common-var-dir $var set result [wait_for_output $test_output($test) $timeout_value $spawn_id] #