From 3c9cf36081829d5e5b96b669a6154f9db2363933 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 10 Apr 2024 20:08:10 +0200 Subject: [PATCH] allow extensions to trigger from packages in [deps] (#54009) There is a use case where you have a weak dependency (for one of your extensions) that is misbehaving and you quickly want to try debug that issue. A workflow that feels reasonable for this could be: ``` pkg> dev WeakDependency julia> using Package, WeakDependency ``` This doesn't work right now for two reasons: 1. Doing the `dev WeakDependency` will add the dependency to `[deps]` but not remove it from `[weakdeps]` which means you all of a sudden are in the scenario described in https://pkgdocs.julialang.org/v1/creating-packages/#Transition-from-normal-dependency-to-extension which is not what is desired. 2. The extension will not actually load because you can right now only trigger extensions from weak deps getting loaded, not from deps getting loaded. Point 1. is fixed by https://github.com/JuliaLang/Pkg.jl/pull/3865 Point 2. is fixed by this PR. (cherry picked from commit f46cb4c67cd2ce7f88da778f5c0355c1100b5ccd) --- base/loading.jl | 53 +++++++++++-------- base/precompilation.jl | 2 +- doc/src/manual/code-loading.md | 12 ++--- test/loading.jl | 5 ++ .../EnvWithHasExtensions/Manifest.toml | 6 ++- .../Extensions/ExtDep3.jl/Project.toml | 4 ++ .../Extensions/ExtDep3.jl/src/ExtDep3.jl | 5 ++ .../HasDepWithExtensions.jl/Manifest.toml | 11 +++- .../HasDepWithExtensions.jl/Project.toml | 1 + .../Extensions/HasExtensions.jl/Manifest.toml | 9 ++-- .../Extensions/HasExtensions.jl/Project.toml | 4 ++ .../HasExtensions.jl/ext/ExtensionDep.jl | 9 ++++ .../HasExtensions.jl/src/HasExtensions.jl | 1 + 13 files changed, 85 insertions(+), 37 deletions(-) create mode 100644 test/project/Extensions/ExtDep3.jl/Project.toml create mode 100644 test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl create mode 100644 test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl diff --git a/base/loading.jl b/base/loading.jl index 360da566a5f88..140131b26e7b2 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1405,13 +1405,12 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi proj_pkg = project_file_name_uuid(implicit_project_file, pkg.name) if pkg == proj_pkg d_proj = parsed_toml(implicit_project_file) - weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return - weakdeps === nothing && return - if weakdeps isa Dict{String, Any} - return _insert_extension_triggers(pkg, extensions, weakdeps) - end + weakdeps = get(Dict{String, Any}, d_proj, "weakdeps")::Dict{String,Any} + deps = get(Dict{String, Any}, d_proj, "deps")::Dict{String,Any} + total_deps = merge(weakdeps, deps) + return _insert_extension_triggers(pkg, extensions, total_deps) end # Now look in manifest @@ -1426,27 +1425,35 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi uuid = get(entry, "uuid", nothing)::Union{String, Nothing} uuid === nothing && continue if UUID(uuid) == pkg.uuid - weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return - weakdeps === nothing && return - if weakdeps isa Dict{String, Any} - return _insert_extension_triggers(pkg, extensions, weakdeps) + weakdeps = get(Dict{String, Any}, entry, "weakdeps")::Union{Vector{String}, Dict{String,Any}} + deps = get(Dict{String, Any}, entry, "deps")::Union{Vector{String}, Dict{String,Any}} + + function expand_deps_list(deps′::Vector{String}) + deps′_expanded = Dict{String, Any}() + for (dep_name, entries) in d + dep_name in deps′ || continue + entries::Vector{Any} + if length(entries) != 1 + error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") + end + entry = first(entries)::Dict{String, Any} + uuid = entry["uuid"]::String + deps′_expanded[dep_name] = uuid + end + return deps′_expanded end - d_weakdeps = Dict{String, Any}() - for (dep_name, entries) in d - dep_name in weakdeps || continue - entries::Vector{Any} - if length(entries) != 1 - error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") - end - entry = first(entries)::Dict{String, Any} - uuid = entry["uuid"]::String - d_weakdeps[dep_name] = uuid + if weakdeps isa Vector{String} + weakdeps = expand_deps_list(weakdeps) + end + if deps isa Vector{String} + deps = expand_deps_list(deps) end - @assert length(d_weakdeps) == length(weakdeps) - return _insert_extension_triggers(pkg, extensions, d_weakdeps) + + total_deps = merge(weakdeps, deps) + return _insert_extension_triggers(pkg, extensions, total_deps) end end end @@ -1454,7 +1461,7 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi return nothing end -function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any}) +function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, totaldeps::Dict{String, Any}) for (ext, triggers) in extensions triggers = triggers::Union{String, Vector{String}} triggers isa String && (triggers = [triggers]) @@ -1468,7 +1475,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} push!(trigger1, gid) for trigger in triggers # TODO: Better error message if this lookup fails? - uuid_trigger = UUID(weakdeps[trigger]::String) + uuid_trigger = UUID(totaldeps[trigger]::String) trigger_id = PkgId(uuid_trigger, trigger) if !haskey(explicit_loaded_modules, trigger_id) || haskey(package_locks, trigger_id) trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id) diff --git a/base/precompilation.jl b/base/precompilation.jl index 6cf421850a17d..c677b20a4e68a 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -401,7 +401,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; all_extdeps_available = true for extdep_uuid in extdep_uuids extdep_name = env.names[extdep_uuid] - if extdep_uuid in keys(env.deps) || Base.in_sysimage(Base.PkgId(extdep_uuid, extdep_name)) + if extdep_uuid in keys(env.deps) push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name)) else all_extdeps_available = false diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 25ebf3475f34c..564ac4679963a 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -351,7 +351,7 @@ Since the primary environment is typically the environment of a project you're w ### [Package Extensions](@id man-extensions) -A package "extension" is a module that is automatically loaded when a specified set of other packages (its "extension dependencies") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The extension dependencies of an extension are a subset of those packages listed under the `[weakdeps]` section of the project file. Those packages can have compat entries like other packages. +A package "extension" is a module that is automatically loaded when a specified set of other packages (its "triggers") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The triggers of an extension are a subset of those packages listed under the `[weakdeps]` (and possibly, but uncommonly the `[deps]`) section of the project file. Those packages can have compat entries like other packages. ```toml name = "MyPackage" @@ -371,8 +371,8 @@ FooExt = "ExtDep" ``` The keys under `extensions` are the names of the extensions. -They are loaded when all the packages on the right hand side (the extension dependencies) of that extension are loaded. -If an extension only has one extension dependency the list of extension dependencies can be written as just a string for brevity. +They are loaded when all the packages on the right hand side (the triggers) of that extension are loaded. +If an extension only has one trigger the list of triggers can be written as just a string for brevity. The location for the entry point of the extension is either in `ext/FooExt.jl` or `ext/FooExt/FooExt.jl` for extension `FooExt`. The content of an extension is often structured as: @@ -380,10 +380,10 @@ The content of an extension is often structured as: ``` module FooExt -# Load main package and extension dependencies +# Load main package and triggers using MyPackage, ExtDep -# Extend functionality in main package with types from the extension dependencies +# Extend functionality in main package with types from the triggers MyPackage.func(x::ExtDep.SomeStruct) = ... end @@ -391,7 +391,7 @@ end When a package with extensions is added to an environment, the `weakdeps` and `extensions` sections are stored in the manifest file in the section for that package. The dependency lookup rules for -a package are the same as for its "parent" except that the listed extension dependencies are also considered as +a package are the same as for its "parent" except that the listed triggers are also considered as dependencies. ### [Package/Environment Preferences](@id preferences) diff --git a/test/loading.jl b/test/loading.jl index 71d0b07b64376..8d0b76caee998 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1070,6 +1070,9 @@ end using ExtDep2 $ew using ExtDep2 $ew HasExtensions.ext_folder_loaded || error("ext_folder_loaded not set") + using ExtDep3 + $ew using ExtDep3 + $ew HasExtensions.ext_dep_loaded || error("ext_dep_loaded not set") end """ return `$(Base.julia_cmd()) $compile --startup-file=no -e $cmd` @@ -1118,6 +1121,8 @@ end test_ext(HasExtensions, :Extension) using ExtDep2 test_ext(HasExtensions, :ExtensionFolder) + using ExtDep3 + test_ext(HasExtensions, :ExtensionDep) end """ for compile in (`--compiled-modules=no`, ``) diff --git a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml index 8ac961fa1a9a9..004ef7892c173 100644 --- a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml +++ b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.0-beta4" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "caa716752e6dff3d77c3de929ebbb5d2024d04ef" +project_hash = "a4c480cfa7da9610333d5c42623bf746bd286c5f" [[deps.ExtDep]] deps = ["SomePackage"] @@ -18,10 +18,12 @@ version = "0.1.0" [deps.HasExtensions.extensions] Extension = "ExtDep" ExtensionFolder = ["ExtDep", "ExtDep2"] + LinearAlgebraExt = "LinearAlgebra" [deps.HasExtensions.weakdeps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.SomePackage]] path = "../SomePackage" diff --git a/test/project/Extensions/ExtDep3.jl/Project.toml b/test/project/Extensions/ExtDep3.jl/Project.toml new file mode 100644 index 0000000000000..690b2f1cffff4 --- /dev/null +++ b/test/project/Extensions/ExtDep3.jl/Project.toml @@ -0,0 +1,4 @@ +name = "ExtDep3" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" +authors = ["Kristoffer "] diff --git a/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl b/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl new file mode 100644 index 0000000000000..96a0b472d06c5 --- /dev/null +++ b/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl @@ -0,0 +1,5 @@ +module ExtDep3 + +greet() = print("Hello World!") + +end # module ExtDep3 diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml index 15f068f250ce3..5706aba59d1e0 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.0" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "d523b3401f72a1ed34b7b43749fd2655c6b78542" +project_hash = "4e196b07f2ee7adc48ac9d528d42b3cf3737c7a0" [[deps.ExtDep]] deps = ["SomePackage"] @@ -15,13 +15,20 @@ path = "../ExtDep2" uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" version = "0.1.0" +[[deps.ExtDep3]] +path = "../ExtDep3.jl" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" + [[deps.HasExtensions]] +deps = ["ExtDep3"] path = "../HasExtensions.jl" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" [deps.HasExtensions.extensions] Extension = "ExtDep" + ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] LinearAlgebraExt = "LinearAlgebra" diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Project.toml b/test/project/Extensions/HasDepWithExtensions.jl/Project.toml index 8f308a9fbee72..aa4956caada74 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Project.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Project.toml @@ -5,4 +5,5 @@ version = "0.1.0" [deps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" +ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1" HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" diff --git a/test/project/Extensions/HasExtensions.jl/Manifest.toml b/test/project/Extensions/HasExtensions.jl/Manifest.toml index 55f7958701a75..429c6598fc4f4 100644 --- a/test/project/Extensions/HasExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasExtensions.jl/Manifest.toml @@ -1,7 +1,10 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.0-DEV" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "c87947f1f1f070eea848950c304d668a112dec3d" +project_hash = "c0bb526b75939a74a6195ee4819e598918a22ad7" -[deps] +[[deps.ExtDep3]] +path = "../ExtDep3.jl" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" diff --git a/test/project/Extensions/HasExtensions.jl/Project.toml b/test/project/Extensions/HasExtensions.jl/Project.toml index a5f9bb1e42d29..fe21a1423f543 100644 --- a/test/project/Extensions/HasExtensions.jl/Project.toml +++ b/test/project/Extensions/HasExtensions.jl/Project.toml @@ -2,6 +2,9 @@ name = "HasExtensions" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" +[deps] +ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1" + [weakdeps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" @@ -9,5 +12,6 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [extensions] Extension = "ExtDep" +ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] LinearAlgebraExt = "LinearAlgebra" diff --git a/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl b/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl new file mode 100644 index 0000000000000..e2710d4d89bbb --- /dev/null +++ b/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl @@ -0,0 +1,9 @@ +module ExtensionDep + +using HasExtensions, ExtDep3 + +function __init__() + HasExtensions.ext_dep_loaded = true +end + +end diff --git a/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl b/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl index dbfaeec4f8812..9d9785f87f790 100644 --- a/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl +++ b/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl @@ -6,5 +6,6 @@ foo(::HasExtensionsStruct) = 1 ext_loaded = false ext_folder_loaded = false +ext_dep_loaded = false end # module