From c9305336f72eba7b59af75f3c34e4ee0f1208b48 Mon Sep 17 00:00:00 2001 From: Lakshan Perera Date: Fri, 1 Dec 2023 11:18:27 +1100 Subject: [PATCH] feat: add support for import maps in generated eszips --- Cargo.lock | 2 ++ Cargo.toml | 2 +- crates/base/Cargo.toml | 5 ++-- crates/base/src/deno_runtime.rs | 51 +++++++------------------------- crates/cli/Cargo.toml | 4 +-- crates/cli/src/main.rs | 23 ++++++++++++-- crates/sb_graph/Cargo.toml | 3 +- crates/sb_graph/import_map.rs | 37 +++++++++++++++++++++++ crates/sb_graph/lib.rs | 20 ++++++++++++- examples/express/import_map.json | 7 +++++ examples/express/index.ts | 8 ++--- examples/main/index.ts | 4 ++- 12 files changed, 111 insertions(+), 55 deletions(-) create mode 100644 crates/sb_graph/import_map.rs create mode 100644 examples/express/import_map.json diff --git a/Cargo.lock b/Cargo.lock index eda24d68e..3d6188eb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -674,6 +674,7 @@ dependencies = [ "anyhow", "base", "clap 4.3.21", + "deno_core", "env_logger 0.10.0", "log", "sb_graph", @@ -3809,6 +3810,7 @@ dependencies = [ "sb_npm", "serde", "tokio", + "urlencoding", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ab1f56f1b..89f370d1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,9 +60,9 @@ tokio-util = "0.7.4" uuid = { version = "1.3.0", features = ["v4"] } rsa = { version = "0.7.0", default-features = false, features = ["std", "pem", "hazmat"] } monch = "=0.4.3" - reqwest = { version = "0.11.20", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] } ring = "=0.16.20" +urlencoding = { version = "2.1.2" } [profile.release] lto = true diff --git a/crates/base/Cargo.toml b/crates/base/Cargo.toml index 35aaaf803..e96a05c7e 100644 --- a/crates/base/Cargo.toml +++ b/crates/base/Cargo.toml @@ -48,12 +48,12 @@ sb_os = { version = "0.1.0", path = "../sb_os" } sb_npm = { version = "0.1.0", path = "../npm" } sb_graph = { version = "0.1.0", path = "../sb_graph" } sb_module_loader = { version = "0.1.0", path = "../sb_module_loader" } -urlencoding = { version = "2.1.2" } uuid = { workspace = true } deno_broadcast_channel.workspace = true sb_node = { version = "0.1.0", path = "../node" } eszip.workspace = true notify = { version = "6.1.1", default-features = false, features = ["macos_kqueue"] } +urlencoding.workspace = true [dev-dependencies] futures-util = { version = "0.3.28" } @@ -79,7 +79,6 @@ deno_websocket = { workspace = true } httparse = { version = "1.8.0" } hyper = { version = "0.14.26", features = ["full"] } http = { version = "0.2" } -import_map = { version = "0.15.0" } log = { workspace = true } module_fetcher = { path = "../module_fetcher" } reqwest.workspace = true @@ -92,4 +91,4 @@ sb_env = { version = "0.1.0", path = "../sb_env" } sb_core = { version = "0.1.0", path = "../sb_core" } sb_os = { version = "0.1.0", path = "../sb_os" } sb_node = { version = "0.1.0", path = "../node" } -deno_broadcast_channel.workspace = true \ No newline at end of file +deno_broadcast_channel.workspace = true diff --git a/crates/base/src/deno_runtime.rs b/crates/base/src/deno_runtime.rs index c0decbb5a..d0a33ec98 100644 --- a/crates/base/src/deno_runtime.rs +++ b/crates/base/src/deno_runtime.rs @@ -9,24 +9,20 @@ use deno_tls::rustls; use deno_tls::rustls::RootCertStore; use deno_tls::rustls_native_certs::load_native_certs; use deno_tls::RootCertStoreProvider; -use import_map::{parse_from_json, ImportMap}; use log::error; use serde::de::DeserializeOwned; use std::collections::HashMap; -use std::path::Path; +use std::fmt; use std::sync::Arc; use std::time::Duration; -use std::{fmt, fs}; use tokio::net::UnixStream; use tokio::sync::mpsc; -use urlencoding::decode; use crate::snapshot; use event_worker::events::{EventMetadata, WorkerEventWithMetadata}; use event_worker::js_interceptors::sb_events_js_interceptors; use event_worker::sb_user_event_worker; use module_fetcher::file_fetcher::CacheSetting; -use module_fetcher::util::diagnostic::print_import_map_diagnostics; use sb_core::cert::ValueRootCertStoreProvider; use sb_core::http_start::sb_core_http; use sb_core::net::sb_core_net; @@ -35,6 +31,7 @@ use sb_core::runtime::sb_core_runtime; use sb_core::sb_core_main_js; use sb_env::sb_env as sb_env_op; use sb_graph::emitter::EmitterFactory; +use sb_graph::import_map::load_import_map; use sb_graph::{generate_binary_eszip, EszipPayloadKind}; use sb_module_loader::standalone::create_module_loader_for_standalone_from_eszip_kind; use sb_module_loader::RuntimeProviders; @@ -42,36 +39,6 @@ use sb_node::deno_node; use sb_workers::context::{UserWorkerMsgs, WorkerContextInitOpts, WorkerRuntimeOpts}; use sb_workers::sb_user_workers; -fn load_import_map(maybe_path: Option) -> Result, Error> { - if let Some(path_str) = maybe_path { - let json_str; - let base_url; - - // check if the path is a data URI (prefixed with data:) - // the data URI takes the following format - // data:{encodeURIComponent(mport_map.json)?{encodeURIComponent(base_path)} - if path_str.starts_with("data:") { - let data_uri = Url::parse(&path_str)?; - json_str = decode(data_uri.path())?.into_owned(); - base_url = - Url::from_directory_path(decode(data_uri.query().unwrap_or(""))?.into_owned()) - .map_err(|_| anyhow!("invalid import map base url"))?; - } else { - let path = Path::new(&path_str); - let abs_path = std::env::current_dir().map(|p| p.join(path))?; - json_str = fs::read_to_string(abs_path.clone())?; - base_url = Url::from_directory_path(abs_path.parent().unwrap()) - .map_err(|_| anyhow!("invalid import map base url"))?; - } - - let result = parse_from_json(&base_url, json_str.as_str())?; - print_import_map_diagnostics(&result.diagnostics); - Ok(Some(result.import_map)) - } else { - Ok(None) - } -} - pub struct DenoRuntimeError(Error); impl PartialEq for DenoRuntimeError { @@ -180,9 +147,13 @@ impl DenoRuntime { None }; - let eszip = - generate_binary_eszip(main_module_url_file_path, arc_emitter_factory, maybe_code) - .await?; + let eszip = generate_binary_eszip( + main_module_url_file_path, + arc_emitter_factory, + maybe_code, + import_map_path.clone(), + ) + .await?; EszipPayloadKind::Eszip(eszip) }; @@ -491,7 +462,7 @@ mod test { .unwrap(); let path_buf = PathBuf::from("./test_cases/eszip-source-test.ts"); let emitter_factory = Arc::new(EmitterFactory::new()); - let bin_eszip = generate_binary_eszip(path_buf, emitter_factory.clone(), None) + let bin_eszip = generate_binary_eszip(path_buf, emitter_factory.clone(), None, None) .await .unwrap(); fs::remove_file("./test_cases/eszip-source-test.ts").unwrap(); @@ -540,7 +511,7 @@ mod test { let file = PathBuf::from("./test_cases/eszip-silly-test/index.ts"); let service_path = PathBuf::from("./test_cases/eszip-silly-test"); let emitter_factory = Arc::new(EmitterFactory::new()); - let binary_eszip = generate_binary_eszip(file, emitter_factory.clone(), None) + let binary_eszip = generate_binary_eszip(file, emitter_factory.clone(), None, None) .await .unwrap(); diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 9839b7216..ddc31a48e 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -10,9 +10,9 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } base = { path = "../base" } -sb_graph = { path = "../sb_graph" } +deno_core = { workspace = true } clap = { version = "4.0.29", features = ["cargo"] } env_logger = "0.10.0" log = { workspace = true } +sb_graph = { path = "../sb_graph" } tokio.workspace = true - diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index ed7789a73..0c651a854 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,12 +1,14 @@ mod logger; -use anyhow::Error; +use anyhow::{anyhow, Error}; use base::commands::start_server; use base::server::WorkerEntrypoints; use clap::builder::FalseyValueParser; use clap::{arg, crate_version, value_parser, ArgAction, Command}; +use deno_core::url::Url; use sb_graph::emitter::EmitterFactory; use sb_graph::generate_binary_eszip; +use sb_graph::import_map::load_import_map; use std::fs::File; use std::io::Write; use std::path::PathBuf; @@ -55,6 +57,7 @@ fn cli() -> Command { .about("Creates an 'eszip' file that can be executed by the EdgeRuntime. Such file contains all the modules in contained in a single binary.") .arg(arg!(--"output" "Path to output eszip file").default_value("bin.eszip")) .arg(arg!(--"entrypoint" "Path to entrypoint to bundle as an eszip").required(true)) + .arg(arg!(--"import-map" "Path to import map file")) ) } @@ -125,6 +128,7 @@ fn main() -> Result<(), anyhow::Error> { } Some(("bundle", sub_matches)) => { let output_path = sub_matches.get_one::("output").cloned().unwrap(); + let import_map_path = sub_matches.get_one::("import-map").cloned(); let entry_point_path = sub_matches .get_one::("entrypoint") @@ -132,10 +136,25 @@ fn main() -> Result<(), anyhow::Error> { .unwrap(); let path = PathBuf::from(entry_point_path.as_str()); + let mut emitter_factory = EmitterFactory::new(); + let maybe_import_map = load_import_map(import_map_path.clone())?; + let mut maybe_import_map_url = None; + if maybe_import_map.is_some() { + let abs_import_map_path = + std::env::current_dir().map(|p| p.join(import_map_path.unwrap()))?; + maybe_import_map_url = Some( + Url::from_file_path(abs_import_map_path) + .map_err(|_| anyhow!("failed get import map url"))? + .to_string(), + ); + } + emitter_factory.set_import_map(maybe_import_map.clone()); + let eszip = generate_binary_eszip( path.canonicalize().unwrap(), - Arc::new(EmitterFactory::new()), + Arc::new(emitter_factory), None, + maybe_import_map_url, ) .await?; let bin = eszip.into_bytes(); diff --git a/crates/sb_graph/Cargo.toml b/crates/sb_graph/Cargo.toml index 9b47e0ad7..ae1919aa7 100644 --- a/crates/sb_graph/Cargo.toml +++ b/crates/sb_graph/Cargo.toml @@ -27,4 +27,5 @@ deno_ast.workspace = true deno_fs.workspace = true deno_npm.workspace = true once_cell.workspace = true -deno_web.workspace = true \ No newline at end of file +deno_web.workspace = true +urlencoding.workspace = true diff --git a/crates/sb_graph/import_map.rs b/crates/sb_graph/import_map.rs new file mode 100644 index 000000000..688be0443 --- /dev/null +++ b/crates/sb_graph/import_map.rs @@ -0,0 +1,37 @@ +use anyhow::{anyhow, Error}; +use deno_core::url::Url; +use import_map::{parse_from_json, ImportMap}; +use module_fetcher::util::diagnostic::print_import_map_diagnostics; +use std::fs; +use std::path::Path; +use urlencoding::decode; + +pub fn load_import_map(maybe_path: Option) -> Result, Error> { + if let Some(path_str) = maybe_path { + let json_str; + let base_url; + + // check if the path is a data URI (prefixed with data:) + // the data URI takes the following format + // data:{encodeURIComponent(mport_map.json)?{encodeURIComponent(base_path)} + if path_str.starts_with("data:") { + let data_uri = Url::parse(&path_str)?; + json_str = decode(data_uri.path())?.into_owned(); + base_url = + Url::from_directory_path(decode(data_uri.query().unwrap_or(""))?.into_owned()) + .map_err(|_| anyhow!("invalid import map base url"))?; + } else { + let path = Path::new(&path_str); + let abs_path = std::env::current_dir().map(|p| p.join(path))?; + json_str = fs::read_to_string(abs_path.clone())?; + base_url = Url::from_directory_path(abs_path.parent().unwrap()) + .map_err(|_| anyhow!("invalid import map base url"))?; + } + + let result = parse_from_json(&base_url, json_str.as_str())?; + print_import_map_diagnostics(&result.diagnostics); + Ok(Some(result.import_map)) + } else { + Ok(None) + } +} diff --git a/crates/sb_graph/lib.rs b/crates/sb_graph/lib.rs index 236a5e886..acd3fa0db 100644 --- a/crates/sb_graph/lib.rs +++ b/crates/sb_graph/lib.rs @@ -5,7 +5,7 @@ use deno_core::error::AnyError; use deno_core::{serde_json, FastString, JsBuffer, ModuleSpecifier}; use deno_fs::{FileSystem, RealFs}; use deno_npm::NpmSystemInfo; -use eszip::EszipV2; +use eszip::{EszipV2, ModuleKind}; use sb_fs::{build_vfs, VfsOpts}; use std::path::PathBuf; use std::sync::Arc; @@ -13,6 +13,7 @@ use std::sync::Arc; pub mod emitter; pub mod graph_resolver; pub mod graph_util; +pub mod import_map; pub const VFS_ESZIP_KEY: &str = "---SUPABASE-VFS-DATA-ESZIP---"; pub const SOURCE_CODE_ESZIP_KEY: &str = "---SUPABASE-SOURCE-CODE-ESZIP---"; @@ -28,6 +29,7 @@ pub async fn generate_binary_eszip( file: PathBuf, emitter_factory: Arc, maybe_module_code: Option, + maybe_import_map_url: Option, ) -> Result { let graph = create_graph(file.clone(), emitter_factory.clone(), &maybe_module_code).await; let eszip = create_eszip_from_graph_raw(graph, Some(emitter_factory.clone())).await; @@ -72,6 +74,22 @@ pub async fn generate_binary_eszip( eszip.add_opaque_data(String::from(VFS_ESZIP_KEY), Arc::from(boxed_slice)); eszip.add_opaque_data(String::from(SOURCE_CODE_ESZIP_KEY), bin_code); + // add import map + if emitter_factory.maybe_import_map.is_some() { + eszip.add_import_map( + ModuleKind::Json, + maybe_import_map_url.unwrap(), + Arc::from( + emitter_factory + .maybe_import_map + .as_ref() + .unwrap() + .to_json() + .as_bytes(), + ), + ); + }; + Ok(eszip) } else { eszip diff --git a/examples/express/import_map.json b/examples/express/import_map.json new file mode 100644 index 000000000..acaa2cd0d --- /dev/null +++ b/examples/express/import_map.json @@ -0,0 +1,7 @@ +{ + "imports": { + "oak": "https://deno.land/x/oak@v11.1.0/mod.ts", + "shared_cors": "./_shared/cors.ts", + "express": "npm:express@4.18.2" + } +} diff --git a/examples/express/index.ts b/examples/express/index.ts index 5085b6c18..47379db22 100644 --- a/examples/express/index.ts +++ b/examples/express/index.ts @@ -1,9 +1,9 @@ -import express from "npm:express@4.18.2"; +import express from 'express'; const app = express(); -app.get("/", (req, res) => { - res.send("Welcome to the Dinosaur API!"); +app.get('/express', (req, res) => { + res.send('Welcome to the Dinosaur API!'); }); -app.listen(8000); \ No newline at end of file +app.listen(8000); diff --git a/examples/main/index.ts b/examples/main/index.ts index 8adf3d7da..b0eea723f 100644 --- a/examples/main/index.ts +++ b/examples/main/index.ts @@ -47,7 +47,9 @@ serve(async (req: Request) => { const netAccessDisabled = false; // load source from an eszip - // const maybeEszip = await Deno.readFile('./sample.eszip'); + //const maybeEszip = await Deno.readFile('./bin.eszip'); + //const maybeEntrypoint = 'file:///src/index.ts'; + // const maybeEntrypoint = 'file:///src/index.ts'; // or load module source from an inline module // const maybeModuleCode = 'Deno.serve((req) => new Response("Hello from Module Code"));';