Merge pull request #221 from supabase/support-import-maps-for-eszip-b…

…undle

feat: add support for import maps in generated eszips

Cargo.toml

@@ -60,9 +60,9 @@ tokio-util = "0.7.4"
 uuid = { version = "1.3.0", features = ["v4"] }
 rsa = { version = "0.7.0", default-features = false, features = ["std", "pem", "hazmat"] }
 monch = "=0.4.3"
-
 reqwest = { version = "0.11.20", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] }
 ring = "=0.16.20"
+urlencoding = { version = "2.1.2" }
 
 [profile.release]
 lto = true

crates/base/Cargo.toml

@@ -48,12 +48,12 @@ sb_os = { version = "0.1.0", path = "../sb_os" }
 sb_npm = { version = "0.1.0", path = "../npm" }
 sb_graph = { version = "0.1.0", path = "../sb_graph" }
 sb_module_loader = { version = "0.1.0", path = "../sb_module_loader" }
-urlencoding = { version = "2.1.2" }
 uuid = { workspace = true }
 deno_broadcast_channel.workspace = true
 sb_node = { version = "0.1.0", path = "../node" }
 eszip.workspace = true
 notify = { version = "6.1.1", default-features = false, features = ["macos_kqueue"] }
+urlencoding.workspace = true
 
 [dev-dependencies]
 futures-util = { version = "0.3.28" }
@@ -79,7 +79,6 @@ deno_websocket = { workspace = true }
 httparse = { version = "1.8.0" }
 hyper = { version = "0.14.26", features = ["full"] }
 http = { version = "0.2" }
-import_map = { version = "0.15.0" }
 log = { workspace = true }
 module_fetcher = { path = "../module_fetcher" }
 reqwest.workspace = true
@@ -92,4 +91,4 @@ sb_env = { version = "0.1.0", path = "../sb_env" }
 sb_core = { version = "0.1.0", path = "../sb_core" }
 sb_os = { version = "0.1.0", path = "../sb_os" }
 sb_node = { version = "0.1.0", path = "../node" }
-deno_broadcast_channel.workspace = true
+deno_broadcast_channel.workspace = true

crates/base/src/deno_runtime.rs

@@ -9,24 +9,20 @@ use deno_tls::rustls;
 use deno_tls::rustls::RootCertStore;
 use deno_tls::rustls_native_certs::load_native_certs;
 use deno_tls::RootCertStoreProvider;
-use import_map::{parse_from_json, ImportMap};
 use log::error;
 use serde::de::DeserializeOwned;
 use std::collections::HashMap;
-use std::path::Path;
+use std::fmt;
 use std::sync::Arc;
 use std::time::Duration;
-use std::{fmt, fs};
 use tokio::net::UnixStream;
 use tokio::sync::mpsc;
-use urlencoding::decode;
 
 use crate::snapshot;
 use event_worker::events::{EventMetadata, WorkerEventWithMetadata};
 use event_worker::js_interceptors::sb_events_js_interceptors;
 use event_worker::sb_user_event_worker;
 use module_fetcher::file_fetcher::CacheSetting;
-use module_fetcher::util::diagnostic::print_import_map_diagnostics;
 use sb_core::cert::ValueRootCertStoreProvider;
 use sb_core::http_start::sb_core_http;
 use sb_core::net::sb_core_net;
@@ -35,43 +31,14 @@ use sb_core::runtime::sb_core_runtime;
 use sb_core::sb_core_main_js;
 use sb_env::sb_env as sb_env_op;
 use sb_graph::emitter::EmitterFactory;
+use sb_graph::import_map::load_import_map;
 use sb_graph::{generate_binary_eszip, EszipPayloadKind};
 use sb_module_loader::standalone::create_module_loader_for_standalone_from_eszip_kind;
 use sb_module_loader::RuntimeProviders;
 use sb_node::deno_node;
 use sb_workers::context::{UserWorkerMsgs, WorkerContextInitOpts, WorkerRuntimeOpts};
 use sb_workers::sb_user_workers;
 
-fn load_import_map(maybe_path: Option<String>) -> Result<Option<ImportMap>, Error> {
-    if let Some(path_str) = maybe_path {
-        let json_str;
-        let base_url;
-
-        // check if the path is a data URI (prefixed with data:)
-        // the data URI takes the following format
-        // data:{encodeURIComponent(mport_map.json)?{encodeURIComponent(base_path)}
-        if path_str.starts_with("data:") {
-            let data_uri = Url::parse(&path_str)?;
-            json_str = decode(data_uri.path())?.into_owned();
-            base_url =
-                Url::from_directory_path(decode(data_uri.query().unwrap_or(""))?.into_owned())
-                    .map_err(|_| anyhow!("invalid import map base url"))?;
-        } else {
-            let path = Path::new(&path_str);
-            let abs_path = std::env::current_dir().map(|p| p.join(path))?;
-            json_str = fs::read_to_string(abs_path.clone())?;
-            base_url = Url::from_directory_path(abs_path.parent().unwrap())
-                .map_err(|_| anyhow!("invalid import map base url"))?;
-        }
-
-        let result = parse_from_json(&base_url, json_str.as_str())?;
-        print_import_map_diagnostics(&result.diagnostics);
-        Ok(Some(result.import_map))
-    } else {
-        Ok(None)
-    }
-}
-
 pub struct DenoRuntimeError(Error);
 
 impl PartialEq for DenoRuntimeError {
@@ -180,9 +147,13 @@ impl DenoRuntime {
                 None
             };
 
-            let eszip =
-                generate_binary_eszip(main_module_url_file_path, arc_emitter_factory, maybe_code)
-                    .await?;
+            let eszip = generate_binary_eszip(
+                main_module_url_file_path,
+                arc_emitter_factory,
+                maybe_code,
+                import_map_path.clone(),
+            )
+            .await?;
 
             EszipPayloadKind::Eszip(eszip)
         };
@@ -491,7 +462,7 @@ mod test {
             .unwrap();
         let path_buf = PathBuf::from("./test_cases/eszip-source-test.ts");
         let emitter_factory = Arc::new(EmitterFactory::new());
-        let bin_eszip = generate_binary_eszip(path_buf, emitter_factory.clone(), None)
+        let bin_eszip = generate_binary_eszip(path_buf, emitter_factory.clone(), None, None)
             .await
             .unwrap();
         fs::remove_file("./test_cases/eszip-source-test.ts").unwrap();
@@ -540,7 +511,7 @@ mod test {
         let file = PathBuf::from("./test_cases/eszip-silly-test/index.ts");
         let service_path = PathBuf::from("./test_cases/eszip-silly-test");
         let emitter_factory = Arc::new(EmitterFactory::new());
-        let binary_eszip = generate_binary_eszip(file, emitter_factory.clone(), None)
+        let binary_eszip = generate_binary_eszip(file, emitter_factory.clone(), None, None)
             .await
             .unwrap();
 

crates/cli/Cargo.toml

@@ -10,9 +10,9 @@ path = "src/main.rs"
 [dependencies]
 anyhow = { workspace = true }
 base = { path = "../base" }
-sb_graph = { path = "../sb_graph" }
+deno_core = { workspace = true }
 clap = { version = "4.0.29", features = ["cargo"] }
 env_logger = "0.10.0"
 log = { workspace = true }
+sb_graph = { path = "../sb_graph" }
 tokio.workspace = true
-

crates/cli/src/main.rs

@@ -1,12 +1,14 @@
 mod logger;
 
-use anyhow::Error;
+use anyhow::{anyhow, Error};
 use base::commands::start_server;
 use base::server::WorkerEntrypoints;
 use clap::builder::FalseyValueParser;
 use clap::{arg, crate_version, value_parser, ArgAction, Command};
+use deno_core::url::Url;
 use sb_graph::emitter::EmitterFactory;
 use sb_graph::generate_binary_eszip;
+use sb_graph::import_map::load_import_map;
 use std::fs::File;
 use std::io::Write;
 use std::path::PathBuf;
@@ -55,6 +57,7 @@ fn cli() -> Command {
                 .about("Creates an 'eszip' file that can be executed by the EdgeRuntime. Such file contains all the modules in contained in a single binary.")
                 .arg(arg!(--"output" <DIR> "Path to output eszip file").default_value("bin.eszip"))
                 .arg(arg!(--"entrypoint" <Path> "Path to entrypoint to bundle as an eszip").required(true))
+                .arg(arg!(--"import-map" <Path> "Path to import map file"))
         )
 }
 
@@ -125,17 +128,33 @@ fn main() -> Result<(), anyhow::Error> {
             }
             Some(("bundle", sub_matches)) => {
                 let output_path = sub_matches.get_one::<String>("output").cloned().unwrap();
+                let import_map_path = sub_matches.get_one::<String>("import-map").cloned();
 
                 let entry_point_path = sub_matches
                     .get_one::<String>("entrypoint")
                     .cloned()
                     .unwrap();
 
                 let path = PathBuf::from(entry_point_path.as_str());
+                let mut emitter_factory = EmitterFactory::new();
+                let maybe_import_map = load_import_map(import_map_path.clone())?;
+                let mut maybe_import_map_url = None;
+                if maybe_import_map.is_some() {
+                    let abs_import_map_path =
+                        std::env::current_dir().map(|p| p.join(import_map_path.unwrap()))?;
+                    maybe_import_map_url = Some(
+                        Url::from_file_path(abs_import_map_path)
+                            .map_err(|_| anyhow!("failed get import map url"))?
+                            .to_string(),
+                    );
+                }
+                emitter_factory.set_import_map(maybe_import_map.clone());
+
                 let eszip = generate_binary_eszip(
                     path.canonicalize().unwrap(),
-                    Arc::new(EmitterFactory::new()),
+                    Arc::new(emitter_factory),
                     None,
+                    maybe_import_map_url,
                 )
                 .await?;
                 let bin = eszip.into_bytes();

crates/sb_graph/Cargo.toml

@@ -27,4 +27,5 @@ deno_ast.workspace = true
 deno_fs.workspace = true
 deno_npm.workspace = true
 once_cell.workspace = true
-deno_web.workspace = true
+deno_web.workspace = true
+urlencoding.workspace = true

crates/sb_graph/import_map.rs

@@ -0,0 +1,37 @@
+use anyhow::{anyhow, Error};
+use deno_core::url::Url;
+use import_map::{parse_from_json, ImportMap};
+use module_fetcher::util::diagnostic::print_import_map_diagnostics;
+use std::fs;
+use std::path::Path;
+use urlencoding::decode;
+
+pub fn load_import_map(maybe_path: Option<String>) -> Result<Option<ImportMap>, Error> {
+    if let Some(path_str) = maybe_path {
+        let json_str;
+        let base_url;
+
+        // check if the path is a data URI (prefixed with data:)
+        // the data URI takes the following format
+        // data:{encodeURIComponent(mport_map.json)?{encodeURIComponent(base_path)}
+        if path_str.starts_with("data:") {
+            let data_uri = Url::parse(&path_str)?;
+            json_str = decode(data_uri.path())?.into_owned();
+            base_url =
+                Url::from_directory_path(decode(data_uri.query().unwrap_or(""))?.into_owned())
+                    .map_err(|_| anyhow!("invalid import map base url"))?;
+        } else {
+            let path = Path::new(&path_str);
+            let abs_path = std::env::current_dir().map(|p| p.join(path))?;
+            json_str = fs::read_to_string(abs_path.clone())?;
+            base_url = Url::from_directory_path(abs_path.parent().unwrap())
+                .map_err(|_| anyhow!("invalid import map base url"))?;
+        }
+
+        let result = parse_from_json(&base_url, json_str.as_str())?;
+        print_import_map_diagnostics(&result.diagnostics);
+        Ok(Some(result.import_map))
+    } else {
+        Ok(None)
+    }
+}

crates/sb_graph/lib.rs

@@ -5,14 +5,15 @@ use deno_core::error::AnyError;
 use deno_core::{serde_json, FastString, JsBuffer, ModuleSpecifier};
 use deno_fs::{FileSystem, RealFs};
 use deno_npm::NpmSystemInfo;
-use eszip::EszipV2;
+use eszip::{EszipV2, ModuleKind};
 use sb_fs::{build_vfs, VfsOpts};
 use std::path::PathBuf;
 use std::sync::Arc;
 
 pub mod emitter;
 pub mod graph_resolver;
 pub mod graph_util;
+pub mod import_map;
 
 pub const VFS_ESZIP_KEY: &str = "---SUPABASE-VFS-DATA-ESZIP---";
 pub const SOURCE_CODE_ESZIP_KEY: &str = "---SUPABASE-SOURCE-CODE-ESZIP---";
@@ -28,6 +29,7 @@ pub async fn generate_binary_eszip(
     file: PathBuf,
     emitter_factory: Arc<EmitterFactory>,
     maybe_module_code: Option<FastString>,
+    maybe_import_map_url: Option<String>,
 ) -> Result<EszipV2, AnyError> {
     let graph = create_graph(file.clone(), emitter_factory.clone(), &maybe_module_code).await;
     let eszip = create_eszip_from_graph_raw(graph, Some(emitter_factory.clone())).await;
@@ -72,6 +74,22 @@ pub async fn generate_binary_eszip(
         eszip.add_opaque_data(String::from(VFS_ESZIP_KEY), Arc::from(boxed_slice));
         eszip.add_opaque_data(String::from(SOURCE_CODE_ESZIP_KEY), bin_code);
 
+        // add import map
+        if emitter_factory.maybe_import_map.is_some() {
+            eszip.add_import_map(
+                ModuleKind::Json,
+                maybe_import_map_url.unwrap(),
+                Arc::from(
+                    emitter_factory
+                        .maybe_import_map
+                        .as_ref()
+                        .unwrap()
+                        .to_json()
+                        .as_bytes(),
+                ),
+            );
+        };
+
         Ok(eszip)
     } else {
         eszip

examples/express/import_map.json

@@ -0,0 +1,7 @@
+{
+  "imports": {
+    "oak": "https://deno.land/x/[email protected]/mod.ts",
+    "shared_cors": "./_shared/cors.ts",
+    "express": "npm:[email protected]"
+  }
+}

examples/express/index.ts

@@ -1,9 +1,9 @@
-import express from "npm:express@4.18.2";
+import express from 'express';
 
 const app = express();
 
-app.get("/", (req, res) => {
-    res.send("Welcome to the Dinosaur API!");
+app.get('/express', (req, res) => {
+	res.send('Welcome to the Dinosaur API!');
 });
 
-app.listen(8000);
+app.listen(8000);

examples/main/index.ts

@@ -47,7 +47,9 @@ serve(async (req: Request) => {
 		const netAccessDisabled = false;
 
 		// load source from an eszip
-		// const maybeEszip = await Deno.readFile('./sample.eszip');
+		//const maybeEszip = await Deno.readFile('./bin.eszip');
+		//const maybeEntrypoint = 'file:///src/index.ts';
+
 		// const maybeEntrypoint = 'file:///src/index.ts';
 		// or load module source from an inline module
 		// const maybeModuleCode = 'Deno.serve((req) => new Response("Hello from Module Code"));';