diff --git a/Cargo.lock b/Cargo.lock index 3aeb98cc5..89921be50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,6 +325,7 @@ name = "base" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "bytes", "cityhash", "cpu_timer", @@ -338,6 +339,8 @@ dependencies = [ "deno_http", "deno_io", "deno_net", + "deno_npm", + "deno_semver 0.5.1", "deno_tls", "deno_url", "deno_web", @@ -353,15 +356,20 @@ dependencies = [ "import_map", "log", "module_fetcher", + "monch", + "notify", + "once_cell", "reqwest", "sb_core", "sb_env", "sb_eszip", "sb_node", + "sb_npm", "sb_os", "sb_worker_context", "sb_workers", "serde", + "thiserror", "tokio", "url", "urlencoding", @@ -2391,6 +2399,26 @@ dependencies = [ "serde", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.3" @@ -2476,6 +2504,26 @@ dependencies = [ "serde_json", ] +[[package]] +name = "kqueue" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7447f1ca1b7b563588a205fe93dea8df60fd981423a768bc1c0ded35ed147d0c" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy-regex" version = "3.0.2" @@ -2679,6 +2727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] @@ -2784,6 +2833,23 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.4.0", + "filetime", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "walkdir", + "windows-sys", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -3671,6 +3737,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", + "deno_ast", "deno_core", "deno_fetch", "deno_fs", @@ -3771,6 +3838,31 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "sb_npm" +version = "0.1.0" +dependencies = [ + "async-trait", + "base32", + "base64 0.13.1", + "bincode", + "deno_ast", + "deno_core", + "deno_fs", + "deno_lockfile", + "deno_npm", + "deno_semver 0.5.1", + "flate2", + "hex", + "module_fetcher", + "once_cell", + "percent-encoding", + "ring", + "sb_node", + "serde", + "tar", +] + [[package]] name = "sb_os" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1c36b49a2..3e600f1a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,8 @@ members = [ "./crates/sb_os", "./crates/cpu_timer", "./crates/event_worker", - "./crates/sb_eszip" + "./crates/sb_eszip", + "./crates/npm" ] resolver = "2" @@ -48,6 +49,7 @@ bytes = { version = "1.4.0" } once_cell = "1.17.1" thiserror = "1.0.40" deno_lockfile = "0.17.1" +async-trait = "0.1.73" indexmap = { version = "2.0.0", features = ["serde"] } flate2 = "=1.0.26" tar = "=0.4.40" diff --git a/crates/base/Cargo.toml b/crates/base/Cargo.toml index 0ec0cee64..563e5e38b 100644 --- a/crates/base/Cargo.toml +++ b/crates/base/Cargo.toml @@ -6,6 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait.workspace = true +thiserror = "1.0.40" +monch = "=0.4.3" +once_cell.workspace = true +deno_semver.workspace = true +deno_npm.workspace = true cpu_timer = { version = "0.1.0", path = "../cpu_timer" } anyhow = { workspace = true } bytes = { version = "1.2.1" } @@ -41,11 +47,13 @@ sb_env = { version = "0.1.0", path = "../sb_env" } sb_core = { version = "0.1.0", path = "../sb_core" } sb_os = { version = "0.1.0", path = "../sb_os" } sb_eszip = { version = "0.1.0", path = "../sb_eszip" } +sb_npm = { version = "0.1.0", path = "../npm" } urlencoding = { version = "2.1.2" } uuid = { workspace = true } deno_broadcast_channel.workspace = true sb_node = { version = "0.1.0", path = "../node" } eszip.workspace = true +notify = { version = "6.1.1", default-features = false, features = ["macos_kqueue"] } [dev-dependencies] futures-util = { version = "0.3.28" } diff --git a/crates/base/build.rs b/crates/base/build.rs index 5a15c1b71..65a0e2c96 100644 --- a/crates/base/build.rs +++ b/crates/base/build.rs @@ -4,13 +4,9 @@ use std::path::PathBuf; mod supabase_startup_snapshot { use super::*; - use deno_ast::MediaType; - use deno_ast::ParseParams; - use deno_ast::SourceTextInfo; use deno_core::error::AnyError; use deno_core::snapshot_util::*; - use deno_core::ExtensionFileSource; - use deno_core::{Extension, ExtensionFileSourceCode}; + use deno_core::Extension; use deno_fs::OpenOptions; use deno_http::DefaultHttpPropertyExtractor; use event_worker::js_interceptors::sb_events_js_interceptors; @@ -20,6 +16,7 @@ mod supabase_startup_snapshot { use sb_core::permissions::sb_core_permissions; use sb_core::runtime::sb_core_runtime; use sb_core::sb_core_main_js; + use sb_core::transpiler::maybe_transpile_source; use sb_env::sb_env; use sb_node::deno_node; use sb_workers::sb_user_workers; @@ -27,42 +24,6 @@ mod supabase_startup_snapshot { use std::sync::Arc; use url::Url; - fn maybe_transpile_source(source: &mut ExtensionFileSource) -> Result<(), AnyError> { - let media_type = if source.specifier.starts_with("node:") { - MediaType::TypeScript - } else { - MediaType::from_path(Path::new(&source.specifier)) - }; - - match media_type { - MediaType::TypeScript => {} - MediaType::JavaScript => return Ok(()), - MediaType::Mjs => return Ok(()), - _ => panic!( - "Unsupported media type for snapshotting {media_type:?} for file {}", - source.specifier - ), - } - let code = source.load()?; - - let parsed = deno_ast::parse_module(ParseParams { - specifier: source.specifier.to_string(), - text_info: SourceTextInfo::from_string(code.as_str().to_owned()), - media_type, - capture_tokens: false, - scope_analysis: false, - maybe_syntax: None, - })?; - let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { - imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, - inline_source_map: false, - ..Default::default() - })?; - - source.code = ExtensionFileSourceCode::Computed(transpiled_source.text.into()); - Ok(()) - } - #[derive(Clone)] pub struct Permissions; @@ -231,10 +192,10 @@ mod supabase_startup_snapshot { for extension in &mut extensions { for source in extension.esm_files.to_mut() { - maybe_transpile_source(source).unwrap(); + let _ = maybe_transpile_source(source).unwrap(); } for source in extension.js_files.to_mut() { - maybe_transpile_source(source).unwrap(); + let _ = maybe_transpile_source(source).unwrap(); } } diff --git a/crates/base/src/cert.rs b/crates/base/src/cert.rs index 00c1b553f..4bfb17657 100644 --- a/crates/base/src/cert.rs +++ b/crates/base/src/cert.rs @@ -1,6 +1,10 @@ use deno_core::error::AnyError; use deno_tls::rustls::RootCertStore; -use deno_tls::RootCertStoreProvider; +use deno_tls::rustls_native_certs::load_native_certs; +use deno_tls::{rustls, rustls_pemfile, webpki_roots, RootCertStoreProvider}; +use std::io::{BufReader, Cursor}; +use std::path::PathBuf; +use thiserror::Error; pub struct ValueRootCertStoreProvider { pub root_cert_store: RootCertStore, @@ -17,3 +21,101 @@ impl RootCertStoreProvider for ValueRootCertStoreProvider { Ok(&self.root_cert_store) } } + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CaData { + /// The string is a file path + File(String), + /// This variant is not exposed as an option in the CLI, it is used internally + /// for standalone binaries. + Bytes(Vec), +} + +#[derive(Error, Debug, Clone)] +pub enum RootCertStoreLoadError { + #[error("Unknown certificate store \"{0}\" specified (allowed: \"system,mozilla\")")] + UnknownStore(String), + #[error("Unable to add pem file to certificate store: {0}")] + FailedAddPemFile(String), + #[error("Failed opening CA file: {0}")] + CaFileOpenError(String), +} + +pub fn get_root_cert_store( + maybe_root_path: Option, + maybe_ca_stores: Option>, + maybe_ca_data: Option, +) -> Result { + let mut root_cert_store = RootCertStore::empty(); + let ca_stores: Vec = maybe_ca_stores + .or_else(|| { + let env_ca_store = std::env::var("DENO_TLS_CA_STORE").ok()?; + Some( + env_ca_store + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(), + ) + }) + .unwrap_or_else(|| vec!["mozilla".to_string()]); + + for store in ca_stores.iter() { + match store.as_str() { + "mozilla" => { + root_cert_store.add_trust_anchors(webpki_roots::TLS_SERVER_ROOTS.iter().map( + |ta| { + rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( + ta.subject, + ta.spki, + ta.name_constraints, + ) + }, + )); + } + "system" => { + let roots = load_native_certs().expect("could not load platform certs"); + for root in roots { + root_cert_store + .add(&rustls::Certificate(root.0)) + .expect("Failed to add platform cert to root cert store"); + } + } + _ => { + return Err(RootCertStoreLoadError::UnknownStore(store.clone())); + } + } + } + + let ca_data = maybe_ca_data.or_else(|| std::env::var("DENO_CERT").ok().map(CaData::File)); + if let Some(ca_data) = ca_data { + let result = match ca_data { + CaData::File(ca_file) => { + let ca_file = if let Some(root) = &maybe_root_path { + root.join(&ca_file) + } else { + PathBuf::from(ca_file) + }; + let certfile = std::fs::File::open(ca_file) + .map_err(|err| RootCertStoreLoadError::CaFileOpenError(err.to_string()))?; + let mut reader = BufReader::new(certfile); + rustls_pemfile::certs(&mut reader) + } + CaData::Bytes(data) => { + let mut reader = BufReader::new(Cursor::new(data)); + rustls_pemfile::certs(&mut reader) + } + }; + + match result { + Ok(certs) => { + root_cert_store.add_parsable_certificates(&certs); + } + Err(e) => { + return Err(RootCertStoreLoadError::FailedAddPemFile(e.to_string())); + } + } + } + + Ok(root_cert_store) +} diff --git a/crates/base/src/deno_runtime.rs b/crates/base/src/deno_runtime.rs index 8813d27d8..087a5d04b 100644 --- a/crates/base/src/deno_runtime.rs +++ b/crates/base/src/deno_runtime.rs @@ -4,7 +4,9 @@ use crate::js_worker::module_loader; use anyhow::{anyhow, bail, Error}; use deno_core::error::AnyError; use deno_core::url::Url; -use deno_core::{located_script_name, serde_v8, JsRuntime, ModuleCode, ModuleId, RuntimeOptions}; +use deno_core::{ + located_script_name, serde_v8, JsRuntime, ModuleCode, ModuleId, ModuleLoader, RuntimeOptions, +}; use deno_http::DefaultHttpPropertyExtractor; use deno_tls::rustls; use deno_tls::rustls::RootCertStore; @@ -14,7 +16,7 @@ use import_map::{parse_from_json, ImportMap}; use log::error; use serde::de::DeserializeOwned; use std::collections::HashMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::rc::Rc; use std::sync::Arc; use std::time::Duration; @@ -25,6 +27,7 @@ use urlencoding::decode; use crate::cert::ValueRootCertStoreProvider; use crate::js_worker::emitter::EmitterFactory; +use crate::js_worker::standalone::create_module_loader_for_standalone_from_eszip_kind; use crate::{errors_rt, snapshot}; use event_worker::events::{EventMetadata, WorkerEventWithMetadata}; use event_worker::js_interceptors::sb_events_js_interceptors; @@ -37,8 +40,8 @@ use sb_core::permissions::{sb_core_permissions, Permissions}; use sb_core::runtime::sb_core_runtime; use sb_core::sb_core_main_js; use sb_env::sb_env as sb_env_op; -use sb_eszip::module_loader::EszipModuleLoader; use sb_node::deno_node; +use sb_npm::CliNpmResolver; use sb_worker_context::essentials::{UserWorkerMsgs, WorkerContextInitOpts, WorkerRuntimeOpts}; use sb_workers::sb_user_workers; @@ -104,6 +107,12 @@ fn set_v8_flags() { ); } +pub struct RuntimeProviders { + pub npm_resolver: Arc, + pub module_loader: Rc, + pub fs: Arc, +} + pub struct DenoRuntime { pub js_runtime: JsRuntime, pub env_vars: HashMap, // TODO: does this need to be pub? @@ -181,12 +190,8 @@ impl DenoRuntime { let mut net_access_disabled = false; let mut allow_remote_modules = true; - let mut module_root_path = base_dir_path.clone(); if conf.is_user_worker() { let user_conf = conf.as_user_worker().unwrap(); - if let Some(custom_module_root) = &user_conf.custom_module_root { - module_root_path = PathBuf::from(custom_module_root); - } net_access_disabled = user_conf.net_access_disabled; allow_remote_modules = user_conf.allow_remote_modules; } @@ -202,7 +207,41 @@ impl DenoRuntime { stderr: deno_io::StdioPipe::File(std::fs::File::create("/dev/null")?), }); } + let mut emitter_factory = EmitterFactory::new(); + emitter_factory.init_npm().await; + let fs = Arc::new(deno_fs::RealFs); + + let import_map = load_import_map(import_map_path)?; + + let rt_providers = if maybe_eszip.is_some() { + create_module_loader_for_standalone_from_eszip_kind(maybe_eszip.unwrap(), import_map) + .await + } else { + let npm_resolver = emitter_factory.npm_resolver().clone(); + + let default_module_loader = DefaultModuleLoader::new( + main_module_url.clone(), + import_map, + emitter_factory, + no_module_cache, + allow_remote_modules, + ) + .await?; + + RuntimeProviders { + npm_resolver, + fs: fs.clone(), + module_loader: Rc::new(default_module_loader), + } + }; + + let RuntimeProviders { + npm_resolver, + fs: file_system, + module_loader, + } = rt_providers; + let extensions = vec![ sb_core_permissions::init_ops(net_access_disabled), deno_webidl::deno_webidl::init_ops(), @@ -240,11 +279,11 @@ impl DenoRuntime { sb_core_main_js::init_ops(), sb_core_net::init_ops(), sb_core_http::init_ops(), - deno_node::init_ops::(None, fs), + deno_node::init_ops::(Some(npm_resolver), file_system), sb_core_runtime::init_ops(Some(main_module_url.clone())), ]; - let mut runtime_options = RuntimeOptions { + let runtime_options = RuntimeOptions { extensions, is_main: true, create_params: { @@ -261,25 +300,10 @@ impl DenoRuntime { shared_array_buffer_store: None, compiled_wasm_module_store: Default::default(), startup_snapshot: Some(snapshot::snapshot()), + module_loader: Some(module_loader), ..Default::default() }; - if maybe_eszip.is_some() { - let eszip_module_loader = - EszipModuleLoader::new(maybe_eszip.unwrap(), import_map_path).await?; - runtime_options.module_loader = Some(Rc::new(eszip_module_loader)); - } else { - let import_map = load_import_map(import_map_path)?; - let emitter = EmitterFactory::new(); - let default_module_loader = DefaultModuleLoader::new( - module_root_path, - import_map, - emitter.emitter().unwrap(), - no_module_cache, - allow_remote_modules, - )?; - runtime_options.module_loader = Some(Rc::new(default_module_loader)); - } let mut js_runtime = JsRuntime::new(runtime_options); let version: Option<&str> = option_env!("GIT_V_TAG"); @@ -403,8 +427,8 @@ impl DenoRuntime { mod test { use crate::deno_runtime::DenoRuntime; use crate::js_worker::emitter::EmitterFactory; - use crate::utils::graph_util::create_graph_and_maybe_check; - use deno_core::{ModuleCode, ModuleSpecifier}; + use crate::standalone::binary::generate_binary_eszip; + use deno_core::ModuleCode; use sb_eszip::module_loader::EszipPayloadKind; use sb_worker_context::essentials::{ MainWorkerRuntimeOpts, UserWorkerMsgs, UserWorkerRuntimeOpts, WorkerContextInitOpts, @@ -412,27 +436,22 @@ mod test { }; use std::collections::HashMap; use std::path::PathBuf; + use std::sync::Arc; use tokio::net::UnixStream; use tokio::sync::mpsc; #[tokio::test] + #[allow(clippy::arc_with_non_send_sync)] async fn test_create_eszip_from_graph() { let (worker_pool_tx, _) = mpsc::unbounded_channel::(); let file = PathBuf::from("./test_cases/eszip-silly-test/index.ts"); let service_path = PathBuf::from("./test_cases/eszip-silly-test"); - let binding = std::fs::canonicalize(&file).unwrap(); - let specifier = binding.to_str().unwrap(); - let format_specifier = format!("file:///{}", specifier); - let module_specifier = ModuleSpecifier::parse(&format_specifier).unwrap(); - let create_module_graph_task = create_graph_and_maybe_check(vec![module_specifier]); - let graph = create_module_graph_task.await.unwrap(); - - let emitter = EmitterFactory::new(); - let parser_arc = emitter.parsed_source_cache().unwrap(); - let parser = parser_arc.as_capturing_parser(); - - let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default()); - let eszip_code = eszip.unwrap().into_bytes(); + let emitter_factory = Arc::new(EmitterFactory::new()); + let binary_eszip = generate_binary_eszip(file, emitter_factory.clone()) + .await + .unwrap(); + + let eszip_code = binary_eszip.into_bytes(); let runtime = DenoRuntime::new(WorkerContextInitOpts { service_path, diff --git a/crates/base/src/js_worker/emitter.rs b/crates/base/src/js_worker/emitter.rs index 35c72841d..8f72068c6 100644 --- a/crates/base/src/js_worker/emitter.rs +++ b/crates/base/src/js_worker/emitter.rs @@ -1,18 +1,99 @@ use crate::js_worker::module_loader::make_http_client; -use crate::utils::graph_resolver::CliGraphResolver; +use crate::js_worker::node_module_loader::{CjsResolutionStore, NpmModuleLoader}; +use crate::utils::graph_resolver::{CliGraphResolver, CliGraphResolverOptions}; use deno_ast::EmitOptions; use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmSystemInfo; use eszip::deno_graph::source::{Loader, Resolver}; +use import_map::ImportMap; +use module_fetcher::args::lockfile::{snapshot_from_lockfile, Lockfile}; +use module_fetcher::args::package_json::{ + get_local_package_json_version_reqs, PackageJsonDeps, PackageJsonDepsProvider, +}; use module_fetcher::args::CacheSetting; -use module_fetcher::cache::{Caches, DenoDir, DenoDirProvider, EmitCache, ParsedSourceCache}; +use module_fetcher::cache::{ + Caches, DenoDir, DenoDirProvider, EmitCache, GlobalHttpCache, NodeAnalysisCache, + ParsedSourceCache, RealDenoCacheEnv, +}; use module_fetcher::emit::Emitter; use module_fetcher::file_fetcher::FileFetcher; +use module_fetcher::http_util::HttpClient; +use module_fetcher::node::CliCjsCodeAnalyzer; use module_fetcher::permissions::Permissions; +use sb_node::analyze::NodeCodeTranslator; +use sb_node::{NodeResolver, PackageJson}; +use sb_npm::{ + create_npm_fs_resolver, CliNpmRegistryApi, CliNpmResolver, NpmCache, NpmCacheDir, + NpmPackageFsResolver, NpmResolution, PackageJsonDepsInstaller, +}; use std::collections::HashMap; +use std::future::Future; +use std::path::PathBuf; use std::sync::Arc; +struct Deferred(once_cell::unsync::OnceCell); + +impl Default for Deferred { + fn default() -> Self { + Self(once_cell::unsync::OnceCell::default()) + } +} + +impl Deferred { + #[allow(dead_code)] + pub fn get_or_try_init( + &self, + create: impl FnOnce() -> Result, + ) -> Result<&T, AnyError> { + self.0.get_or_try_init(create) + } + + pub fn get_or_init(&self, create: impl FnOnce() -> T) -> &T { + self.0.get_or_init(create) + } + + #[allow(dead_code)] + pub async fn get_or_try_init_async( + &self, + create: impl Future>, + ) -> Result<&T, AnyError> { + if self.0.get().is_none() { + // todo(dsherret): it would be more ideal if this enforced a + // single executor and then we could make some initialization + // concurrent + let val = create.await?; + _ = self.0.set(val); + } + Ok(self.0.get().unwrap()) + } +} + +#[derive(Clone)] +pub struct LockfileOpts { + path: PathBuf, + overwrite: bool, +} + pub struct EmitterFactory { deno_dir: DenoDir, + pub npm_snapshot: Option, + lockfile: Deferred>>>, + cjs_resolutions: Deferred>, + package_json_deps_provider: Deferred>, + package_json_deps_installer: Deferred>, + npm_api: Deferred>, + npm_cache: Deferred>, + npm_resolution: Deferred>, + node_resolver: Deferred>, + maybe_package_json_deps: Option, + maybe_lockfile: Option, + npm_resolver: Deferred>, + resolver: Deferred>, + file_fetcher_cache_strategy: Option, + file_fetcher_allow_remote: bool, + maybe_import_map: Option>, } impl Default for EmitterFactory { @@ -24,7 +105,66 @@ impl Default for EmitterFactory { impl EmitterFactory { pub fn new() -> Self { let deno_dir = DenoDir::new(None).unwrap(); - Self { deno_dir } + + Self { + deno_dir, + npm_snapshot: None, + lockfile: Default::default(), + cjs_resolutions: Default::default(), + package_json_deps_provider: Default::default(), + package_json_deps_installer: Default::default(), + npm_api: Default::default(), + npm_cache: Default::default(), + node_resolver: Default::default(), + npm_resolution: Default::default(), + maybe_package_json_deps: None, + maybe_lockfile: None, + npm_resolver: Default::default(), + resolver: Default::default(), + file_fetcher_cache_strategy: None, + file_fetcher_allow_remote: true, + maybe_import_map: None, + } + } + + pub fn set_file_fetcher_cache_strategy(&mut self, strategy: CacheSetting) { + self.file_fetcher_cache_strategy = Some(strategy); + } + + pub fn set_file_fetcher_allow_remote(&mut self, allow_remote: bool) { + self.file_fetcher_allow_remote = allow_remote; + } + + pub fn set_import_map(&mut self, import_map: Option) { + self.maybe_import_map = import_map + .map(|import_map| Some(Arc::new(import_map))) + .unwrap_or_else(|| None); + } + + pub async fn init_npm(&mut self) { + let _init_lock_file = self.get_lock_file(); + + self.npm_snapshot_from_lockfile().await; + } + + pub async fn npm_snapshot_from_lockfile(&mut self) { + if let Some(lockfile) = self.get_lock_file().clone() { + let npm_api = self.npm_api(); + let snapshot = snapshot_from_lockfile(lockfile, &*npm_api.clone()) + .await + .unwrap(); + self.npm_snapshot = Some(snapshot); + } else { + panic!("Lockfile not available"); + } + } + + pub fn init_package_json_deps(&mut self, package: &PackageJson) { + self.maybe_package_json_deps = Some(get_local_package_json_version_reqs(package)); + } + + pub fn set_npm_snapshot(&mut self, npm_snapshot: Option) { + self.npm_snapshot = npm_snapshot; } pub fn deno_dir_provider(&self) -> Arc { @@ -70,18 +210,178 @@ impl EmitterFactory { Box::::default() } + pub fn global_http_cache(&self) -> GlobalHttpCache { + GlobalHttpCache::new(self.deno_dir.deps_folder_path(), RealDenoCacheEnv) + } + + pub fn http_client(&self) -> Arc { + Arc::new(make_http_client().unwrap()) + } + + pub fn real_fs(&self) -> Arc { + Arc::new(deno_fs::RealFs) + } + + pub fn npm_cache(&self) -> &Arc { + self.npm_cache.get_or_init(|| { + Arc::new(NpmCache::new( + NpmCacheDir::new(self.deno_dir.npm_folder_path().clone()), + CacheSetting::Use, // TODO: Maybe ?, + self.real_fs(), + self.http_client(), + )) + }) + } + + pub fn npm_api(&self) -> &Arc { + self.npm_api.get_or_init(|| { + Arc::new(CliNpmRegistryApi::new( + CliNpmRegistryApi::default_url().to_owned(), + self.npm_cache().clone(), + self.http_client(), + )) + }) + } + + pub fn npm_resolution(&self) -> &Arc { + self.npm_resolution.get_or_init(|| { + let npm_registry_api = self.npm_api(); + Arc::new(NpmResolution::from_serialized( + npm_registry_api.clone(), + self.npm_snapshot.clone(), + self.get_lock_file(), + )) + }) + } + + pub fn get_lock_file_deferred(&self) -> &Option>> { + self.lockfile.get_or_init(|| { + if let Some(lockfile_data) = self.maybe_lockfile.clone() { + Some(Arc::new(Mutex::new( + Lockfile::new(lockfile_data.path.clone(), lockfile_data.overwrite).unwrap(), + ))) + } else { + let default_lockfile_path = std::env::current_dir() + .map(|p| p.join(".supabase.lock")) + .unwrap(); + Some(Arc::new(Mutex::new( + Lockfile::new(default_lockfile_path, true).unwrap(), + ))) + } + }) + } + + pub fn get_lock_file(&self) -> Option>> { + self.get_lock_file_deferred().as_ref().cloned() + } + + pub fn node_resolver(&self) -> &Arc { + self.node_resolver.get_or_init(|| { + let fs = self.real_fs().clone(); + Arc::new(NodeResolver::new(fs, self.npm_resolver().clone())) + }) + } + + pub fn cjs_resolution_store(&self) -> &Arc { + self.cjs_resolutions.get_or_init(Default::default) + } + + pub fn npm_module_loader(&self) -> Arc { + let cache_db = Caches::new(self.deno_dir_provider()); + let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); + let cjs_esm_code_analyzer = + CliCjsCodeAnalyzer::new(node_analysis_cache, self.real_fs().clone()); + let node_code_translator = Arc::new(NodeCodeTranslator::new( + cjs_esm_code_analyzer, + self.real_fs().clone(), + self.node_resolver().clone(), + self.npm_resolver().clone(), + )); + + Arc::new(NpmModuleLoader::new( + self.cjs_resolution_store().clone(), + node_code_translator, + self.real_fs(), + self.node_resolver().clone(), + )) + } + + pub fn npm_fs(&self) -> Arc { + let fs = self.real_fs(); + create_npm_fs_resolver( + fs.clone(), + self.npm_cache().clone(), + CliNpmRegistryApi::default_url().to_owned(), + self.npm_resolution().clone(), + None, + NpmSystemInfo::default(), + ) + } + + pub fn npm_resolver(&self) -> &Arc { + let npm_resolution = self.npm_resolution(); + let npm_fs_resolver = self.npm_fs(); + self.npm_resolver.get_or_init(|| { + Arc::new(CliNpmResolver::new( + self.real_fs(), + npm_resolution.clone(), + npm_fs_resolver, + self.get_lock_file(), + )) + }) + } + + pub fn package_json_deps_provider(&self) -> &Arc { + self.package_json_deps_provider.get_or_init(|| { + Arc::new(PackageJsonDepsProvider::new( + self.maybe_package_json_deps.clone(), + )) + }) + } + + pub fn package_json_deps_installer(&self) -> &Arc { + self.package_json_deps_installer.get_or_init(|| { + Arc::new(PackageJsonDepsInstaller::new( + self.package_json_deps_provider().clone(), + self.npm_api().clone(), + self.npm_resolution().clone(), + )) + }) + } + + pub fn cli_graph_resolver_options(&self) -> CliGraphResolverOptions { + CliGraphResolverOptions { + maybe_import_map: self.maybe_import_map.clone(), + ..Default::default() + } + } + + pub fn cli_graph_resolver(&self) -> &Arc { + self.resolver.get_or_init(|| { + Arc::new(CliGraphResolver::new( + self.npm_api().clone(), + self.npm_resolution().clone(), + self.package_json_deps_provider().clone(), + self.package_json_deps_installer().clone(), + self.cli_graph_resolver_options(), + )) + }) + } + pub fn file_fetcher(&self) -> FileFetcher { use module_fetcher::cache::*; let global_cache_struct = GlobalHttpCache::new(self.deno_dir.deps_folder_path(), RealDenoCacheEnv); let global_cache: Arc = Arc::new(global_cache_struct); - let http_client = Arc::new(make_http_client().unwrap()); + let http_client = self.http_client(); let blob_store = Arc::new(deno_web::BlobStore::default()); FileFetcher::new( global_cache.clone(), - CacheSetting::ReloadAll, // TODO: Maybe ? - true, + self.file_fetcher_cache_strategy + .clone() + .unwrap_or(CacheSetting::ReloadAll), + self.file_fetcher_allow_remote, http_client, blob_store, ) diff --git a/crates/base/src/js_worker/mod.rs b/crates/base/src/js_worker/mod.rs index b67e7a5e1..35507de7e 100644 --- a/crates/base/src/js_worker/mod.rs +++ b/crates/base/src/js_worker/mod.rs @@ -1,2 +1,4 @@ pub mod emitter; pub mod module_loader; +pub mod node_module_loader; +pub mod standalone; diff --git a/crates/base/src/js_worker/module_loader.rs b/crates/base/src/js_worker/module_loader.rs index 1ebf91507..a231dff18 100644 --- a/crates/base/src/js_worker/module_loader.rs +++ b/crates/base/src/js_worker/module_loader.rs @@ -1,42 +1,27 @@ -use anyhow::{anyhow, bail, Error}; +use crate::js_worker::emitter::EmitterFactory; +use crate::js_worker::node_module_loader::ModuleCodeSource; +use crate::utils::graph_util::{create_graph, create_graph_from_specifiers}; +use anyhow::{anyhow, Context, Error}; use deno_ast::MediaType; -use deno_core::error::AnyError; +use deno_core::error::{custom_error, AnyError}; +use deno_core::futures::Future; use deno_core::futures::FutureExt; -use deno_core::ModuleLoader; use deno_core::ModuleSource; use deno_core::ModuleSourceFuture; use deno_core::ModuleSpecifier; use deno_core::ModuleType; use deno_core::ResolutionKind; +use deno_core::{ModuleCode, ModuleLoader}; +use eszip::deno_graph; +use eszip::deno_graph::source::Resolver; +use eszip::deno_graph::{EsmModule, JsonModule, Module, ModuleGraph, Resolution}; use import_map::ImportMap; -use module_fetcher::cache::{DenoDir, GlobalHttpCache, HttpCache}; -use module_fetcher::emit::Emitter; -use module_fetcher::file_fetcher::{CacheSetting, FileFetcher}; +use module_fetcher::file_fetcher::CacheSetting; use module_fetcher::http_util::HttpClient; -use std::path::{Path, PathBuf}; +use module_fetcher::node; +use module_fetcher::util::text_encoding::code_without_source_map; use std::pin::Pin; use std::sync::Arc; -use url::Url; - -fn get_module_type(media_type: MediaType) -> Result { - let module_type = match media_type { - MediaType::JavaScript | MediaType::Mjs | MediaType::Cjs | MediaType::Unknown => { - ModuleType::JavaScript - } - MediaType::Jsx => ModuleType::JavaScript, - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Dts - | MediaType::Dmts - | MediaType::Dcts - | MediaType::Tsx => ModuleType::JavaScript, - MediaType::Json => ModuleType::Json, - _ => bail!("{:?} module type not supported", media_type,), - }; - - Ok(module_type) -} pub fn make_http_client() -> Result { let root_cert_store = None; @@ -47,52 +32,163 @@ pub fn make_http_client() -> Result { )) } +struct PreparedModuleLoader { + graph: ModuleGraph, + emitter: Arc, +} + +impl PreparedModuleLoader { + pub fn load_prepared_module( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + ) -> Result { + if specifier.scheme() == "node" { + unreachable!(); // Node built-in modules should be handled internally. + } + + match self.graph.get(specifier) { + Some(deno_graph::Module::Json(JsonModule { + source, + media_type, + specifier, + .. + })) => Ok(ModuleCodeSource { + code: source.clone().into(), + found_url: specifier.clone(), + media_type: *media_type, + }), + Some(deno_graph::Module::Esm(EsmModule { + source, + media_type, + specifier, + .. + })) => { + let code: ModuleCode = match media_type { + MediaType::JavaScript + | MediaType::Unknown + | MediaType::Cjs + | MediaType::Mjs + | MediaType::Json => source.clone().into(), + MediaType::Dts | MediaType::Dcts | MediaType::Dmts => Default::default(), + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Jsx + | MediaType::Tsx => { + // get emit text + self.emitter.emitter().unwrap().emit_parsed_source( + specifier, + *media_type, + source, + )? + } + MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { + panic!("Unexpected media type {media_type} for {specifier}") + } + }; + + Ok(ModuleCodeSource { + code, + found_url: specifier.clone(), + media_type: *media_type, + }) + } + _ => { + let mut msg = format!("Loading unprepared module: {specifier}"); + if let Some(referrer) = maybe_referrer { + msg = format!("{}, imported from: {}", msg, referrer.as_str()); + } + Err(anyhow!(msg)) + } + } + } + + pub async fn prepare_module_load( + &self, + roots: Vec, + is_dynamic: bool, + ) -> Result<(), AnyError> { + create_graph_from_specifiers(roots, is_dynamic, self.emitter.clone()).await?; + + // If there is a lockfile... + if let Some(lockfile) = self.emitter.get_lock_file() { + let lockfile = lockfile.lock(); + // update it with anything new + lockfile.write().context("Failed writing lockfile.")?; + } + + Ok(()) + } +} + pub struct DefaultModuleLoader { - file_fetcher: FileFetcher, - permissions: module_fetcher::permissions::Permissions, - emitter: Arc, - maybe_import_map: Option, + graph: ModuleGraph, + prepared_module_loader: Arc, + emitter: Arc, } impl DefaultModuleLoader { - pub fn new( - root_path: PathBuf, + #[allow(clippy::arc_with_non_send_sync)] + pub async fn new( + main_module: ModuleSpecifier, maybe_import_map: Option, - emitter: Arc, + mut emitter: EmitterFactory, no_cache: bool, allow_remote: bool, ) -> Result { - // Note: we are reusing Deno dependency cache path - let deno_dir = DenoDir::new(None)?; - let deps_cache_location = deno_dir.deps_folder_path(); - let cache_setting = if no_cache { CacheSetting::ReloadAll } else { CacheSetting::Use }; - let http_client = Arc::new(make_http_client()?); - let blob_store = Arc::new(deno_web::BlobStore::default()); - - let global_cache_struct = - GlobalHttpCache::new(deps_cache_location, module_fetcher::cache::RealDenoCacheEnv); - let global_cache: Arc = Arc::new(global_cache_struct); - let file_fetcher = FileFetcher::new( - global_cache.clone(), - cache_setting, - allow_remote, - http_client, - blob_store, - ); - let permissions = module_fetcher::permissions::Permissions::new(root_path); + + emitter.set_file_fetcher_cache_strategy(cache_setting); + emitter.set_file_fetcher_allow_remote(allow_remote); + emitter.set_import_map(maybe_import_map); + + let emitter = Arc::new(emitter); + let graph = create_graph(main_module.to_file_path().unwrap(), emitter.clone()).await; Ok(Self { - file_fetcher, - permissions, - maybe_import_map, + graph: graph.clone(), + prepared_module_loader: Arc::new(PreparedModuleLoader { + graph, + emitter: emitter.clone(), + }), emitter, }) } + + fn load_sync( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + _is_dynamic: bool, + ) -> Result { + let code_source = if let Some(result) = self + .emitter + .npm_module_loader() + .load_sync_if_in_npm_package(specifier, maybe_referrer, &*sb_node::allow_all()) + { + result? + } else { + self.prepared_module_loader + .load_prepared_module(specifier, maybe_referrer)? + }; + + let code = code_without_source_map(code_source.code); + + Ok(ModuleSource::new_with_redirect( + match code_source.media_type { + MediaType::Json => ModuleType::Json, + _ => ModuleType::JavaScript, + }, + code, + specifier, + &code_source.found_url, + )) + } } impl ModuleLoader for DefaultModuleLoader { @@ -102,29 +198,82 @@ impl ModuleLoader for DefaultModuleLoader { referrer: &str, _kind: ResolutionKind, ) -> Result { - if let Some(import_map) = &self.maybe_import_map { - let referrer_relative = Path::new(referrer).is_relative(); - let referrer_url = if referrer_relative { - import_map.base_url().join(referrer) - } else { - Url::parse(referrer) + let cwd = std::env::current_dir().context("Unable to get CWD")?; + let referrer_result = deno_core::resolve_url_or_path(referrer, &cwd); + let permissions = sb_node::allow_all(); + let npm_module_loader = self.emitter.npm_module_loader(); + + if let Ok(referrer) = referrer_result.as_ref() { + if let Some(result) = + npm_module_loader.resolve_if_in_npm_package(specifier, referrer, &*permissions) + { + return result; + } + + let graph = self.graph.clone(); + let maybe_resolved = match graph.get(referrer) { + Some(Module::Esm(module)) => { + module.dependencies.get(specifier).map(|d| &d.maybe_code) + } + _ => None, }; - if referrer_url.is_err() { - return referrer_url.map_err(|err| err.into()); + + match maybe_resolved { + Some(Resolution::Ok(resolved)) => { + let specifier = &resolved.specifier; + + return match graph.get(specifier) { + Some(Module::Npm(module)) => { + npm_module_loader.resolve_nv_ref(&module.nv_reference, &*permissions) + } + Some(Module::Node(module)) => Ok(module.specifier.clone()), + Some(Module::Esm(module)) => Ok(module.specifier.clone()), + Some(Module::Json(module)) => Ok(module.specifier.clone()), + Some(Module::External(module)) => { + Ok(node::resolve_specifier_into_node_modules(&module.specifier)) + } + None => Ok(specifier.clone()), + }; + } + Some(Resolution::Err(err)) => { + return Err(custom_error( + "TypeError", + format!("{}\n", err.to_string_with_range()), + )) + } + Some(Resolution::None) | None => {} } + } - let referrer_url = referrer_url.unwrap(); - import_map - .resolve(specifier, &referrer_url) - .map_err(|err| err.into()) - } else { - // // Built-in Node modules - // if let Some(module_name) = specifier.strip_prefix("node:") { - // return module_fetcher::node::resolve_builtin_node_module(module_name); - // } + self.emitter + .cli_graph_resolver() + .clone() + .resolve(specifier, &referrer_result?) + } + + fn prepare_load( + &self, + specifier: &ModuleSpecifier, + _maybe_referrer: Option, + is_dynamic: bool, + ) -> Pin>>> { + if let Some(result) = self + .emitter + .npm_module_loader() + .maybe_prepare_load(specifier) + { + return Box::pin(deno_core::futures::future::ready(result)); + } + + let specifier = specifier.clone(); + let module_load_preparer = self.prepared_module_loader.clone(); - deno_core::resolve_import(specifier, referrer).map_err(|err| err.into()) + async move { + module_load_preparer + .prepare_module_load(vec![specifier], is_dynamic) + .await } + .boxed_local() } // TODO: implement prepare_load method @@ -134,53 +283,10 @@ impl ModuleLoader for DefaultModuleLoader { _maybe_referrer: Option<&ModuleSpecifier>, _is_dyn_import: bool, ) -> Pin> { - let file_fetcher = self.file_fetcher.clone(); - let permissions = self.permissions.clone(); - let module_specifier = module_specifier.clone(); - let emitter = self.emitter.clone(); - - async move { - let fetched_file = file_fetcher - .fetch(&module_specifier, permissions) - .await - .map_err(|err| { - anyhow!( - "Failed to load module: {:?} - {:?}", - module_specifier.as_str(), - err - ) - })?; - let module_type = get_module_type(fetched_file.media_type)?; - - let code = fetched_file.source; - let code = match fetched_file.media_type { - MediaType::JavaScript - | MediaType::Unknown - | MediaType::Cjs - | MediaType::Mjs - | MediaType::Json => code.into(), - MediaType::Dts | MediaType::Dcts | MediaType::Dmts => Default::default(), - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx => { - emitter.emit_parsed_source(&module_specifier, fetched_file.media_type, &code)? - } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { - panic!("Unexpected media type during import.") - } - }; - - let module = ModuleSource::new_with_redirect( - module_type, - code, - &module_specifier, - &fetched_file.specifier, - ); - - Ok(module) - } - .boxed_local() + Box::pin(deno_core::futures::future::ready(self.load_sync( + module_specifier, + _maybe_referrer, + _is_dyn_import, + ))) } } diff --git a/crates/base/src/js_worker/node_module_loader.rs b/crates/base/src/js_worker/node_module_loader.rs new file mode 100644 index 000000000..e2e6cc2f9 --- /dev/null +++ b/crates/base/src/js_worker/node_module_loader.rs @@ -0,0 +1,196 @@ +use anyhow::Context; +use deno_ast::MediaType; +use deno_core::error::{generic_error, AnyError}; +use deno_core::parking_lot::Mutex; +use deno_core::{ModuleCode, ModuleSpecifier}; +use deno_semver::npm::{NpmPackageNvReference, NpmPackageReqReference}; +use module_fetcher::node::CliNodeCodeTranslator; +use sb_node::{NodePermissions, NodeResolution, NodeResolutionMode, NodeResolver}; +use std::collections::HashSet; +use std::sync::Arc; + +pub struct NpmModuleLoader { + cjs_resolutions: Arc, + node_code_translator: Arc, + fs: Arc, + node_resolver: Arc, +} + +pub struct ModuleCodeSource { + pub code: ModuleCode, + pub found_url: ModuleSpecifier, + pub media_type: MediaType, +} + +impl NpmModuleLoader { + pub fn new( + cjs_resolutions: Arc, + node_code_translator: Arc, + fs: Arc, + node_resolver: Arc, + ) -> Self { + Self { + cjs_resolutions, + node_code_translator, + fs, + node_resolver, + } + } + + pub fn resolve_if_in_npm_package( + &self, + specifier: &str, + referrer: &ModuleSpecifier, + permissions: &dyn NodePermissions, + ) -> Option> { + if self.node_resolver.in_npm_package(referrer) { + // we're in an npm package, so use node resolution + Some( + self.handle_node_resolve_result(self.node_resolver.resolve( + specifier, + referrer, + NodeResolutionMode::Execution, + permissions, + )) + .with_context(|| format!("Could not resolve '{specifier}' from '{referrer}'.")), + ) + } else { + None + } + } + + pub fn resolve_nv_ref( + &self, + nv_ref: &NpmPackageNvReference, + permissions: &dyn NodePermissions, + ) -> Result { + self.handle_node_resolve_result(self.node_resolver.resolve_npm_reference( + nv_ref, + NodeResolutionMode::Execution, + permissions, + )) + .with_context(|| format!("Could not resolve '{}'.", nv_ref)) + } + + pub fn resolve_req_reference( + &self, + reference: &NpmPackageReqReference, + permissions: &dyn NodePermissions, + ) -> Result { + self.handle_node_resolve_result(self.node_resolver.resolve_npm_req_reference( + reference, + NodeResolutionMode::Execution, + permissions, + )) + .with_context(|| format!("Could not resolve '{reference}'.")) + } + + pub fn maybe_prepare_load(&self, specifier: &ModuleSpecifier) -> Option> { + if self.node_resolver.in_npm_package(specifier) { + // nothing to prepare + Some(Ok(())) + } else { + None + } + } + + pub fn load_sync_if_in_npm_package( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + permissions: &dyn NodePermissions, + ) -> Option> { + if self.node_resolver.in_npm_package(specifier) { + Some(self.load_sync(specifier, maybe_referrer, permissions)) + } else { + None + } + } + + fn load_sync( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + permissions: &dyn NodePermissions, + ) -> Result { + let file_path = specifier.to_file_path().unwrap(); + let code = self + .fs + .read_text_file_sync(&file_path) + .map_err(AnyError::from) + .with_context(|| { + if file_path.is_dir() { + // directory imports are not allowed when importing from an + // ES module, so provide the user with a helpful error message + let dir_path = file_path; + let mut msg = "Directory import ".to_string(); + msg.push_str(&dir_path.to_string_lossy()); + if let Some(referrer) = &maybe_referrer { + msg.push_str(" is not supported resolving import from "); + msg.push_str(referrer.as_str()); + let entrypoint_name = ["index.mjs", "index.js", "index.cjs"] + .iter() + .find(|e| dir_path.join(e).is_file()); + if let Some(entrypoint_name) = entrypoint_name { + msg.push_str("\nDid you mean to import "); + msg.push_str(entrypoint_name); + msg.push_str(" within the directory?"); + } + } + msg + } else { + let mut msg = "Unable to load ".to_string(); + msg.push_str(&file_path.to_string_lossy()); + if let Some(referrer) = &maybe_referrer { + msg.push_str(" imported from "); + msg.push_str(referrer.as_str()); + } + msg + } + })?; + + let code = if self.cjs_resolutions.contains(specifier) { + // translate cjs to esm if it's cjs and inject node globals + self.node_code_translator.translate_cjs_to_esm( + specifier, + Some(code.as_str()), + permissions, + )? + } else { + // esm and json code is untouched + code + }; + Ok(ModuleCodeSource { + code: code.into(), + found_url: specifier.clone(), + media_type: MediaType::from_specifier(specifier), + }) + } + + fn handle_node_resolve_result( + &self, + result: Result, AnyError>, + ) -> Result { + let response = match result? { + Some(response) => response, + None => return Err(generic_error("not found")), + }; + if let NodeResolution::CommonJs(specifier) = &response { + // remember that this was a common js resolution + self.cjs_resolutions.insert(specifier.clone()); + } + Ok(response.into_url()) + } +} + +#[derive(Default)] +pub struct CjsResolutionStore(Mutex>); +impl CjsResolutionStore { + pub fn contains(&self, specifier: &ModuleSpecifier) -> bool { + self.0.lock().contains(specifier) + } + + pub fn insert(&self, specifier: ModuleSpecifier) { + self.0.lock().insert(specifier); + } +} diff --git a/crates/base/src/js_worker/standalone.rs b/crates/base/src/js_worker/standalone.rs new file mode 100644 index 000000000..e3f30380d --- /dev/null +++ b/crates/base/src/js_worker/standalone.rs @@ -0,0 +1,218 @@ +use crate::deno_runtime::RuntimeProviders; +use crate::js_worker::emitter::EmitterFactory; +use crate::js_worker::node_module_loader::NpmModuleLoader; +use crate::standalone::binary::Metadata; +use crate::standalone::create_module_loader_for_eszip; +use crate::utils::graph_resolver::MappedSpecifierResolver; +use deno_ast::MediaType; +use deno_core::error::{generic_error, type_error, AnyError}; +use deno_core::futures::FutureExt; +use deno_core::{ModuleLoader, ModuleSpecifier, ModuleType, ResolutionKind}; +use deno_semver::npm::NpmPackageReqReference; +use import_map::ImportMap; +use module_fetcher::file_fetcher::get_source_from_data_url; +use sb_eszip::module_loader::EszipPayloadKind; +use std::pin::Pin; +use std::sync::Arc; + +pub struct SharedModuleLoaderState { + eszip: eszip::EszipV2, + mapped_specifier_resolver: MappedSpecifierResolver, + npm_module_loader: Arc, +} + +#[derive(Clone)] +pub struct EmbeddedModuleLoader { + shared: Arc, +} + +fn arc_u8_to_arc_str(arc_u8: Arc<[u8]>) -> Result, std::str::Utf8Error> { + // Check that the string is valid UTF-8. + std::str::from_utf8(&arc_u8)?; + // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as + // Arc. This is proven by the From> impl for Arc<[u8]> from the + // standard library. + Ok(unsafe { std::mem::transmute(arc_u8) }) +} + +impl ModuleLoader for EmbeddedModuleLoader { + fn resolve( + &self, + specifier: &str, + referrer: &str, + kind: ResolutionKind, + ) -> Result { + let referrer = if referrer == "." { + if kind != ResolutionKind::MainModule { + return Err(generic_error(format!( + "Expected to resolve main module, got {:?} instead.", + kind + ))); + } + let current_dir = std::env::current_dir().unwrap(); + deno_core::resolve_path(".", ¤t_dir)? + } else { + ModuleSpecifier::parse(referrer) + .map_err(|err| type_error(format!("Referrer uses invalid specifier: {}", err)))? + }; + + let permissions = sb_node::allow_all(); + if let Some(result) = self.shared.npm_module_loader.resolve_if_in_npm_package( + specifier, + &referrer, + &*permissions, + ) { + return result; + } + + let maybe_mapped = self + .shared + .mapped_specifier_resolver + .resolve(specifier, &referrer)? + .into_specifier(); + + // npm specifier + let specifier_text = maybe_mapped + .as_ref() + .map(|r| r.as_str()) + .unwrap_or(specifier); + if let Ok(reference) = NpmPackageReqReference::from_str(specifier_text) { + return self + .shared + .npm_module_loader + .resolve_req_reference(&reference, &*permissions); + } + + match maybe_mapped { + Some(resolved) => Ok(resolved), + None => { + deno_core::resolve_import(specifier, referrer.as_str()).map_err(|err| err.into()) + } + } + } + + fn load( + &self, + original_specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + _is_dynamic: bool, + ) -> Pin> { + let is_data_uri = get_source_from_data_url(original_specifier).ok(); + if let Some((source, _)) = is_data_uri { + return Box::pin(deno_core::futures::future::ready(Ok( + deno_core::ModuleSource::new( + deno_core::ModuleType::JavaScript, + source.into(), + original_specifier, + ), + ))); + } + + let permissions = sb_node::allow_all(); + if let Some(result) = self.shared.npm_module_loader.load_sync_if_in_npm_package( + original_specifier, + maybe_referrer, + &*permissions, + ) { + return match result { + Ok(code_source) => Box::pin(deno_core::futures::future::ready(Ok( + deno_core::ModuleSource::new_with_redirect( + match code_source.media_type { + MediaType::Json => ModuleType::Json, + _ => ModuleType::JavaScript, + }, + code_source.code, + original_specifier, + &code_source.found_url, + ), + ))), + Err(err) => Box::pin(deno_core::futures::future::ready(Err(err))), + }; + } + + let Some(module) = self.shared.eszip.get_module(original_specifier.as_str()) else { + return Box::pin(deno_core::futures::future::ready(Err(type_error(format!( + "Module not found: {}", + original_specifier + ))))); + }; + let original_specifier = original_specifier.clone(); + let found_specifier = + ModuleSpecifier::parse(&module.specifier).expect("invalid url in eszip"); + + async move { + let code = module + .source() + .await + .ok_or_else(|| type_error(format!("Module not found: {}", original_specifier)))?; + let code = + arc_u8_to_arc_str(code).map_err(|_| type_error("Module source is not utf-8"))?; + Ok(deno_core::ModuleSource::new_with_redirect( + match module.kind { + eszip::ModuleKind::JavaScript => ModuleType::JavaScript, + eszip::ModuleKind::Json => ModuleType::Json, + eszip::ModuleKind::Jsonc => { + return Err(type_error("jsonc modules not supported")) + } + eszip::ModuleKind::OpaqueData => { + unreachable!(); + } + }, + code.into(), + &original_specifier, + &found_specifier, + )) + } + .boxed_local() + } +} + +pub fn create_shared_state_for_module_loader( + mut eszip: eszip::EszipV2, + maybe_import_map: Option>, +) -> SharedModuleLoaderState { + let mut emitter = EmitterFactory::new(); + + if let Some(snapshot) = eszip.take_npm_snapshot() { + emitter.set_npm_snapshot(Some(snapshot)); + } + + let shared_module_state = SharedModuleLoaderState { + eszip, + mapped_specifier_resolver: MappedSpecifierResolver::new( + maybe_import_map, + emitter.package_json_deps_provider().clone(), + ), + npm_module_loader: emitter.npm_module_loader(), + }; + shared_module_state +} + +pub async fn create_module_loader_for_standalone_from_eszip_kind( + eszip_payload_kind: EszipPayloadKind, + maybe_import_map: Option, +) -> RuntimeProviders { + use deno_core::futures::io::{AllowStdIo, BufReader}; + let bytes = match eszip_payload_kind { + EszipPayloadKind::JsBufferKind(js_buffer) => Vec::from(&*js_buffer), + EszipPayloadKind::VecKind(vec) => vec, + }; + + let bufreader = BufReader::new(AllowStdIo::new(bytes.as_slice())); + let (eszip, loader) = eszip::EszipV2::parse(bufreader).await.unwrap(); + + loader.await.unwrap(); + + create_module_loader_for_eszip( + eszip, + Metadata { + ca_stores: None, + ca_data: None, + unsafely_ignore_certificate_errors: None, + package_json_deps: None, + }, + maybe_import_map, + ) + .await + .unwrap() +} diff --git a/crates/base/src/lib.rs b/crates/base/src/lib.rs index d30a2a95a..344c11933 100644 --- a/crates/base/src/lib.rs +++ b/crates/base/src/lib.rs @@ -9,4 +9,5 @@ pub mod macros; pub mod rt_worker; pub mod server; pub mod snapshot; +pub mod standalone; pub mod utils; diff --git a/crates/base/src/standalone/binary.rs b/crates/base/src/standalone/binary.rs new file mode 100644 index 000000000..560a979cf --- /dev/null +++ b/crates/base/src/standalone/binary.rs @@ -0,0 +1,165 @@ +use crate::js_worker::emitter::EmitterFactory; +use crate::standalone::virtual_fs::{FileBackedVfs, VfsBuilder, VfsRoot, VirtualDirectory}; +use crate::standalone::VFS_ESZIP_KEY; +use crate::utils::graph_util::{create_eszip_from_graph_raw, create_graph}; +use deno_core::error::AnyError; +use deno_core::serde_json; +use deno_npm::registry::PackageDepNpmSchemeValueParseError; +use deno_npm::NpmSystemInfo; +use deno_semver::package::PackageReq; +use deno_semver::VersionReqSpecifierParseError; +use eszip::EszipV2; +use module_fetcher::args::package_json::{PackageJsonDepValueParseError, PackageJsonDeps}; +use serde::Deserialize; +use serde::Serialize; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; + +#[derive(Serialize, Deserialize)] +enum SerializablePackageJsonDepValueParseError { + SchemeValue(String), + Specifier(String), + Unsupported { scheme: String }, +} + +impl SerializablePackageJsonDepValueParseError { + pub fn from_err(err: PackageJsonDepValueParseError) -> Self { + match err { + PackageJsonDepValueParseError::SchemeValue(err) => Self::SchemeValue(err.value), + PackageJsonDepValueParseError::Specifier(err) => { + Self::Specifier(err.source.to_string()) + } + PackageJsonDepValueParseError::Unsupported { scheme } => Self::Unsupported { scheme }, + } + } + + pub fn into_err(self) -> PackageJsonDepValueParseError { + match self { + SerializablePackageJsonDepValueParseError::SchemeValue(value) => { + PackageJsonDepValueParseError::SchemeValue(PackageDepNpmSchemeValueParseError { + value, + }) + } + SerializablePackageJsonDepValueParseError::Specifier(source) => { + PackageJsonDepValueParseError::Specifier(VersionReqSpecifierParseError { + source: monch::ParseErrorFailureError::new(source), + }) + } + SerializablePackageJsonDepValueParseError::Unsupported { scheme } => { + PackageJsonDepValueParseError::Unsupported { scheme } + } + } + } +} + +#[derive(Serialize, Deserialize)] +pub struct SerializablePackageJsonDeps( + BTreeMap>, +); + +impl SerializablePackageJsonDeps { + pub fn from_deps(deps: PackageJsonDeps) -> Self { + Self( + deps.into_iter() + .map(|(name, req)| { + let res = req.map_err(SerializablePackageJsonDepValueParseError::from_err); + (name, res) + }) + .collect(), + ) + } + + pub fn into_deps(self) -> PackageJsonDeps { + self.0 + .into_iter() + .map(|(name, res)| (name, res.map_err(|err| err.into_err()))) + .collect() + } +} + +#[derive(Deserialize, Serialize)] +pub struct Metadata { + // pub argv: Vec, + // pub unstable: bool, + // pub seed: Option, + // pub location: Option, + // pub v8_flags: Vec, + pub ca_stores: Option>, + pub ca_data: Option>, + pub unsafely_ignore_certificate_errors: Option>, + // pub entrypoint: ModuleSpecifier, + // /// Whether this uses a node_modules directory (true) or the global cache (false). + // pub node_modules_dir: bool, + pub package_json_deps: Option, +} + +pub fn load_npm_vfs(root_dir_path: PathBuf, vfs_data: &[u8]) -> Result { + let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?; + + // align the name of the directory with the root dir + dir.name = root_dir_path + .file_name() + .unwrap() + .to_string_lossy() + .to_string(); + + let fs_root = VfsRoot { + dir, + root_path: root_dir_path, + }; + Ok(FileBackedVfs::new(fs_root)) +} + +pub fn build_vfs(emitter_factory: Arc) -> Result { + let npm_resolver = emitter_factory.npm_resolver(); + if let Some(node_modules_path) = npm_resolver.node_modules_path() { + let mut builder = VfsBuilder::new(node_modules_path.clone())?; + builder.add_dir_recursive(&node_modules_path)?; + Ok(builder) + } else { + // DO NOT include the user's registry url as it may contain credentials, + // but also don't make this dependent on the registry url + let registry_url = emitter_factory.npm_api().base_url(); + let root_path = emitter_factory.npm_cache().registry_folder(registry_url); + let mut builder = VfsBuilder::new(root_path)?; + for package in emitter_factory + .npm_resolution() + .all_system_packages(&NpmSystemInfo::default()) + { + let folder = emitter_factory + .npm_resolver() + .resolve_pkg_folder_from_pkg_id(&package.id)?; + builder.add_dir_recursive(&folder)?; + } + // overwrite the root directory's name to obscure the user's registry url + builder.set_root_dir_name("node_modules".to_string()); + Ok(builder) + } +} + +pub async fn generate_binary_eszip( + file: PathBuf, + emitter_factory: Arc, +) -> Result { + let graph = create_graph(file, emitter_factory.clone()).await; + let mut eszip = create_eszip_from_graph_raw(graph, Some(emitter_factory.clone())).await; + + let npm_res = emitter_factory.npm_resolution(); + + let (npm_vfs, _npm_files) = if npm_res.has_packages() { + let (root_dir, files) = build_vfs(emitter_factory.clone())?.into_dir_and_files(); + let snapshot = npm_res.serialized_valid_snapshot_for_system(&NpmSystemInfo::default()); + eszip.add_npm_snapshot(snapshot); + (Some(root_dir), files) + } else { + (None, Vec::new()) + }; + + let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); + let boxed_slice = npm_vfs.into_boxed_slice(); + + eszip.add_opaque_data(String::from(VFS_ESZIP_KEY), Arc::from(boxed_slice)); + + Ok(eszip) +} diff --git a/crates/base/src/standalone/file_system.rs b/crates/base/src/standalone/file_system.rs new file mode 100644 index 000000000..141409e58 --- /dev/null +++ b/crates/base/src/standalone/file_system.rs @@ -0,0 +1,286 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use deno_fs::{FileSystem, FsDirEntry, FsFileType, OpenOptions, RealFs}; +use deno_io::fs::{File, FsError, FsResult, FsStat}; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; + +use super::virtual_fs::FileBackedVfs; + +#[derive(Debug, Clone)] +pub struct DenoCompileFileSystem(Arc); + +impl DenoCompileFileSystem { + pub fn new(vfs: FileBackedVfs) -> Self { + Self(Arc::new(vfs)) + } + + fn error_if_in_vfs(&self, path: &Path) -> FsResult<()> { + if self.0.is_path_within(path) { + Err(FsError::NotSupported) + } else { + Ok(()) + } + } + + fn copy_to_real_path(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + let old_file = self.0.file_entry(oldpath)?; + let old_file_bytes = self.0.read_file_all(old_file)?; + RealFs.write_file_sync( + newpath, + OpenOptions { + read: false, + write: true, + create: true, + truncate: true, + append: false, + create_new: false, + mode: None, + }, + &old_file_bytes, + ) + } +} + +#[async_trait::async_trait(?Send)] +impl FileSystem for DenoCompileFileSystem { + fn cwd(&self) -> FsResult { + RealFs.cwd() + } + + fn tmp_dir(&self) -> FsResult { + RealFs.tmp_dir() + } + + fn chdir(&self, path: &Path) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.chdir(path) + } + + fn umask(&self, mask: Option) -> FsResult { + RealFs.umask(mask) + } + + fn open_sync(&self, path: &Path, options: OpenOptions) -> FsResult> { + if self.0.is_path_within(path) { + Ok(self.0.open_file(path)?) + } else { + RealFs.open_sync(path, options) + } + } + async fn open_async(&self, path: PathBuf, options: OpenOptions) -> FsResult> { + if self.0.is_path_within(&path) { + Ok(self.0.open_file(&path)?) + } else { + RealFs.open_async(path, options).await + } + } + + fn mkdir_sync(&self, path: &Path, recursive: bool, mode: u32) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.mkdir_sync(path, recursive, mode) + } + async fn mkdir_async(&self, path: PathBuf, recursive: bool, mode: u32) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.mkdir_async(path, recursive, mode).await + } + + fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.chmod_sync(path, mode) + } + async fn chmod_async(&self, path: PathBuf, mode: u32) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.chmod_async(path, mode).await + } + + fn chown_sync(&self, path: &Path, uid: Option, gid: Option) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.chown_sync(path, uid, gid) + } + async fn chown_async(&self, path: PathBuf, uid: Option, gid: Option) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.chown_async(path, uid, gid).await + } + + fn remove_sync(&self, path: &Path, recursive: bool) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.remove_sync(path, recursive) + } + async fn remove_async(&self, path: PathBuf, recursive: bool) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.remove_async(path, recursive).await + } + + fn copy_file_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + self.error_if_in_vfs(newpath)?; + if self.0.is_path_within(oldpath) { + self.copy_to_real_path(oldpath, newpath) + } else { + RealFs.copy_file_sync(oldpath, newpath) + } + } + async fn copy_file_async(&self, oldpath: PathBuf, newpath: PathBuf) -> FsResult<()> { + self.error_if_in_vfs(&newpath)?; + if self.0.is_path_within(&oldpath) { + let fs = self.clone(); + tokio::task::spawn_blocking(move || fs.copy_to_real_path(&oldpath, &newpath)).await? + } else { + RealFs.copy_file_async(oldpath, newpath).await + } + } + + fn stat_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.stat(path)?) + } else { + RealFs.stat_sync(path) + } + } + async fn stat_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.stat(&path)?) + } else { + RealFs.stat_async(path).await + } + } + + fn lstat_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.lstat(path)?) + } else { + RealFs.lstat_sync(path) + } + } + async fn lstat_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.lstat(&path)?) + } else { + RealFs.lstat_async(path).await + } + } + + fn realpath_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.canonicalize(path)?) + } else { + RealFs.realpath_sync(path) + } + } + async fn realpath_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.canonicalize(&path)?) + } else { + RealFs.realpath_async(path).await + } + } + + fn read_dir_sync(&self, path: &Path) -> FsResult> { + if self.0.is_path_within(path) { + Ok(self.0.read_dir(path)?) + } else { + RealFs.read_dir_sync(path) + } + } + async fn read_dir_async(&self, path: PathBuf) -> FsResult> { + if self.0.is_path_within(&path) { + Ok(self.0.read_dir(&path)?) + } else { + RealFs.read_dir_async(path).await + } + } + + fn rename_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + self.error_if_in_vfs(oldpath)?; + self.error_if_in_vfs(newpath)?; + RealFs.rename_sync(oldpath, newpath) + } + async fn rename_async(&self, oldpath: PathBuf, newpath: PathBuf) -> FsResult<()> { + self.error_if_in_vfs(&oldpath)?; + self.error_if_in_vfs(&newpath)?; + RealFs.rename_async(oldpath, newpath).await + } + + fn link_sync(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { + self.error_if_in_vfs(oldpath)?; + self.error_if_in_vfs(newpath)?; + RealFs.link_sync(oldpath, newpath) + } + async fn link_async(&self, oldpath: PathBuf, newpath: PathBuf) -> FsResult<()> { + self.error_if_in_vfs(&oldpath)?; + self.error_if_in_vfs(&newpath)?; + RealFs.link_async(oldpath, newpath).await + } + + fn symlink_sync( + &self, + oldpath: &Path, + newpath: &Path, + file_type: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(oldpath)?; + self.error_if_in_vfs(newpath)?; + RealFs.symlink_sync(oldpath, newpath, file_type) + } + async fn symlink_async( + &self, + oldpath: PathBuf, + newpath: PathBuf, + file_type: Option, + ) -> FsResult<()> { + self.error_if_in_vfs(&oldpath)?; + self.error_if_in_vfs(&newpath)?; + RealFs.symlink_async(oldpath, newpath, file_type).await + } + + fn read_link_sync(&self, path: &Path) -> FsResult { + if self.0.is_path_within(path) { + Ok(self.0.read_link(path)?) + } else { + RealFs.read_link_sync(path) + } + } + async fn read_link_async(&self, path: PathBuf) -> FsResult { + if self.0.is_path_within(&path) { + Ok(self.0.read_link(&path)?) + } else { + RealFs.read_link_async(path).await + } + } + + fn truncate_sync(&self, path: &Path, len: u64) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.truncate_sync(path, len) + } + async fn truncate_async(&self, path: PathBuf, len: u64) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs.truncate_async(path, len).await + } + + fn utime_sync( + &self, + path: &Path, + atime_secs: i64, + atime_nanos: u32, + mtime_secs: i64, + mtime_nanos: u32, + ) -> FsResult<()> { + self.error_if_in_vfs(path)?; + RealFs.utime_sync(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) + } + async fn utime_async( + &self, + path: PathBuf, + atime_secs: i64, + atime_nanos: u32, + mtime_secs: i64, + mtime_nanos: u32, + ) -> FsResult<()> { + self.error_if_in_vfs(&path)?; + RealFs + .utime_async(path, atime_secs, atime_nanos, mtime_secs, mtime_nanos) + .await + } +} diff --git a/crates/base/src/standalone/mod.rs b/crates/base/src/standalone/mod.rs new file mode 100644 index 000000000..0cedf75ff --- /dev/null +++ b/crates/base/src/standalone/mod.rs @@ -0,0 +1,348 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use deno_ast::MediaType; +use deno_core::anyhow::Context; +use deno_core::error::generic_error; +use deno_core::error::type_error; +use deno_core::error::AnyError; +use deno_core::futures::FutureExt; +use deno_core::ModuleLoader; +use deno_core::ModuleSpecifier; +use deno_core::ModuleType; +use deno_core::ResolutionKind; +use deno_fs; +use deno_npm::NpmSystemInfo; +use deno_semver::npm::NpmPackageReqReference; +use deno_tls::rustls::RootCertStore; +use deno_tls::RootCertStoreProvider; +use import_map::ImportMap; +use module_fetcher::args::package_json::PackageJsonDepsProvider; +use module_fetcher::args::CacheSetting; +use module_fetcher::cache::{Caches, DenoDirProvider, NodeAnalysisCache}; +use module_fetcher::file_fetcher::get_source_from_data_url; +use module_fetcher::http_util::HttpClient; +use module_fetcher::node::CliCjsCodeAnalyzer; +use sb_node::analyze::NodeCodeTranslator; +use std::pin::Pin; +use std::rc::Rc; +use std::sync::Arc; + +pub mod binary; +pub mod file_system; +pub mod virtual_fs; + +use crate::cert::{get_root_cert_store, CaData}; +use crate::deno_runtime::RuntimeProviders; +use crate::js_worker::node_module_loader::{CjsResolutionStore, NpmModuleLoader}; +use crate::utils::graph_resolver::MappedSpecifierResolver; +use sb_node::NodeResolver; +use sb_npm::{ + create_npm_fs_resolver, CliNpmRegistryApi, CliNpmResolver, NpmCache, NpmCacheDir, NpmResolution, +}; + +use self::binary::load_npm_vfs; +use self::binary::Metadata; +use self::file_system::DenoCompileFileSystem; + +pub const VFS_ESZIP_KEY: &str = "---SUPABASE-VFS-DATA-ESZIP---"; + +pub struct SharedModuleLoaderState { + eszip: eszip::EszipV2, + mapped_specifier_resolver: MappedSpecifierResolver, + npm_module_loader: Arc, +} + +#[derive(Clone)] +pub struct EmbeddedModuleLoader { + shared: Arc, +} + +impl ModuleLoader for EmbeddedModuleLoader { + fn resolve( + &self, + specifier: &str, + referrer: &str, + kind: ResolutionKind, + ) -> Result { + let referrer = if referrer == "." { + if kind != ResolutionKind::MainModule { + return Err(generic_error(format!( + "Expected to resolve main module, got {:?} instead.", + kind + ))); + } + let current_dir = std::env::current_dir().unwrap(); + deno_core::resolve_path(".", ¤t_dir)? + } else { + ModuleSpecifier::parse(referrer) + .map_err(|err| type_error(format!("Referrer uses invalid specifier: {}", err)))? + }; + + let permissions = sb_node::allow_all(); + if let Some(result) = self.shared.npm_module_loader.resolve_if_in_npm_package( + specifier, + &referrer, + &*permissions, + ) { + return result; + } + + let maybe_mapped = self + .shared + .mapped_specifier_resolver + .resolve(specifier, &referrer)? + .into_specifier(); + + // npm specifier + let specifier_text = maybe_mapped + .as_ref() + .map(|r| r.as_str()) + .unwrap_or(specifier); + if let Ok(reference) = NpmPackageReqReference::from_str(specifier_text) { + return self + .shared + .npm_module_loader + .resolve_req_reference(&reference, &*permissions); + } + + match maybe_mapped { + Some(resolved) => Ok(resolved), + None => { + deno_core::resolve_import(specifier, referrer.as_str()).map_err(|err| err.into()) + } + } + } + + fn load( + &self, + original_specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + _is_dynamic: bool, + ) -> Pin> { + let is_data_uri = get_source_from_data_url(original_specifier).ok(); + let permissions = sb_node::allow_all(); + if let Some((source, _)) = is_data_uri { + return Box::pin(deno_core::futures::future::ready(Ok( + deno_core::ModuleSource::new( + deno_core::ModuleType::JavaScript, + source.into(), + original_specifier, + ), + ))); + } + + if let Some(result) = self.shared.npm_module_loader.load_sync_if_in_npm_package( + original_specifier, + maybe_referrer, + &*permissions, + ) { + return match result { + Ok(code_source) => Box::pin(deno_core::futures::future::ready(Ok( + deno_core::ModuleSource::new_with_redirect( + match code_source.media_type { + MediaType::Json => ModuleType::Json, + _ => ModuleType::JavaScript, + }, + code_source.code, + original_specifier, + &code_source.found_url, + ), + ))), + Err(err) => Box::pin(deno_core::futures::future::ready(Err(err))), + }; + } + + let Some(module) = self.shared.eszip.get_module(original_specifier.as_str()) else { + return Box::pin(deno_core::futures::future::ready(Err(type_error(format!( + "Module not found: {}", + original_specifier + ))))); + }; + let original_specifier = original_specifier.clone(); + let found_specifier = + ModuleSpecifier::parse(&module.specifier).expect("invalid url in eszip"); + + async move { + let code = module + .source() + .await + .ok_or_else(|| type_error(format!("Module not found: {}", original_specifier)))?; + let code = + arc_u8_to_arc_str(code).map_err(|_| type_error("Module source is not utf-8"))?; + Ok(deno_core::ModuleSource::new_with_redirect( + match module.kind { + eszip::ModuleKind::JavaScript => ModuleType::JavaScript, + eszip::ModuleKind::Json => ModuleType::Json, + eszip::ModuleKind::Jsonc => { + return Err(type_error("jsonc modules not supported")) + } + eszip::ModuleKind::OpaqueData => { + unreachable!(); + } + }, + code.into(), + &original_specifier, + &found_specifier, + )) + } + .boxed_local() + } +} + +fn arc_u8_to_arc_str(arc_u8: Arc<[u8]>) -> Result, std::str::Utf8Error> { + // Check that the string is valid UTF-8. + std::str::from_utf8(&arc_u8)?; + // SAFETY: the string is valid UTF-8, and the layout Arc<[u8]> is the same as + // Arc. This is proven by the From> impl for Arc<[u8]> from the + // standard library. + Ok(unsafe { std::mem::transmute(arc_u8) }) +} + +pub struct StandaloneModuleLoaderFactory { + shared: Arc, +} + +struct StandaloneRootCertStoreProvider { + ca_stores: Option>, + ca_data: Option, + cell: once_cell::sync::OnceCell, +} + +impl RootCertStoreProvider for StandaloneRootCertStoreProvider { + fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { + self.cell.get_or_try_init(|| { + get_root_cert_store(None, self.ca_stores.clone(), self.ca_data.clone()) + .map_err(|err| err.into()) + }) + } +} + +pub async fn create_module_loader_for_eszip( + mut eszip: eszip::EszipV2, + metadata: Metadata, + maybe_import_map: Option, +) -> Result { + // let main_module = &metadata.entrypoint; + let current_exe_path = std::env::current_exe().unwrap(); + let current_exe_name = current_exe_path.file_name().unwrap().to_string_lossy(); + let deno_dir_provider = Arc::new(DenoDirProvider::new(None)); + let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider { + ca_stores: metadata.ca_stores, + ca_data: metadata.ca_data.map(CaData::Bytes), + cell: Default::default(), + }); + let http_client = Arc::new(HttpClient::new( + Some(root_cert_store_provider.clone()), + metadata.unsafely_ignore_certificate_errors.clone(), + )); + + // use a dummy npm registry url + let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap(); + let root_path = std::env::temp_dir() + .join(format!("sb-compile-{}", current_exe_name)) + .join("node_modules"); + let npm_cache_dir = NpmCacheDir::new(root_path.clone()); + let (fs, snapshot) = if let Some(snapshot) = eszip.take_npm_snapshot() { + // TODO: Support node_modules + let vfs_root_dir_path = npm_cache_dir.registry_folder(&npm_registry_url); + + let vfs_data: Vec = eszip + .get_module(VFS_ESZIP_KEY) + .unwrap() + .take_source() + .await + .unwrap() + .to_vec(); + + let vfs = load_npm_vfs(vfs_root_dir_path, &vfs_data).context("Failed to load npm vfs.")?; + + ( + Arc::new(DenoCompileFileSystem::new(vfs)) as Arc, + Some(snapshot), + ) + } else { + ( + Arc::new(deno_fs::RealFs) as Arc, + None, + ) + }; + + let npm_cache = Arc::new(NpmCache::new( + npm_cache_dir, + CacheSetting::Only, + fs.clone(), + http_client.clone(), + )); + + let npm_api = Arc::new(CliNpmRegistryApi::new( + npm_registry_url.clone(), + npm_cache.clone(), + http_client.clone(), + )); + + let npm_resolution = Arc::new(NpmResolution::from_serialized( + npm_api.clone(), + snapshot, + None, + )); + + let npm_fs_resolver = create_npm_fs_resolver( + fs.clone(), + npm_cache, + npm_registry_url, + npm_resolution.clone(), + None, + NpmSystemInfo::default(), + ); + + let npm_resolver = Arc::new(CliNpmResolver::new( + fs.clone(), + npm_resolution.clone(), + npm_fs_resolver, + None, + )); + + let node_resolver = Arc::new(NodeResolver::new(fs.clone(), npm_resolver.clone())); + let cjs_resolutions = Arc::new(CjsResolutionStore::default()); + let cache_db = Caches::new(deno_dir_provider.clone()); + let node_analysis_cache = NodeAnalysisCache::new(cache_db.node_analysis_db()); + let cjs_esm_code_analyzer = CliCjsCodeAnalyzer::new(node_analysis_cache, fs.clone()); + let node_code_translator = Arc::new(NodeCodeTranslator::new( + cjs_esm_code_analyzer, + fs.clone(), + node_resolver.clone(), + npm_resolver.clone(), + )); + let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new( + metadata + .package_json_deps + .map(|serialized| serialized.into_deps()), + )); + let maybe_import_map = maybe_import_map + .map(|import_map| Some(Arc::new(import_map))) + .unwrap_or_else(|| None); + + let module_loader_factory = StandaloneModuleLoaderFactory { + shared: Arc::new(SharedModuleLoaderState { + eszip, + mapped_specifier_resolver: MappedSpecifierResolver::new( + maybe_import_map, + package_json_deps_provider.clone(), + ), + npm_module_loader: Arc::new(NpmModuleLoader::new( + cjs_resolutions, + node_code_translator, + fs.clone(), + node_resolver.clone(), + )), + }), + }; + + Ok(RuntimeProviders { + module_loader: Rc::new(EmbeddedModuleLoader { + shared: module_loader_factory.shared.clone(), + }), + npm_resolver, + fs, + }) +} diff --git a/crates/base/src/standalone/virtual_fs.rs b/crates/base/src/standalone/virtual_fs.rs new file mode 100644 index 000000000..d5fd30f49 --- /dev/null +++ b/crates/base/src/standalone/virtual_fs.rs @@ -0,0 +1,803 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::collections::HashSet; +use std::io; +use std::io::SeekFrom; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; + +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::BufMutView; +use deno_core::BufView; +use deno_core::ResourceHandleFd; +use deno_fs::FsDirEntry; +use deno_io; +use deno_io::fs::FsError; +use deno_io::fs::FsResult; +use deno_io::fs::FsStat; +use module_fetcher::util; +use module_fetcher::util::fs::canonicalize_path; +use serde::Deserialize; +use serde::Serialize; +use thiserror::Error; + +#[derive(Error, Debug)] +#[error( +"Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() +)] +pub struct StripRootError { + root_path: PathBuf, + target: PathBuf, +} + +pub struct VfsBuilder { + root_path: PathBuf, + root_dir: VirtualDirectory, + files: Vec>, + current_offset: u64, + file_offsets: HashMap, +} + +impl VfsBuilder { + pub fn new(root_path: PathBuf) -> Result { + let root_path = canonicalize_path(&root_path)?; + log::debug!("Building vfs with root '{}'", root_path.display()); + Ok(Self { + root_dir: VirtualDirectory { + name: root_path + .file_stem() + .unwrap() + .to_string_lossy() + .into_owned(), + entries: Vec::new(), + }, + root_path, + files: Vec::new(), + current_offset: 0, + file_offsets: Default::default(), + }) + } + + pub fn set_root_dir_name(&mut self, name: String) { + self.root_dir.name = name; + } + + pub fn add_dir_recursive(&mut self, path: &Path) -> Result<(), AnyError> { + let path = canonicalize_path(path)?; + self.add_dir_recursive_internal(&path) + } + + fn add_dir_recursive_internal(&mut self, path: &Path) -> Result<(), AnyError> { + self.add_dir(path)?; + let read_dir = + std::fs::read_dir(path).with_context(|| format!("Reading {}", path.display()))?; + + for entry in read_dir { + let entry = entry?; + let file_type = entry.file_type()?; + let path = entry.path(); + + if file_type.is_dir() { + self.add_dir_recursive_internal(&path)?; + } else if file_type.is_file() { + let file_bytes = + std::fs::read(&path).with_context(|| format!("Reading {}", path.display()))?; + self.add_file(&path, file_bytes)?; + } else if file_type.is_symlink() { + let target = util::fs::canonicalize_path(&path) + .with_context(|| format!("Reading symlink {}", path.display()))?; + if let Err(StripRootError { .. }) = self.add_symlink(&path, &target) { + if target.is_file() { + // this may change behavior, so warn the user about it + log::warn!( + "Symlink target is outside '{}'. Inlining symlink at '{}' to '{}' as file.", + self.root_path.display(), + path.display(), + target.display(), + ); + // inline the symlink and make the target file + let file_bytes = std::fs::read(&target) + .with_context(|| format!("Reading {}", path.display()))?; + self.add_file(&path, file_bytes)?; + } else { + log::warn!( + "Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", + self.root_path.display(), + path.display(), + target.display(), + ); + } + } + } + } + + Ok(()) + } + + fn add_dir(&mut self, path: &Path) -> Result<&mut VirtualDirectory, StripRootError> { + log::debug!("Ensuring directory '{}'", path.display()); + let path = self.path_relative_root(path)?; + let mut current_dir = &mut self.root_dir; + + for component in path.components() { + let name = component.as_os_str().to_string_lossy(); + let index = match current_dir + .entries + .binary_search_by(|e| e.name().cmp(&name)) + { + Ok(index) => index, + Err(insert_index) => { + current_dir.entries.insert( + insert_index, + VfsEntry::Dir(VirtualDirectory { + name: name.to_string(), + entries: Vec::new(), + }), + ); + insert_index + } + }; + match &mut current_dir.entries[index] { + VfsEntry::Dir(dir) => { + current_dir = dir; + } + _ => unreachable!(), + }; + } + + Ok(current_dir) + } + + fn add_file(&mut self, path: &Path, data: Vec) -> Result<(), AnyError> { + log::debug!("Adding file '{}'", path.display()); + let checksum = util::checksum::gen(&[&data]); + let offset = if let Some(offset) = self.file_offsets.get(&checksum) { + // duplicate file, reuse an old offset + *offset + } else { + self.file_offsets.insert(checksum, self.current_offset); + self.current_offset + }; + + let dir = self.add_dir(path.parent().unwrap())?; + let name = path.file_name().unwrap().to_string_lossy(); + let data_len = data.len(); + match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { + Ok(_) => unreachable!(), + Err(insert_index) => { + dir.entries.insert( + insert_index, + VfsEntry::File(VirtualFile { + name: name.to_string(), + offset, + len: data.len() as u64, + content: Some(data), + }), + ); + } + } + + // new file, update the list of files + if self.current_offset == offset { + // self.files.push(data); + self.current_offset += data_len as u64; + } + + Ok(()) + } + + fn add_symlink(&mut self, path: &Path, target: &Path) -> Result<(), StripRootError> { + log::debug!( + "Adding symlink '{}' to '{}'", + path.display(), + target.display() + ); + let dest = self.path_relative_root(target)?; + let dir = self.add_dir(path.parent().unwrap())?; + let name = path.file_name().unwrap().to_string_lossy(); + match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { + Ok(_) => unreachable!(), + Err(insert_index) => { + dir.entries.insert( + insert_index, + VfsEntry::Symlink(VirtualSymlink { + name: name.to_string(), + dest_parts: dest + .components() + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect::>(), + }), + ); + } + } + Ok(()) + } + + pub fn into_dir_and_files(self) -> (VirtualDirectory, Vec>) { + (self.root_dir, self.files) + } + + fn path_relative_root(&self, path: &Path) -> Result { + match path.strip_prefix(&self.root_path) { + Ok(p) => Ok(p.to_path_buf()), + Err(_) => Err(StripRootError { + root_path: self.root_path.clone(), + target: path.to_path_buf(), + }), + } + } +} + +#[derive(Debug)] +enum VfsEntryRef<'a> { + Dir(&'a VirtualDirectory), + File(&'a VirtualFile), + Symlink(&'a VirtualSymlink), +} + +impl<'a> VfsEntryRef<'a> { + pub fn as_fs_stat(&self) -> FsStat { + match self { + VfsEntryRef::Dir(_) => FsStat { + is_directory: true, + is_file: false, + is_symlink: false, + atime: None, + birthtime: None, + mtime: None, + blksize: 0, + size: 0, + dev: 0, + ino: 0, + mode: 0, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blocks: 0, + is_block_device: false, + is_char_device: false, + is_fifo: false, + is_socket: false, + }, + VfsEntryRef::File(file) => FsStat { + is_directory: false, + is_file: true, + is_symlink: false, + atime: None, + birthtime: None, + mtime: None, + blksize: 0, + size: file.len, + dev: 0, + ino: 0, + mode: 0, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blocks: 0, + is_block_device: false, + is_char_device: false, + is_fifo: false, + is_socket: false, + }, + VfsEntryRef::Symlink(_) => FsStat { + is_directory: false, + is_file: false, + is_symlink: true, + atime: None, + birthtime: None, + mtime: None, + blksize: 0, + size: 0, + dev: 0, + ino: 0, + mode: 0, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blocks: 0, + is_block_device: false, + is_char_device: false, + is_fifo: false, + is_socket: false, + }, + } + } +} + +// todo(dsherret): we should store this more efficiently in the binary +#[derive(Debug, Serialize, Deserialize)] +pub enum VfsEntry { + Dir(VirtualDirectory), + File(VirtualFile), + Symlink(VirtualSymlink), +} + +impl VfsEntry { + pub fn name(&self) -> &str { + match self { + VfsEntry::Dir(dir) => &dir.name, + VfsEntry::File(file) => &file.name, + VfsEntry::Symlink(symlink) => &symlink.name, + } + } + + fn as_ref(&self) -> VfsEntryRef { + match self { + VfsEntry::Dir(dir) => VfsEntryRef::Dir(dir), + VfsEntry::File(file) => VfsEntryRef::File(file), + VfsEntry::Symlink(symlink) => VfsEntryRef::Symlink(symlink), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualDirectory { + pub name: String, + // should be sorted by name + pub entries: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VirtualFile { + pub name: String, + pub offset: u64, + pub len: u64, + pub content: Option>, // Not Deno Original, but it's the best way to store it in the ESZIP. +} + +impl VirtualFile { + pub fn read_file(&self, _pos: u64, buf: &mut [u8]) -> std::io::Result { + match &self.content { + Some(content) => { + let read_length = buf.len().min(content.len()); + buf[..read_length].copy_from_slice(&content[..read_length]); + Ok(read_length) + } + None => Err(io::Error::new( + io::ErrorKind::NotFound, + "No content available", + )), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct VirtualSymlink { + pub name: String, + pub dest_parts: Vec, +} + +impl VirtualSymlink { + pub fn resolve_dest_from_root(&self, root: &Path) -> PathBuf { + let mut dest = root.to_path_buf(); + for part in &self.dest_parts { + dest.push(part); + } + dest + } +} + +#[derive(Debug)] +pub struct VfsRoot { + pub dir: VirtualDirectory, + pub root_path: PathBuf, +} + +impl VfsRoot { + fn find_entry<'a>(&'a self, path: &Path) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { + self.find_entry_inner(path, &mut HashSet::new()) + } + + fn find_entry_inner<'a>( + &'a self, + path: &Path, + seen: &mut HashSet, + ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { + let mut path = Cow::Borrowed(path); + loop { + let (resolved_path, entry) = self.find_entry_no_follow_inner(&path, seen)?; + match entry { + VfsEntryRef::Symlink(symlink) => { + if !seen.insert(path.to_path_buf()) { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "circular symlinks", + )); + } + path = Cow::Owned(symlink.resolve_dest_from_root(&self.root_path)); + } + _ => { + return Ok((resolved_path, entry)); + } + } + } + } + + fn find_entry_no_follow(&self, path: &Path) -> std::io::Result<(PathBuf, VfsEntryRef)> { + self.find_entry_no_follow_inner(path, &mut HashSet::new()) + } + + fn find_entry_no_follow_inner<'a>( + &'a self, + path: &Path, + seen: &mut HashSet, + ) -> std::io::Result<(PathBuf, VfsEntryRef<'a>)> { + let relative_path = match path.strip_prefix(&self.root_path) { + Ok(p) => p, + Err(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + }; + let mut final_path = self.root_path.clone(); + let mut current_entry = VfsEntryRef::Dir(&self.dir); + for component in relative_path.components() { + let component = component.as_os_str().to_string_lossy(); + let current_dir = match current_entry { + VfsEntryRef::Dir(dir) => { + final_path.push(component.as_ref()); + dir + } + VfsEntryRef::Symlink(symlink) => { + let dest = symlink.resolve_dest_from_root(&self.root_path); + let (resolved_path, entry) = self.find_entry_inner(&dest, seen)?; + final_path = resolved_path; // overwrite with the new resolved path + match entry { + VfsEntryRef::Dir(dir) => { + final_path.push(component.as_ref()); + dir + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + } + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + }; + match current_dir + .entries + .binary_search_by(|e| e.name().cmp(&component)) + { + Ok(index) => { + current_entry = current_dir.entries[index].as_ref(); + } + Err(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "path not found", + )); + } + } + } + + Ok((final_path, current_entry)) + } +} + +#[derive(Clone)] +struct FileBackedVfsFile { + file: VirtualFile, + pos: Arc>, + vfs: Arc, +} + +impl FileBackedVfsFile { + fn seek(&self, pos: SeekFrom) -> FsResult { + match pos { + SeekFrom::Start(pos) => { + *self.pos.lock() = pos; + Ok(pos) + } + SeekFrom::End(offset) => { + if offset < 0 && -offset as u64 > self.file.len { + let msg = "An attempt was made to move the file pointer before the beginning of the file."; + Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, msg).into()) + } else { + let mut current_pos = self.pos.lock(); + *current_pos = if offset >= 0 { + self.file.len - (offset as u64) + } else { + self.file.len + (-offset as u64) + }; + Ok(*current_pos) + } + } + SeekFrom::Current(offset) => { + let mut current_pos = self.pos.lock(); + if offset >= 0 { + *current_pos += offset as u64; + } else if -offset as u64 > *current_pos { + return Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "An attempt was made to move the file pointer before the beginning of the file.").into()); + } else { + *current_pos -= -offset as u64; + } + Ok(*current_pos) + } + } + } + + fn read_to_buf(&self, buf: &mut [u8]) -> FsResult { + let pos = { + let mut pos = self.pos.lock(); + let read_pos = *pos; + // advance the position due to the read + *pos = std::cmp::min(self.file.len, *pos + buf.len() as u64); + read_pos + }; + self.vfs + .read_file(&self.file, pos, buf) + .map_err(|err| err.into()) + } + + fn read_to_end(&self) -> FsResult> { + let pos = { + let mut pos = self.pos.lock(); + let read_pos = *pos; + // todo(dsherret): should this always set it to the end of the file? + if *pos < self.file.len { + // advance the position due to the read + *pos = self.file.len; + } + read_pos + }; + if pos > self.file.len { + return Ok(Vec::new()); + } + let size = (self.file.len - pos) as usize; + let mut buf = vec![0; size]; + self.vfs.read_file(&self.file, pos, &mut buf)?; + Ok(buf) + } +} + +#[async_trait::async_trait(?Send)] +impl deno_io::fs::File for FileBackedVfsFile { + fn read_sync(self: Rc, buf: &mut [u8]) -> FsResult { + self.read_to_buf(buf) + } + async fn read_byob(self: Rc, mut buf: BufMutView) -> FsResult<(usize, BufMutView)> { + let inner = (*self).clone(); + tokio::task::spawn(async move { + let nread = inner.read_to_buf(&mut buf)?; + Ok((nread, buf)) + }) + .await? + } + + fn write_sync(self: Rc, _buf: &[u8]) -> FsResult { + Err(FsError::NotSupported) + } + async fn write(self: Rc, _buf: BufView) -> FsResult { + Err(FsError::NotSupported) + } + + fn write_all_sync(self: Rc, _buf: &[u8]) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn write_all(self: Rc, _buf: BufView) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn read_all_sync(self: Rc) -> FsResult> { + self.read_to_end() + } + async fn read_all_async(self: Rc) -> FsResult> { + let inner = (*self).clone(); + tokio::task::spawn_blocking(move || inner.read_to_end()).await? + } + + fn chmod_sync(self: Rc, _pathmode: u32) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn chmod_async(self: Rc, _mode: u32) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn seek_sync(self: Rc, pos: SeekFrom) -> FsResult { + self.seek(pos) + } + async fn seek_async(self: Rc, pos: SeekFrom) -> FsResult { + self.seek(pos) + } + + fn datasync_sync(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn datasync_async(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn sync_sync(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn sync_async(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn stat_sync(self: Rc) -> FsResult { + Err(FsError::NotSupported) + } + async fn stat_async(self: Rc) -> FsResult { + Err(FsError::NotSupported) + } + + fn lock_sync(self: Rc, _exclusive: bool) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn lock_async(self: Rc, _exclusive: bool) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn unlock_sync(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn unlock_async(self: Rc) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn truncate_sync(self: Rc, _len: u64) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn truncate_async(self: Rc, _len: u64) -> FsResult<()> { + Err(FsError::NotSupported) + } + + fn utime_sync( + self: Rc, + _atime_secs: i64, + _atime_nanos: u32, + _mtime_secs: i64, + _mtime_nanos: u32, + ) -> FsResult<()> { + Err(FsError::NotSupported) + } + async fn utime_async( + self: Rc, + _atime_secs: i64, + _atime_nanos: u32, + _mtime_secs: i64, + _mtime_nanos: u32, + ) -> FsResult<()> { + Err(FsError::NotSupported) + } + + // lower level functionality + fn as_stdio(self: Rc) -> FsResult { + Err(FsError::NotSupported) + } + fn backing_fd(self: Rc) -> Option { + None + } + fn try_clone_inner(self: Rc) -> FsResult> { + Ok(self) + } +} + +#[derive(Debug)] +pub struct FileBackedVfs { + fs_root: VfsRoot, +} + +impl FileBackedVfs { + pub fn new(fs_root: VfsRoot) -> Self { + Self { fs_root } + } + + pub fn root(&self) -> &Path { + &self.fs_root.root_path + } + + pub fn is_path_within(&self, path: &Path) -> bool { + path.starts_with(&self.fs_root.root_path) + } + + pub fn open_file(self: &Arc, path: &Path) -> std::io::Result> { + let file = self.file_entry(path)?; + Ok(Rc::new(FileBackedVfsFile { + file: file.clone(), + vfs: self.clone(), + pos: Default::default(), + })) + } + + pub fn read_dir(&self, path: &Path) -> std::io::Result> { + let dir = self.dir_entry(path)?; + Ok(dir + .entries + .iter() + .map(|entry| FsDirEntry { + name: entry.name().to_string(), + is_file: matches!(entry, VfsEntry::File(_)), + is_directory: matches!(entry, VfsEntry::Dir(_)), + is_symlink: matches!(entry, VfsEntry::Symlink(_)), + }) + .collect()) + } + + pub fn read_link(&self, path: &Path) -> std::io::Result { + let (_, entry) = self.fs_root.find_entry_no_follow(path)?; + match entry { + VfsEntryRef::Symlink(symlink) => { + Ok(symlink.resolve_dest_from_root(&self.fs_root.root_path)) + } + VfsEntryRef::Dir(_) | VfsEntryRef::File(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "not a symlink", + )), + } + } + + pub fn lstat(&self, path: &Path) -> std::io::Result { + let (_, entry) = self.fs_root.find_entry_no_follow(path)?; + Ok(entry.as_fs_stat()) + } + + pub fn stat(&self, path: &Path) -> std::io::Result { + let (_, entry) = self.fs_root.find_entry(path)?; + Ok(entry.as_fs_stat()) + } + + pub fn canonicalize(&self, path: &Path) -> std::io::Result { + let (path, _) = self.fs_root.find_entry(path)?; + Ok(path) + } + + pub fn read_file_all(&self, file: &VirtualFile) -> std::io::Result> { + let mut buf = vec![0; file.len as usize]; + self.read_file(file, 0, &mut buf)?; + Ok(buf) + } + + pub fn read_file( + &self, + file: &VirtualFile, + pos: u64, + buf: &mut [u8], + ) -> std::io::Result { + file.read_file(pos, buf) + } + + pub fn dir_entry(&self, path: &Path) -> std::io::Result<&VirtualDirectory> { + let (_, entry) = self.fs_root.find_entry(path)?; + match entry { + VfsEntryRef::Dir(dir) => Ok(dir), + VfsEntryRef::Symlink(_) => unreachable!(), + VfsEntryRef::File(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "path is a file", + )), + } + } + + pub fn file_entry(&self, path: &Path) -> std::io::Result<&VirtualFile> { + let (_, entry) = self.fs_root.find_entry(path)?; + match entry { + VfsEntryRef::Dir(_) => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "path is a directory", + )), + VfsEntryRef::Symlink(_) => unreachable!(), + VfsEntryRef::File(file) => Ok(file), + } + } +} diff --git a/crates/base/src/utils/graph_resolver.rs b/crates/base/src/utils/graph_resolver.rs index 9f2f46e1d..c3112007d 100644 --- a/crates/base/src/utils/graph_resolver.rs +++ b/crates/base/src/utils/graph_resolver.rs @@ -1,11 +1,23 @@ +use crate::js_worker::emitter::EmitterFactory; use anyhow::{anyhow, bail}; use deno_core::error::AnyError; +use deno_core::futures::future::LocalBoxFuture; +use deno_core::futures::FutureExt; use deno_core::ModuleSpecifier; +use deno_npm::registry::NpmRegistryApi; +use deno_semver::package::PackageReq; use eszip::deno_graph; -use eszip::deno_graph::source::{Resolver, DEFAULT_JSX_IMPORT_SOURCE_MODULE}; +use eszip::deno_graph::source::{ + NpmResolver, Resolver, UnknownBuiltInNodeModuleError, DEFAULT_JSX_IMPORT_SOURCE_MODULE, +}; +use eszip::deno_graph::NpmPackageReqResolution; use import_map::ImportMap; use module_fetcher::args::package_json::{PackageJsonDeps, PackageJsonDepsProvider}; +use module_fetcher::args::JsxImportSourceConfig; use module_fetcher::util::sync::AtomicFlag; +use sb_node::is_builtin_node_module; +use sb_npm::{CliNpmRegistryApi, NpmResolution, PackageJsonDepsInstaller}; +use std::path::PathBuf; use std::sync::Arc; /// Result of checking if a specifier is mapped via @@ -104,23 +116,86 @@ pub struct CliGraphResolver { maybe_default_jsx_import_source: Option, maybe_jsx_import_source_module: Option, maybe_vendor_specifier: Option, - // no_npm: bool, - // npm_registry_api: Arc, - // npm_resolution: Arc, - // package_json_deps_installer: Arc, + no_npm: bool, + npm_registry_api: Arc, + npm_resolution: Arc, + package_json_deps_installer: Arc, found_package_json_dep_flag: Arc, } +#[derive(Default)] +pub struct CliGraphResolverOptions<'a> { + pub maybe_jsx_import_source_config: Option, + pub maybe_import_map: Option>, + pub maybe_vendor_dir: Option<&'a PathBuf>, + pub no_npm: bool, +} + +impl CliGraphResolver { + pub fn new( + npm_registry_api: Arc, + npm_resolution: Arc, + package_json_deps_provider: Arc, + package_json_deps_installer: Arc, + options: CliGraphResolverOptions, + ) -> Self { + Self { + mapped_specifier_resolver: MappedSpecifierResolver { + maybe_import_map: options.maybe_import_map, + package_json_deps_provider, + }, + maybe_default_jsx_import_source: options + .maybe_jsx_import_source_config + .as_ref() + .and_then(|c| c.default_specifier.clone()), + maybe_jsx_import_source_module: options + .maybe_jsx_import_source_config + .map(|c| c.module), + maybe_vendor_specifier: options + .maybe_vendor_dir + .and_then(|v| ModuleSpecifier::from_directory_path(v).ok()), + no_npm: options.no_npm, + npm_registry_api, + npm_resolution, + package_json_deps_installer, + found_package_json_dep_flag: Default::default(), + } + } + + pub fn as_graph_resolver(&self) -> &dyn Resolver { + self + } + + pub fn as_graph_npm_resolver(&self) -> &dyn NpmResolver { + self + } + + pub async fn force_top_level_package_json_install(&self) -> Result<(), AnyError> { + self.package_json_deps_installer + .ensure_top_level_install() + .await + } + + pub async fn top_level_package_json_install_if_necessary(&self) -> Result<(), AnyError> { + if self.found_package_json_dep_flag.is_raised() { + self.force_top_level_package_json_install().await?; + } + Ok(()) + } +} + impl Default for CliGraphResolver { fn default() -> Self { // This is not ideal, but necessary for the LSP. In the future, we should // refactor the LSP and force this to be initialized. - // let npm_registry_api = Arc::new(CliNpmRegistryApi::new_uninitialized()); - // let npm_resolution = Arc::new(NpmResolution::from_serialized( - // npm_registry_api.clone(), - // None, - // None, - // )); + let emitter_factory = EmitterFactory::new(); + let npm_registry_api = emitter_factory.npm_api().clone(); + let npm_resolution = Arc::new(NpmResolution::from_serialized( + npm_registry_api.clone(), + None, + None, + )); + Self { mapped_specifier_resolver: MappedSpecifierResolver { maybe_import_map: Default::default(), @@ -129,10 +204,10 @@ impl Default for CliGraphResolver { maybe_default_jsx_import_source: None, maybe_jsx_import_source_module: None, maybe_vendor_specifier: None, - // no_npm: false, - // npm_registry_api, - // npm_resolution, - // package_json_deps_installer: Default::default(), + no_npm: false, + npm_registry_api, + npm_resolution, + package_json_deps_installer: Default::default(), found_package_json_dep_flag: Default::default(), } } @@ -184,3 +259,65 @@ impl Resolver for CliGraphResolver { result } } + +impl NpmResolver for CliGraphResolver { + fn resolve_builtin_node_module( + &self, + specifier: &ModuleSpecifier, + ) -> Result, UnknownBuiltInNodeModuleError> { + if specifier.scheme() != "node" { + return Ok(None); + } + + let module_name = specifier.path().to_string(); + if is_builtin_node_module(&module_name) { + Ok(Some(module_name)) + } else { + Err(UnknownBuiltInNodeModuleError { module_name }) + } + } + + fn load_and_cache_npm_package_info( + &self, + package_name: &str, + ) -> LocalBoxFuture<'static, Result<(), AnyError>> { + if self.no_npm { + // return it succeeded and error at the import site below + return Box::pin(deno_core::futures::future::ready(Ok(()))); + } + // this will internally cache the package information + let package_name = package_name.to_string(); + let api = self.npm_registry_api.clone(); + + async move { + api.package_info(&package_name) + .await + .map(|_| ()) + .map_err(|err| err.into()) + } + .boxed() + } + + fn resolve_npm(&self, package_req: &PackageReq) -> NpmPackageReqResolution { + if self.no_npm { + return NpmPackageReqResolution::Err(anyhow!( + "npm specifiers were requested; but --no-npm is specified" + )); + } + + let result = self + .npm_resolution + .resolve_package_req_as_pending(package_req); + match result { + Ok(nv) => NpmPackageReqResolution::Ok(nv), + Err(err) => { + if self.npm_registry_api.mark_force_reload() { + println!("Restarting npm specifier resolution to check for new registry information. Error: {:#}", err); + NpmPackageReqResolution::ReloadRegistryInfo(err.into()) + } else { + NpmPackageReqResolution::Err(err.into()) + } + } + } + } +} diff --git a/crates/base/src/utils/graph_util.rs b/crates/base/src/utils/graph_util.rs index c8ecc5d5d..c5923283f 100644 --- a/crates/base/src/utils/graph_util.rs +++ b/crates/base/src/utils/graph_util.rs @@ -1,12 +1,20 @@ use crate::errors_rt::get_error_class_name; use crate::js_worker::emitter::EmitterFactory; +use crate::utils::graph_resolver::CliGraphResolver; use crate::utils::graph_util::deno_graph::ModuleError; use crate::utils::graph_util::deno_graph::ResolutionError; use deno_core::error::{custom_error, AnyError}; +use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; -use eszip::deno_graph; -use eszip::deno_graph::{ModuleGraph, ModuleGraphError}; +use deno_semver::package::{PackageNv, PackageReq}; +use eszip::deno_graph::source::Loader; +use eszip::deno_graph::{GraphKind, ModuleGraph, ModuleGraphError}; +use eszip::{deno_graph, EszipV2}; +use module_fetcher::args::lockfile::Lockfile; +use module_fetcher::cache::ParsedSourceCache; +use sb_npm::CliNpmResolver; use std::path::PathBuf; +use std::sync::Arc; #[derive(Clone, Copy)] pub struct GraphValidOptions { @@ -33,6 +41,189 @@ pub fn graph_valid_with_cli_options( ) } +pub struct ModuleGraphBuilder { + resolver: Arc, + npm_resolver: Arc, + parsed_source_cache: Arc, + lockfile: Option>>, + type_check: bool, // type_checker: Arc, + emitter_factory: Arc, +} + +impl ModuleGraphBuilder { + pub fn new(emitter_factory: Arc, type_check: bool) -> Self { + let lockfile = emitter_factory.get_lock_file(); + let graph_resolver = emitter_factory.cli_graph_resolver().clone(); + let npm_resolver = emitter_factory.npm_resolver().clone(); + let parsed_source_cache = emitter_factory.parsed_source_cache().unwrap(); + Self { + resolver: graph_resolver, + npm_resolver, + parsed_source_cache, + lockfile, + type_check, + emitter_factory, + } + } + + pub async fn create_graph_with_loader( + &self, + graph_kind: GraphKind, + roots: Vec, + loader: &mut dyn Loader, + ) -> Result { + let cli_resolver = self.resolver.clone(); + let graph_resolver = cli_resolver.as_graph_resolver(); + let graph_npm_resolver = cli_resolver.as_graph_npm_resolver(); + let analyzer = self.parsed_source_cache.as_analyzer(); + + let mut graph = ModuleGraph::new(graph_kind); + self.build_graph_with_npm_resolution( + &mut graph, + roots, + loader, + deno_graph::BuildOptions { + is_dynamic: false, + imports: vec![], + resolver: Some(graph_resolver), + npm_resolver: Some(graph_npm_resolver), + module_analyzer: Some(&*analyzer), + reporter: None, + // todo(dsherret): workspace support + workspace_members: vec![], + }, + ) + .await?; + + if graph.has_node_specifier && self.type_check { + self.npm_resolver + .inject_synthetic_types_node_package() + .await?; + } + + Ok(graph) + } + + pub async fn build_graph_with_npm_resolution<'a>( + &self, + graph: &mut ModuleGraph, + roots: Vec, + loader: &mut dyn deno_graph::source::Loader, + options: deno_graph::BuildOptions<'a>, + ) -> Result<(), AnyError> { + // TODO: Option here similar to: https://github.com/denoland/deno/blob/v1.37.1/cli/graph_util.rs#L323C5-L405C11 + // self.resolver.force_top_level_package_json_install().await?; TODO + // add the lockfile redirects to the graph if it's the first time executing + if graph.redirects.is_empty() { + if let Some(lockfile) = &self.lockfile { + let lockfile = lockfile.lock(); + for (from, to) in &lockfile.content.redirects { + if let Ok(from) = ModuleSpecifier::parse(from) { + if let Ok(to) = ModuleSpecifier::parse(to) { + if !matches!(from.scheme(), "file" | "npm" | "jsr") { + graph.redirects.insert(from, to); + } + } + } + } + } + } + + // add the jsr specifiers to the graph if it's the first time executing + if graph.packages.is_empty() { + if let Some(lockfile) = &self.lockfile { + let lockfile = lockfile.lock(); + for (key, value) in &lockfile.content.packages.specifiers { + if let Some(key) = key + .strip_prefix("jsr:") + .and_then(|key| PackageReq::from_str(key).ok()) + { + if let Some(value) = value + .strip_prefix("jsr:") + .and_then(|value| PackageNv::from_str(value).ok()) + { + graph.packages.add(key, value); + } + } + } + } + } + + graph.build(roots, loader, options).await; + + // add the redirects in the graph to the lockfile + if !graph.redirects.is_empty() { + if let Some(lockfile) = &self.lockfile { + let graph_redirects = graph + .redirects + .iter() + .filter(|(from, _)| !matches!(from.scheme(), "npm" | "file" | "deno")); + let mut lockfile = lockfile.lock(); + for (from, to) in graph_redirects { + lockfile.insert_redirect(from.to_string(), to.to_string()); + } + } + } + + // add the jsr specifiers in the graph to the lockfile + if !graph.packages.is_empty() { + if let Some(lockfile) = &self.lockfile { + let mappings = graph.packages.mappings(); + let mut lockfile = lockfile.lock(); + for (from, to) in mappings { + lockfile + .insert_package_specifier(format!("jsr:{}", from), format!("jsr:{}", to)); + } + } + } + + // ensure that the top level package.json is installed if a + // specifier was matched in the package.json + self.resolver + .top_level_package_json_install_if_necessary() + .await?; + + // resolve the dependencies of any pending dependencies + // that were inserted by building the graph + self.npm_resolver.resolve_pending().await?; + + Ok(()) + } + + #[allow(clippy::borrow_deref_ref)] + pub async fn create_graph_and_maybe_check( + &self, + roots: Vec, + ) -> Result { + // + let mut cache = self.emitter_factory.file_fetcher_loader(); + let cli_resolver = self.resolver.clone(); + let graph_resolver = cli_resolver.as_graph_resolver(); + let graph_npm_resolver = cli_resolver.as_graph_npm_resolver(); + let analyzer = self.parsed_source_cache.as_analyzer(); + let graph_kind = deno_graph::GraphKind::CodeOnly; + let mut graph = ModuleGraph::new(graph_kind); + + self.build_graph_with_npm_resolution( + &mut graph, + roots, + cache.as_mut(), + deno_graph::BuildOptions { + is_dynamic: false, + imports: vec![], + resolver: Some(&*graph_resolver), + npm_resolver: Some(&*graph_npm_resolver), + module_analyzer: Some(&*analyzer), + reporter: None, + workspace_members: vec![], + }, + ) + .await?; + + Ok(graph) + } +} + /// Check if `roots` and their deps are available. Returns `Ok(())` if /// so. Returns `Err(_)` if there is a known module graph or resolution /// error statically reachable from `roots`. @@ -100,106 +291,38 @@ pub fn graph_valid( } } -pub async fn build_graph_with_npm_resolution<'a>( - graph: &mut ModuleGraph, - roots: Vec, - loader: &mut dyn deno_graph::source::Loader, - options: deno_graph::BuildOptions<'a>, -) -> Result<(), AnyError> { - //TODO: NPM resolvers - - // ensure an "npm install" is done if the user has explicitly - // opted into using a node_modules directory - // if self.options.node_modules_dir_enablement() == Some(true) { - // self.resolver.force_top_level_package_json_install().await?; - // } - - graph.build(roots, loader, options).await; - - // ensure that the top level package.json is installed if a - // specifier was matched in the package.json - // self - // .resolver - // .top_level_package_json_install_if_necessary() - // .await?; - - // resolve the dependencies of any pending dependencies - // that were inserted by building the graph - // self.npm_resolver.resolve_pending().await?; - - Ok(()) -} - -pub async fn create_graph_and_maybe_check( - roots: Vec, -) -> Result { - let emitter_factory = EmitterFactory::new(); +#[allow(clippy::arc_with_non_send_sync)] +pub async fn create_eszip_from_graph_raw( + graph: ModuleGraph, + emitter_factory: Option>, +) -> EszipV2 { + let emitter = emitter_factory.unwrap_or_else(|| Arc::new(EmitterFactory::new())); + let parser_arc = emitter.clone().parsed_source_cache().unwrap(); + let parser = parser_arc.as_capturing_parser(); - let mut cache = emitter_factory.file_fetcher_loader(); - let analyzer = emitter_factory.parsed_source_cache().unwrap().as_analyzer(); - let graph_kind = deno_graph::GraphKind::CodeOnly; - let mut graph = ModuleGraph::new(graph_kind); - let graph_resolver = emitter_factory.graph_resolver(); + let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default()); - build_graph_with_npm_resolution( - &mut graph, - roots, - cache.as_mut(), - deno_graph::BuildOptions { - is_dynamic: false, - imports: vec![], - resolver: Some(&*graph_resolver), - npm_resolver: None, - module_analyzer: Some(&*analyzer), - reporter: None, - workspace_members: vec![], - }, - ) - .await?; - - //let graph = Arc::new(graph); - // graph_valid_with_cli_options(&graph, &graph.roots, &self.options)?; - // if let Some(lockfile) = &self.lockfile { - // graph_lock_or_exit(&graph, &mut lockfile.lock()); - // } - - // if self.options.type_check_mode().is_true() { - // self - // .type_checker - // .check( - // graph.clone(), - // check::CheckOptions { - // lib: self.options.ts_type_lib_window(), - // log_ignored_options: true, - // reload: true, // TODO: ? - // }, - // ) - // .await?; - // } - - Ok(graph) + eszip.unwrap() } -pub async fn create_module_graph_from_path( - main_service_path: &str, -) -> Result> { - let index = PathBuf::from(main_service_path); - let binding = std::fs::canonicalize(&index)?; - let specifier = binding.to_str().ok_or("Failed to convert path to string")?; +pub async fn create_graph(file: PathBuf, emitter_factory: Arc) -> ModuleGraph { + let binding = std::fs::canonicalize(&file).unwrap(); + let specifier = binding.to_str().unwrap(); let format_specifier = format!("file:///{}", specifier); - let module_specifier = ModuleSpecifier::parse(&format_specifier)?; - let graph = create_graph_and_maybe_check(vec![module_specifier]) - .await - .unwrap(); - Ok(graph) -} + let module_specifier = ModuleSpecifier::parse(&format_specifier).unwrap(); -pub async fn create_eszip_from_graph(graph: ModuleGraph) -> Vec { - let emitter = EmitterFactory::new(); - let parser_arc = emitter.parsed_source_cache().unwrap(); - let parser = parser_arc.as_capturing_parser(); + let builder = ModuleGraphBuilder::new(emitter_factory, false); - let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default()); + let create_module_graph_task = builder.create_graph_and_maybe_check(vec![module_specifier]); + create_module_graph_task.await.unwrap() +} - eszip.unwrap().into_bytes() +pub async fn create_graph_from_specifiers( + specifiers: Vec, + _is_dynamic: bool, + maybe_emitter_factory: Arc, +) -> Result { + let builder = ModuleGraphBuilder::new(maybe_emitter_factory, false); + let create_module_graph_task = builder.create_graph_and_maybe_check(specifiers); + create_module_graph_task.await } diff --git a/crates/base/test_cases/eszip-silly-test/index.ts b/crates/base/test_cases/eszip-silly-test/index.ts index be39adf9b..19069399c 100644 --- a/crates/base/test_cases/eszip-silly-test/index.ts +++ b/crates/base/test_cases/eszip-silly-test/index.ts @@ -1,3 +1,5 @@ -import { is_even } from "https://deno.land/x/is_even@v1.0/mod.ts" +import isEven from "npm:is-even"; -globalThis.isTenEven = is_even(10); \ No newline at end of file +console.log('Hello A'); +globalThis.isTenEven = isEven(10); +console.log('Hello'); \ No newline at end of file diff --git a/crates/base/test_cases/npm/index.ts b/crates/base/test_cases/npm/index.ts new file mode 100644 index 000000000..387ed07f2 --- /dev/null +++ b/crates/base/test_cases/npm/index.ts @@ -0,0 +1,9 @@ +import isEven from "npm:is-even"; +import { serve } from "https://deno.land/std@0.131.0/http/server.ts" + +serve(async (req: Request) => { + return new Response( + JSON.stringify({ is_even: isEven(10) }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ) +}) \ No newline at end of file diff --git a/crates/base/tests/user_worker_tests.rs b/crates/base/tests/user_worker_tests.rs index eea6f2f28..52614dd3e 100644 --- a/crates/base/tests/user_worker_tests.rs +++ b/crates/base/tests/user_worker_tests.rs @@ -39,3 +39,37 @@ async fn test_user_worker_json_imports() { assert_eq!(body_bytes, r#"{"version":"1.0.0"}"#); } + +#[tokio::test] +async fn test_user_imports_npm() { + let user_rt_opts = UserWorkerRuntimeOpts::default(); + let opts = WorkerContextInitOpts { + service_path: "./test_cases/npm".into(), + no_module_cache: false, + import_map_path: None, + env_vars: HashMap::new(), + events_rx: None, + maybe_eszip: None, + maybe_entrypoint: None, + maybe_module_code: None, + conf: WorkerRuntimeOpts::UserWorker(user_rt_opts), + }; + let worker_req_tx = create_worker(opts).await.unwrap(); + let (res_tx, res_rx) = oneshot::channel::, hyper::Error>>(); + + let req = Request::builder() + .uri("/") + .method("GET") + .body(Body::empty()) + .unwrap(); + + let msg = WorkerRequestMsg { req, res_tx }; + let _ = worker_req_tx.send(msg); + + let res = res_rx.await.unwrap().unwrap(); + assert!(res.status().as_u16() == 200); + + let body_bytes = hyper::body::to_bytes(res.into_body()).await.unwrap(); + + assert_eq!(body_bytes, r#"{"is_even":true}"#); +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index aa38e8006..305db9471 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -2,12 +2,15 @@ mod logger; use anyhow::Error; use base::commands::start_server; +use base::js_worker::emitter::EmitterFactory; use base::server::WorkerEntrypoints; -use base::utils::graph_util::{create_eszip_from_graph, create_module_graph_from_path}; +use base::standalone::binary::generate_binary_eszip; use clap::builder::FalseyValueParser; use clap::{arg, crate_version, value_parser, ArgAction, Command}; use std::fs::File; use std::io::Write; +use std::path::PathBuf; +use std::sync::Arc; fn cli() -> Command { Command::new("edge-runtime") @@ -83,6 +86,7 @@ fn main() -> Result<(), anyhow::Error> { } #[allow(clippy::single_match)] + #[allow(clippy::arc_with_non_send_sync)] match matches.subcommand() { Some(("start", sub_matches)) => { let ip = sub_matches.get_one::("ip").cloned().unwrap(); @@ -127,13 +131,16 @@ fn main() -> Result<(), anyhow::Error> { .cloned() .unwrap(); - let create_graph_from_path = - create_module_graph_from_path(entry_point_path.as_str()) - .await - .unwrap(); - let create_eszip = create_eszip_from_graph(create_graph_from_path).await; + let path = PathBuf::from(entry_point_path.as_str()); + let eszip = generate_binary_eszip( + path.canonicalize().unwrap(), + Arc::new(EmitterFactory::new()), + ) + .await?; + let bin = eszip.into_bytes(); + let mut file = File::create(output_path.as_str()).unwrap(); - file.write_all(&create_eszip).unwrap(); + file.write_all(&bin).unwrap(); } _ => { // unrecognized command diff --git a/crates/module_fetcher/src/lib.rs b/crates/module_fetcher/src/lib.rs index 38edcf300..1e8253a82 100644 --- a/crates/module_fetcher/src/lib.rs +++ b/crates/module_fetcher/src/lib.rs @@ -5,7 +5,6 @@ pub mod emit; pub mod file_fetcher; pub mod http_util; pub mod node; -pub mod npm; pub mod permissions; pub mod util; pub mod version; diff --git a/crates/node/clippy.toml b/crates/node/clippy.toml index 02fd259d0..31d9d7d47 100644 --- a/crates/node/clippy.toml +++ b/crates/node/clippy.toml @@ -38,6 +38,3 @@ disallowed-methods = [ { path = "std::fs::symlink_metadata", reason = "File system operations should be done using FileSystem trait" }, { path = "std::fs::write", reason = "File system operations should be done using FileSystem trait" }, ] -disallowed-types = [ - { path = "std::sync::Arc", reason = "use deno_fs::sync::MaybeArc instead" }, -] diff --git a/crates/node/global.rs b/crates/node/global.rs index 5a6d5302f..e7a51cb69 100644 --- a/crates/node/global.rs +++ b/crates/node/global.rs @@ -248,16 +248,18 @@ fn is_managed_key(scope: &mut v8::HandleScope, key: v8::Local) -> bool fn current_mode(scope: &mut v8::HandleScope) -> Mode { let Some(v8_string) = v8::StackTrace::current_script_name_or_source_url(scope) else { + println!("current_script_name_or_source_url, Using Deno"); return Mode::Deno; }; let op_state = deno_core::JsRuntime::op_state_from(scope); let op_state = op_state.borrow(); let Some(node_resolver) = op_state.try_borrow::>() else { + println!("Node resolver not available, using Deno"); return Mode::Deno; }; let mut buffer = [MaybeUninit::uninit(); 2048]; let str = v8_string.to_rust_cow_lossy(scope, &mut buffer); - if node_resolver.in_npm_package_with_cache(str) { + if node_resolver.in_npm_package_with_cache(str.clone()) { Mode::Node } else { Mode::Deno diff --git a/crates/node/lib.rs b/crates/node/lib.rs index 856613586..9c1ef5c36 100644 --- a/crates/node/lib.rs +++ b/crates/node/lib.rs @@ -4,6 +4,7 @@ use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; +use std::sync::Arc; use deno_core::error::AnyError; use deno_core::located_script_name; @@ -50,7 +51,7 @@ pub trait NodePermissions { fn check_sys(&self, kind: &str, api_name: &str) -> Result<(), AnyError>; } -pub(crate) struct AllowAllNodePermissions; +pub struct AllowAllNodePermissions; impl NodePermissions for AllowAllNodePermissions { fn check_net_url(&mut self, _url: &Url, _api_name: &str) -> Result<(), AnyError> { @@ -586,3 +587,7 @@ pub fn load_cjs_module( js_runtime.execute_script(located_script_name!(), source_code)?; Ok(()) } + +pub fn allow_all() -> Arc { + Arc::new(AllowAllNodePermissions) +} diff --git a/crates/npm/Cargo.toml b/crates/npm/Cargo.toml new file mode 100644 index 000000000..111542b22 --- /dev/null +++ b/crates/npm/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "sb_npm" +version = "0.1.0" +authors = ["Supabase "] +edition = "2021" +resolver = "2" +description = "We'll take care of this later" +license = "MIT" + +[lib] +path = "lib.rs" + +[dependencies] +deno_ast.workspace = true +deno_core.workspace = true +deno_npm.workspace = true +deno_fs.workspace = true +deno_semver.workspace = true +once_cell.workspace = true +async-trait.workspace = true +base32 = "=0.4.0" +module_fetcher = { path = "../module_fetcher" } +deno_lockfile.workspace = true +tar.workspace = true +flate2.workspace = true +ring.workspace = true +sb_node = { version = "0.1.0", path = "../node" } +serde.workspace = true +percent-encoding = "=2.3.0" +hex = "0.4" +base64 = { version = "=0.13.1" } +bincode = "=1.3.3" \ No newline at end of file diff --git a/crates/module_fetcher/src/npm/cache.rs b/crates/npm/cache.rs similarity index 83% rename from crates/module_fetcher/src/npm/cache.rs rename to crates/npm/cache.rs index 79a776598..bf924474b 100644 --- a/crates/module_fetcher/src/npm/cache.rs +++ b/crates/npm/cache.rs @@ -18,12 +18,10 @@ use deno_npm::registry::NpmPackageVersionDistInfo; use deno_npm::NpmPackageCacheFolderId; use deno_semver::package::PackageNv; use deno_semver::Version; - -use crate::args::CacheSetting; -use crate::http_util::HttpClient; -use crate::util::fs::canonicalize_path; -use crate::util::fs::hard_link_dir_recursive; -use crate::util::path::root_url_to_safe_local_dirname; +use module_fetcher::args::CacheSetting; +use module_fetcher::http_util::HttpClient; +use module_fetcher::util::fs::{canonicalize_path, hard_link_dir_recursive}; +use module_fetcher::util::path::root_url_to_safe_local_dirname; use super::tarball::verify_and_extract_tarball; @@ -430,89 +428,3 @@ pub fn mixed_case_package_name_decode(name: &str) -> Option { base32::decode(base32::Alphabet::RFC4648 { padding: false }, name) .and_then(|b| String::from_utf8(b).ok()) } - -#[cfg(test)] -mod test { - use deno_core::url::Url; - use deno_semver::package::PackageNv; - use deno_semver::Version; - - use super::NpmCacheDir; - use crate::npm::cache::NpmPackageCacheFolderId; - - #[test] - fn should_get_package_folder() { - let deno_dir = crate::cache::DenoDir::new(None).unwrap(); - let root_dir = deno_dir.npm_folder_path(); - let cache = NpmCacheDir::new(root_dir.clone()); - let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 1, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5_1"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_jjju6tq") - .join("2.1.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "@types/JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_ib2hs4dfomxuuu2pjy") - .join("2.1.5"), - ); - } -} diff --git a/crates/module_fetcher/src/npm/installer.rs b/crates/npm/installer.rs similarity index 92% rename from crates/module_fetcher/src/npm/installer.rs rename to crates/npm/installer.rs index b2c96913a..e93df1d18 100644 --- a/crates/module_fetcher/src/npm/installer.rs +++ b/crates/npm/installer.rs @@ -9,9 +9,8 @@ use deno_core::futures::StreamExt; use deno_npm::registry::NpmRegistryApi; use deno_npm::registry::NpmRegistryPackageInfoLoadError; use deno_semver::package::PackageReq; - -use crate::args::package_json::PackageJsonDepsProvider; -use crate::util::sync::AtomicFlag; +use module_fetcher::args::package_json::PackageJsonDepsProvider; +use module_fetcher::util::sync::AtomicFlag; use super::CliNpmRegistryApi; use super::NpmResolution; @@ -92,7 +91,7 @@ impl PackageJsonDepsInstaller { .resolve_pkg_id_from_pkg_req(req) .is_ok() }) { - log::debug!("All package.json deps resolvable. Skipping top level install."); + println!("All package.json deps resolvable. Skipping top level install."); return Ok(()); // everything is already resolvable } @@ -105,7 +104,7 @@ impl PackageJsonDepsInstaller { .resolve_package_req_as_pending_with_info(req, &info); if let Err(err) = result { if inner.npm_registry_api.mark_force_reload() { - log::debug!("Failed to resolve package. Retrying. Error: {err:#}"); + println!("Failed to resolve package. Retrying. Error: {err:#}"); // re-initialize reqs_with_info_futures = inner.reqs_with_info_futures(&package_reqs); } else { diff --git a/crates/module_fetcher/src/npm/mod.rs b/crates/npm/lib.rs similarity index 78% rename from crates/module_fetcher/src/npm/mod.rs rename to crates/npm/lib.rs index 41eb09a57..bd4c13c28 100644 --- a/crates/module_fetcher/src/npm/mod.rs +++ b/crates/npm/lib.rs @@ -1,11 +1,11 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -mod cache; -mod installer; -mod registry; -mod resolution; -mod resolvers; -mod tarball; +pub mod cache; +pub mod installer; +pub mod registry; +pub mod resolution; +pub mod resolvers; +pub mod tarball; pub use cache::NpmCache; pub use cache::NpmCacheDir; diff --git a/crates/module_fetcher/src/npm/registry.rs b/crates/npm/registry.rs similarity index 96% rename from crates/module_fetcher/src/npm/registry.rs rename to crates/npm/registry.rs index 995189572..99abb0147 100644 --- a/crates/module_fetcher/src/npm/registry.rs +++ b/crates/npm/registry.rs @@ -21,14 +21,13 @@ use deno_core::url::Url; use deno_npm::registry::NpmPackageInfo; use deno_npm::registry::NpmRegistryApi; use deno_npm::registry::NpmRegistryPackageInfoLoadError; +use module_fetcher::args::CacheSetting; +use module_fetcher::cache::CACHE_PERM; +use module_fetcher::http_util::HttpClient; +use module_fetcher::util::fs::atomic_write_file; +use module_fetcher::util::sync::AtomicFlag; use once_cell::sync::Lazy; -use crate::args::CacheSetting; -use crate::cache::CACHE_PERM; -use crate::http_util::HttpClient; -use crate::util::fs::atomic_write_file; -use crate::util::sync::AtomicFlag; - use super::cache::NpmCache; static NPM_REGISTRY_DEFAULT_URL: Lazy = Lazy::new(|| { @@ -41,7 +40,7 @@ static NPM_REGISTRY_DEFAULT_URL: Lazy = Lazy::new(|| { return url; } Err(err) => { - log::debug!("Invalid {} environment variable: {:#}", env_var_name, err,); + println!("Invalid {} environment variable: {:#}", env_var_name, err,); } } } @@ -242,10 +241,9 @@ impl CliNpmRegistryApiInner { // This scenario might mean we need to load more data from the // npm registry than before. So, just debug log while in debug // rather than panic. - log::debug!( + println!( "error deserializing registry.json for '{}'. Reloading. {:?}", - name, - err + name, err ); Ok(None) } @@ -290,6 +288,7 @@ impl CliNpmRegistryApiInner { &self, name: &str, ) -> Result, AnyError> { + println!("Downloading load_package_info_from_registry_inner"); if *self.cache.cache_setting() == CacheSetting::Only { return Err(custom_error( "NotCached", diff --git a/crates/module_fetcher/src/npm/resolution.rs b/crates/npm/resolution.rs similarity index 97% rename from crates/module_fetcher/src/npm/resolution.rs rename to crates/npm/resolution.rs index 70d334867..7aa8b1eb8 100644 --- a/crates/module_fetcher/src/npm/resolution.rs +++ b/crates/npm/resolution.rs @@ -30,11 +30,10 @@ use deno_npm::NpmSystemInfo; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use deno_semver::VersionReq; - -use crate::args::lockfile::Lockfile; -use crate::util::sync::TaskQueue; +use module_fetcher::util::sync::TaskQueue; use super::registry::CliNpmRegistryApi; +use module_fetcher::args::lockfile::Lockfile; /// Handles updating and storing npm resolution in memory where the underlying /// snapshot can be updated concurrently. Additionally handles updating the lockfile @@ -293,7 +292,7 @@ async fn add_package_reqs_to_snapshot( .iter() .all(|req| snapshot.package_reqs().contains_key(req)) { - log::debug!("Snapshot already up to date. Skipping pending resolution."); + println!("Snapshot already up to date. Skipping pending resolution."); snapshot } else { let pending_resolver = get_npm_pending_resolver(api); @@ -304,8 +303,8 @@ async fn add_package_reqs_to_snapshot( match result { Ok(snapshot) => snapshot, Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => { - log::debug!("{err:#}"); - log::debug!("npm resolution failed. Trying again..."); + println!("{err:#}"); + println!("npm resolution failed. Trying again..."); // try again let snapshot = get_new_snapshot(); diff --git a/crates/module_fetcher/src/npm/resolvers/common.rs b/crates/npm/resolvers/common.rs similarity index 93% rename from crates/module_fetcher/src/npm/resolvers/common.rs rename to crates/npm/resolvers/common.rs index e011f2e4b..d3ff31485 100644 --- a/crates/module_fetcher/src/npm/resolvers/common.rs +++ b/crates/npm/resolvers/common.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; +use crate::NpmCache; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; @@ -20,8 +21,6 @@ use deno_npm::NpmResolutionPackage; use sb_node::NodePermissions; use sb_node::NodeResolutionMode; -use crate::npm::NpmCache; - /// Part of the resolution that interacts with the file system. #[async_trait] pub trait NpmPackageFsResolver: Send + Sync { @@ -153,17 +152,3 @@ pub fn types_package_name(package_name: &str) -> String { // https://github.com/DefinitelyTyped/DefinitelyTyped/tree/15f1ece08f7b498f4b9a2147c2a46e94416ca777#what-about-scoped-packages format!("@types/{}", package_name.replace('/', "__")) } - -#[cfg(test)] -mod test { - use super::types_package_name; - - #[test] - fn test_types_package_name() { - assert_eq!(types_package_name("name"), "@types/name"); - assert_eq!( - types_package_name("@scoped/package"), - "@types/@scoped__package" - ); - } -} diff --git a/crates/module_fetcher/src/npm/resolvers/global.rs b/crates/npm/resolvers/global.rs similarity index 97% rename from crates/module_fetcher/src/npm/resolvers/global.rs rename to crates/npm/resolvers/global.rs index 3f4482e04..d30d569b4 100644 --- a/crates/module_fetcher/src/npm/resolvers/global.rs +++ b/crates/npm/resolvers/global.rs @@ -20,9 +20,9 @@ use deno_npm::NpmSystemInfo; use sb_node::NodePermissions; use sb_node::NodeResolutionMode; -use crate::npm::resolution::NpmResolution; -use crate::npm::resolvers::common::cache_packages; -use crate::npm::NpmCache; +use crate::resolution::NpmResolution; +use crate::resolvers::common::cache_packages; +use crate::NpmCache; use super::common::types_package_name; use super::common::NpmPackageFsResolver; diff --git a/crates/module_fetcher/src/npm/resolvers/local.rs b/crates/npm/resolvers/local.rs similarity index 97% rename from crates/module_fetcher/src/npm/resolvers/local.rs rename to crates/npm/resolvers/local.rs index e38f82dbe..b9faa0112 100644 --- a/crates/module_fetcher/src/npm/resolvers/local.rs +++ b/crates/npm/resolvers/local.rs @@ -11,12 +11,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use crate::cache::CACHE_PERM; -use crate::npm::cache::mixed_case_package_name_decode; -use crate::util::fs::atomic_write_file; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; -use crate::util::fs::symlink_dir; -use crate::util::fs::LaxSingleProcessFsFlag; +use crate::cache::mixed_case_package_name_decode; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; @@ -33,17 +28,20 @@ use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; use deno_semver::package::PackageNv; +use module_fetcher::cache::CACHE_PERM; +use module_fetcher::util::fs::{ + atomic_write_file, canonicalize_path_maybe_not_exists_with_fs, copy_dir_recursive, + hard_link_dir_recursive, symlink_dir, LaxSingleProcessFsFlag, +}; use sb_node::NodePermissions; use sb_node::NodeResolutionMode; use sb_node::PackageJson; use serde::Deserialize; use serde::Serialize; -use crate::npm::cache::mixed_case_package_name_encode; -use crate::npm::resolution::NpmResolution; -use crate::npm::NpmCache; -use crate::util::fs::copy_dir_recursive; -use crate::util::fs::hard_link_dir_recursive; +use crate::cache::mixed_case_package_name_encode; +use crate::resolution::NpmResolution; +use crate::NpmCache; use super::common::types_package_name; use super::common::NpmPackageFsResolver; @@ -632,7 +630,7 @@ fn junction_or_symlink_dir(old_path: &Path, new_path: &Path) -> Result<(), AnyEr if cfg!(debug) { // When running the tests, junctions should be created, but if not then // surface this error. - log::warn!("Error creating junction. {:#}", junction_err); + println!("Error creating junction. {:#}", junction_err); } match symlink_dir(old_path, new_path) { diff --git a/crates/module_fetcher/src/npm/resolvers/mod.rs b/crates/npm/resolvers/mod.rs similarity index 94% rename from crates/module_fetcher/src/npm/resolvers/mod.rs rename to crates/npm/resolvers/mod.rs index 70703aecc..2978376af 100644 --- a/crates/module_fetcher/src/npm/resolvers/mod.rs +++ b/crates/npm/resolvers/mod.rs @@ -29,8 +29,8 @@ use sb_node::NpmResolver; use serde::Deserialize; use serde::Serialize; -use crate::args::lockfile::Lockfile; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use module_fetcher::args::lockfile::Lockfile; +use module_fetcher::util::fs::canonicalize_path_maybe_not_exists_with_fs; use self::local::LocalNpmPackageResolver; use super::resolution::NpmResolution; @@ -113,11 +113,6 @@ impl CliNpmResolver { .realpath_sync(path) .map_err(|err| err.into_io_error()) })?; - log::debug!( - "Resolved package folder of {} to {}", - pkg_id.as_serialized(), - path.display() - ); Ok(path) } @@ -134,11 +129,6 @@ impl CliNpmResolver { else { return Ok(None); }; - log::debug!( - "Resolved package folder of {} to {}", - specifier, - path.display() - ); Ok(Some(path)) } @@ -162,7 +152,7 @@ impl CliNpmResolver { /// Attempts to get the package size in bytes. pub fn package_size(&self, package_id: &NpmPackageId) -> Result { let package_folder = self.fs_resolver.package_folder(package_id)?; - Ok(crate::util::fs::dir_size(&package_folder)?) + Ok(module_fetcher::util::fs::dir_size(&package_folder)?) } /// Adds package requirements to the resolver and ensures everything is setup. @@ -243,7 +233,6 @@ impl NpmResolver for CliNpmResolver { let path = self .fs_resolver .resolve_package_folder_from_package(name, referrer, mode)?; - log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); Ok(path) } diff --git a/crates/module_fetcher/src/npm/tarball.rs b/crates/npm/tarball.rs similarity index 62% rename from crates/module_fetcher/src/npm/tarball.rs rename to crates/npm/tarball.rs index 09cfbe158..51c130a5b 100644 --- a/crates/module_fetcher/src/npm/tarball.rs +++ b/crates/npm/tarball.rs @@ -131,7 +131,7 @@ fn extract_tarball(data: &[u8], output_folder: &Path) -> Result<(), AnyError> { // symlinks to the npm registry. If ever adding symlink or hardlink // support, we will need to validate that the hardlink and symlink // target are within the package directory. - log::warn!( + println!( "Ignoring npm tarball entry type {:?} for '{}'", entry_type, absolute_path.display() @@ -144,98 +144,3 @@ fn extract_tarball(data: &[u8], output_folder: &Path) -> Result<(), AnyError> { } Ok(()) } - -#[cfg(test)] -mod test { - use deno_semver::Version; - - use super::*; - - #[test] - pub fn test_verify_tarball() { - let package = PackageNv { - name: "package".to_string(), - version: Version::parse_from_npm("1.0.0").unwrap(), - }; - let actual_checksum = - "z4phnx7vul3xvchq1m2ab9yg5aulvxxcg/spidns6c5h0ne8xyxysp+dgnkhfuwvy7kxvudbeoglodj6+sfapg=="; - assert_eq!( - verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::UnknownIntegrity("test") - ) - .unwrap_err() - .to_string(), - "Not implemented integrity kind for package@1.0.0: test", - ); - assert_eq!( - verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::Integrity { - algorithm: "notimplemented", - base64_hash: "test" - } - ) - .unwrap_err() - .to_string(), - "Not implemented hash function for package@1.0.0: notimplemented", - ); - assert_eq!( - verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::Integrity { - algorithm: "sha1", - base64_hash: "test" - } - ) - .unwrap_err() - .to_string(), - concat!( - "Tarball checksum did not match what was provided by npm ", - "registry for package@1.0.0.\n\nExpected: test\nActual: 2jmj7l5rsw0yvb/vlwaykk/ybwk=", - ), - ); - assert_eq!( - verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::Integrity { - algorithm: "sha512", - base64_hash: "test" - } - ) - .unwrap_err() - .to_string(), - format!("Tarball checksum did not match what was provided by npm registry for package@1.0.0.\n\nExpected: test\nActual: {actual_checksum}"), - ); - assert!(verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::Integrity { - algorithm: "sha512", - base64_hash: actual_checksum, - }, - ) - .is_ok()); - let actual_hex = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; - assert_eq!( - verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::LegacySha1Hex("test"), - ) - .unwrap_err() - .to_string(), - format!("Tarball checksum did not match what was provided by npm registry for package@1.0.0.\n\nExpected: test\nActual: {actual_hex}"), - ); - assert!(verify_tarball_integrity( - &package, - &Vec::new(), - &NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(actual_hex), - ) - .is_ok()); - } -} diff --git a/crates/sb_core/Cargo.toml b/crates/sb_core/Cargo.toml index 1f0c5a759..2ce94166c 100644 --- a/crates/sb_core/Cargo.toml +++ b/crates/sb_core/Cargo.toml @@ -11,6 +11,7 @@ license = "MIT" path = "lib.rs" [dependencies] +deno_ast.workspace = true deno_net.workspace = true deno_web.workspace = true deno_fetch.workspace = true diff --git a/crates/sb_core/js/bootstrap.js b/crates/sb_core/js/bootstrap.js index 3beb7e455..4dbb4cec4 100644 --- a/crates/sb_core/js/bootstrap.js +++ b/crates/sb_core/js/bootstrap.js @@ -298,5 +298,11 @@ globalThis.bootstrapSBEdge = (opts, isUserWorker, isEventsWorker, version) => { }); } + const nodeBootstrap = globalThis.nodeBootstrap; + if(nodeBootstrap) { + nodeBootstrap(false, undefined); + delete globalThis.nodeBootstrap; + } + delete globalThis.bootstrapSBEdge; }; diff --git a/crates/sb_core/lib.rs b/crates/sb_core/lib.rs index 0f9845468..96b410964 100644 --- a/crates/sb_core/lib.rs +++ b/crates/sb_core/lib.rs @@ -2,6 +2,7 @@ pub mod http_start; pub mod net; pub mod permissions; pub mod runtime; +pub mod transpiler; deno_core::extension!( sb_core_main_js, diff --git a/crates/sb_core/transpiler.rs b/crates/sb_core/transpiler.rs new file mode 100644 index 000000000..59155aef8 --- /dev/null +++ b/crates/sb_core/transpiler.rs @@ -0,0 +1,42 @@ +use deno_ast::{MediaType, ParseParams, SourceTextInfo}; +use deno_core::error::AnyError; +use deno_core::{ExtensionFileSource, ExtensionFileSourceCode}; +use std::path::Path; + +pub fn maybe_transpile_source( + source: &mut ExtensionFileSource, +) -> Result<&mut ExtensionFileSource, AnyError> { + let media_type = if source.specifier.starts_with("node:") { + MediaType::TypeScript + } else { + MediaType::from_path(Path::new(&source.specifier)) + }; + + match media_type { + MediaType::TypeScript => {} + MediaType::JavaScript => return Ok(source), + MediaType::Mjs => return Ok(source), + _ => panic!( + "Unsupported media type for snapshotting {media_type:?} for file {}", + source.specifier + ), + } + let code = source.load()?; + + let parsed = deno_ast::parse_module(ParseParams { + specifier: source.specifier.to_string(), + text_info: SourceTextInfo::from_string(code.as_str().to_owned()), + media_type, + capture_tokens: false, + scope_analysis: false, + maybe_syntax: None, + })?; + let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { + imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, + inline_source_map: false, + ..Default::default() + })?; + + source.code = ExtensionFileSourceCode::Computed(transpiled_source.text.into()); + Ok(source) +} diff --git a/deno.lock b/deno.lock new file mode 100644 index 000000000..e6377b07c --- /dev/null +++ b/deno.lock @@ -0,0 +1,41 @@ +{ + "version": "3", + "packages": { + "specifiers": { + "npm:is-even": "npm:is-even@1.0.0" + }, + "npm": { + "is-buffer@1.1.6": { + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dependencies": {} + }, + "is-even@1.0.0": { + "integrity": "sha512-LEhnkAdJqic4Dbqn58A0y52IXoHWlsueqQkKfMfdEnIYG8A1sm/GHidKkS6yvXlMoRrkM34csHnXQtOqcb+Jzg==", + "dependencies": { + "is-odd": "is-odd@0.1.2" + } + }, + "is-number@3.0.0": { + "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dependencies": { + "kind-of": "kind-of@3.2.2" + } + }, + "is-odd@0.1.2": { + "integrity": "sha512-Ri7C2K7o5IrUU9UEI8losXJCCD/UtsaIrkR5sxIcFg4xQ9cRJXlWA5DQvTE0yDc0krvSNLsRGXN11UPS6KyfBw==", + "dependencies": { + "is-number": "is-number@3.0.0" + } + }, + "kind-of@3.2.2": { + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dependencies": { + "is-buffer": "is-buffer@1.1.6" + } + } + } + }, + "remote": { + "https://deno.land/x/is_even@v1.0/mod.ts": "f0596cc34079796565f5fb7ebc0496c1dfe435f1039a7751e1922c89e6f85d6e" + } +} diff --git a/examples/express/index.ts b/examples/express/index.ts new file mode 100644 index 000000000..5085b6c18 --- /dev/null +++ b/examples/express/index.ts @@ -0,0 +1,9 @@ +import express from "npm:express@4.18.2"; + +const app = express(); + +app.get("/", (req, res) => { + res.send("Welcome to the Dinosaur API!"); +}); + +app.listen(8000); \ No newline at end of file diff --git a/examples/main-empty/index.ts b/examples/main-empty/index.ts index d76a6f810..ddfb96edb 100644 --- a/examples/main-empty/index.ts +++ b/examples/main-empty/index.ts @@ -1,10 +1,12 @@ import { serve } from "https://deno.land/std@0.131.0/http/server.ts"; +import {sum} from "./some-import.ts"; console.log(Deno.version); +let val = sum(1, 2); serve(async (req: Request) => { return new Response( JSON.stringify({ hello: "world" }), { status: 200, headers: { "Content-Type": "application/json" } }, ) -}) +}); \ No newline at end of file diff --git a/examples/main-empty/some-import.ts b/examples/main-empty/some-import.ts new file mode 100644 index 000000000..a2d7489c9 --- /dev/null +++ b/examples/main-empty/some-import.ts @@ -0,0 +1 @@ +export const sum = (n1, n2) => n1 + n2; \ No newline at end of file diff --git a/examples/npm/file.js b/examples/npm/file.js new file mode 100644 index 000000000..9001e5522 --- /dev/null +++ b/examples/npm/file.js @@ -0,0 +1 @@ +console.log(process.cwd()); \ No newline at end of file diff --git a/examples/npm/index.ts b/examples/npm/index.ts new file mode 100644 index 000000000..423d588e9 --- /dev/null +++ b/examples/npm/index.ts @@ -0,0 +1,9 @@ +import express from "npm:express@4.18.2"; + +const app = express(); + +app.get(/(.*)/, (req, res) => { + res.send("Welcome to Supabase"); +}); + +app.listen(8000); \ No newline at end of file diff --git a/examples/oak-issue/index.ts b/examples/oak-issue/index.ts new file mode 100644 index 000000000..8c2ff748f --- /dev/null +++ b/examples/oak-issue/index.ts @@ -0,0 +1,6 @@ +import { Application } from 'https://deno.land/x/oak/mod.ts'; +const app = new Application(); +app.use((ctx) => { + ctx.response.body = 'Hello world!'; +}); +await app.listen({ port: 8000 }); \ No newline at end of file