From 742c94ec059be1114eb62df769dc324e4afe62cc Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 27 Dec 2023 14:43:07 -0600 Subject: [PATCH 1/2] refactor(index): Delay initialize index --- src/ops/index.rs | 20 +++++++++++++------- src/steps/hook.rs | 2 +- src/steps/publish.rs | 2 +- src/steps/release.rs | 2 +- src/steps/replace.rs | 2 +- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/ops/index.rs b/src/ops/index.rs index 7a4639209..a73f7a184 100644 --- a/src/ops/index.rs +++ b/src/ops/index.rs @@ -1,18 +1,19 @@ use tame_index::krate::IndexKrate; use tame_index::utils::flock::FileLock; +#[derive(Default)] pub struct CratesIoIndex { - index: RemoteIndex, + index: Option, cache: std::collections::HashMap>, } impl CratesIoIndex { #[inline] - pub fn open() -> Result { - Ok(Self { - index: RemoteIndex::open()?, - cache: Default::default(), - }) + pub fn new() -> Self { + Self { + index: None, + cache: std::collections::HashMap::new(), + } } /// Determines if the specified crate exists in the crates.io index @@ -46,8 +47,13 @@ impl CratesIoIndex { return Ok(entry.clone()); } + if self.index.is_none() { + log::trace!("Connecting to index"); + self.index = Some(RemoteIndex::open()?); + } + let index = self.index.as_mut().unwrap(); log::trace!("Downloading index for {name}"); - let entry = self.index.krate(name)?; + let entry = index.krate(name)?; self.cache.insert(name.to_owned(), entry.clone()); Ok(entry) } diff --git a/src/steps/hook.rs b/src/steps/hook.rs index 6f57d3f89..abe6cb144 100644 --- a/src/steps/hook.rs +++ b/src/steps/hook.rs @@ -47,7 +47,7 @@ pub struct HookStep { impl HookStep { pub fn run(&self) -> Result<(), CliError> { git::git_version()?; - let mut index = crate::ops::index::CratesIoIndex::open()?; + let mut index = crate::ops::index::CratesIoIndex::new(); if self.dry_run { let _ = diff --git a/src/steps/publish.rs b/src/steps/publish.rs index c411ab6b6..9881dbc92 100644 --- a/src/steps/publish.rs +++ b/src/steps/publish.rs @@ -80,7 +80,7 @@ impl PublishStep { let mut pkgs = plan::plan(pkgs)?; - let mut index = crate::ops::index::CratesIoIndex::open()?; + let mut index = crate::ops::index::CratesIoIndex::new(); for pkg in pkgs.values_mut() { if pkg.config.registry().is_none() && pkg.config.release() { let crate_name = pkg.meta.name.as_str(); diff --git a/src/steps/release.rs b/src/steps/release.rs index 050c00e42..572c70451 100644 --- a/src/steps/release.rs +++ b/src/steps/release.rs @@ -46,7 +46,7 @@ pub struct ReleaseStep { impl ReleaseStep { pub fn run(&self) -> Result<(), CliError> { git::git_version()?; - let mut index = crate::ops::index::CratesIoIndex::open()?; + let mut index = crate::ops::index::CratesIoIndex::new(); if self.dry_run { let _ = diff --git a/src/steps/replace.rs b/src/steps/replace.rs index d63d27750..b5fe47004 100644 --- a/src/steps/replace.rs +++ b/src/steps/replace.rs @@ -43,7 +43,7 @@ pub struct ReplaceStep { impl ReplaceStep { pub fn run(&self) -> Result<(), CliError> { git::git_version()?; - let mut index = crate::ops::index::CratesIoIndex::open()?; + let mut index = crate::ops::index::CratesIoIndex::new(); if self.dry_run { let _ = From a536c386e6a016ab24cb055190a3bd665779cfe7 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 27 Dec 2023 15:04:47 -0600 Subject: [PATCH 2/2] fix(index): Don't talk to crates.io when using alt registry Ideally we'd find the proper registry and talk to it but that takes a lot more work to support. Until then, we'll just act like the registry has no packages in it and put in some specific hacks to support it. This should be less brittle than our previous approach which just hoped we sprinkled `config.registr().is_none()` in the right places. Fixes #732 --- src/ops/cargo.rs | 14 +++++++++--- src/ops/index.rs | 23 +++++++++++++++---- src/steps/hook.rs | 1 + src/steps/mod.rs | 3 ++- src/steps/publish.rs | 53 ++++++++++++++++++++++---------------------- src/steps/release.rs | 40 ++++++++++++++++++++++----------- src/steps/replace.rs | 1 + 7 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/ops/cargo.rs b/src/ops/cargo.rs index 7c19a8a96..d8a42d69e 100644 --- a/src/ops/cargo.rs +++ b/src/ops/cargo.rs @@ -112,18 +112,25 @@ pub fn publish( pub fn wait_for_publish( index: &mut crate::ops::index::CratesIoIndex, + registry: Option<&str>, name: &str, version: &str, timeout: std::time::Duration, dry_run: bool, ) -> CargoResult<()> { if !dry_run { + if registry.is_none() { + // HACK: `index` never reports crates as present for alternative registries + log::debug!("Not waiting for publish as that is only supported for crates.io; ensure you are using at least cargo v1.66 which will wait for you."); + return Ok(()); + } + let now = std::time::Instant::now(); let sleep_time = std::time::Duration::from_secs(1); let mut logged = false; loop { - index.update_krate(name); - if is_published(index, name, version) { + index.update_krate(registry, name); + if is_published(index, registry, name, version) { break; } else if timeout < now.elapsed() { anyhow::bail!("timeout waiting for crate to be published"); @@ -145,10 +152,11 @@ pub fn wait_for_publish( pub fn is_published( index: &mut crate::ops::index::CratesIoIndex, + registry: Option<&str>, name: &str, version: &str, ) -> bool { - match index.has_krate_version(name, version) { + match index.has_krate_version(registry, name, version) { Ok(has_krate_version) => has_krate_version.unwrap_or(false), Err(err) => { // For both http and git indices, this _might_ be an error that goes away in diff --git a/src/ops/index.rs b/src/ops/index.rs index a73f7a184..abae5be46 100644 --- a/src/ops/index.rs +++ b/src/ops/index.rs @@ -18,30 +18,45 @@ impl CratesIoIndex { /// Determines if the specified crate exists in the crates.io index #[inline] - pub fn has_krate(&mut self, name: &str) -> Result { - Ok(self.krate(name)?.map(|_| true).unwrap_or(false)) + pub fn has_krate( + &mut self, + registry: Option<&str>, + name: &str, + ) -> Result { + Ok(self.krate(registry, name)?.map(|_| true).unwrap_or(false)) } /// Determines if the specified crate version exists in the crates.io index #[inline] pub fn has_krate_version( &mut self, + registry: Option<&str>, name: &str, version: &str, ) -> Result, crate::error::CliError> { - let krate = self.krate(name)?; + let krate = self.krate(registry, name)?; Ok(krate.map(|ik| ik.versions.iter().any(|iv| iv.version == version))) } #[inline] - pub fn update_krate(&mut self, name: &str) { + pub fn update_krate(&mut self, registry: Option<&str>, name: &str) { + if registry.is_some() { + return; + } + self.cache.remove(name); } pub(crate) fn krate( &mut self, + registry: Option<&str>, name: &str, ) -> Result, crate::error::CliError> { + if let Some(registry) = registry { + log::trace!("Cannot connect to registry `{registry}`"); + return Ok(None); + } + if let Some(entry) = self.cache.get(name) { log::trace!("Reusing index for {name}"); return Ok(entry.clone()); diff --git a/src/steps/hook.rs b/src/steps/hook.rs index abe6cb144..22ad9bf89 100644 --- a/src/steps/hook.rs +++ b/src/steps/hook.rs @@ -88,6 +88,7 @@ impl HookStep { let version = &pkg.initial_version; if !crate::ops::cargo::is_published( &mut index, + pkg.config.registry(), crate_name, &version.full_version_string, ) { diff --git a/src/steps/mod.rs b/src/steps/mod.rs index 6003fa748..c35c41a69 100644 --- a/src/steps/mod.rs +++ b/src/steps/mod.rs @@ -225,9 +225,10 @@ pub fn verify_rate_limit( let mut new = 0; let mut existing = 0; for pkg in pkgs { + // Note: these rate limits are only known for default registry if pkg.config.registry().is_none() && pkg.config.publish() { let crate_name = pkg.meta.name.as_str(); - if index.has_krate(crate_name)? { + if index.has_krate(None, crate_name)? { existing += 1; } else { new += 1; diff --git a/src/steps/publish.rs b/src/steps/publish.rs index 9881dbc92..1119378cb 100644 --- a/src/steps/publish.rs +++ b/src/steps/publish.rs @@ -82,11 +82,12 @@ impl PublishStep { let mut index = crate::ops::index::CratesIoIndex::new(); for pkg in pkgs.values_mut() { - if pkg.config.registry().is_none() && pkg.config.release() { + if pkg.config.release() { let crate_name = pkg.meta.name.as_str(); let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version); if crate::ops::cargo::is_published( &mut index, + pkg.config.registry(), crate_name, &version.full_version_string, ) { @@ -200,33 +201,31 @@ pub fn publish( return Err(101.into()); } - if pkg.config.registry().is_none() { - let timeout = std::time::Duration::from_secs(300); - let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version); - crate::ops::cargo::wait_for_publish( - index, - crate_name, - &version.full_version_string, - timeout, - dry_run, - )?; - // HACK: Even once the index is updated, there seems to be another step before the publish is fully ready. - // We don't have a way yet to check for that, so waiting for now in hopes everything is ready - if !dry_run { - let publish_grace_sleep = std::env::var("PUBLISH_GRACE_SLEEP") - .unwrap_or_else(|_| Default::default()) - .parse() - .unwrap_or(0); - if 0 < publish_grace_sleep { - log::debug!( - "waiting an additional {} seconds for crates.io to update its indices...", - publish_grace_sleep - ); - std::thread::sleep(std::time::Duration::from_secs(publish_grace_sleep)); - } + let timeout = std::time::Duration::from_secs(300); + let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version); + crate::ops::cargo::wait_for_publish( + index, + pkg.config.registry(), + crate_name, + &version.full_version_string, + timeout, + dry_run, + )?; + // HACK: Even once the index is updated, there seems to be another step before the publish is fully ready. + // We don't have a way yet to check for that, so waiting for now in hopes everything is ready + if !dry_run { + let publish_grace_sleep = std::env::var("PUBLISH_GRACE_SLEEP") + .unwrap_or_else(|_| Default::default()) + .parse() + .unwrap_or(0); + if 0 < publish_grace_sleep { + log::debug!( + "waiting an additional {} seconds for {} to update its indices...", + publish_grace_sleep, + pkg.config.registry().unwrap_or("crates.io") + ); + std::thread::sleep(std::time::Duration::from_secs(publish_grace_sleep)); } - } else { - log::debug!("not waiting for publish because the registry is not crates.io and doesn't get updated automatically"); } } diff --git a/src/steps/release.rs b/src/steps/release.rs index 572c70451..52079edf1 100644 --- a/src/steps/release.rs +++ b/src/steps/release.rs @@ -73,7 +73,7 @@ impl ReleaseStep { pkg.bump(level_or_version, self.metadata.as_deref())?; } } - if index.has_krate(&pkg.meta.name)? { + if index.has_krate(pkg.config.registry(), &pkg.meta.name)? { // Already published, skip it. Use `cargo release owner` for one-time updates pkg.ensure_owners = false; } @@ -101,7 +101,12 @@ impl ReleaseStep { && !explicitly_excluded { let version = &pkg.initial_version; - if !cargo::is_published(&mut index, crate_name, &version.full_version_string) { + if !cargo::is_published( + &mut index, + pkg.config.registry(), + crate_name, + &version.full_version_string, + ) { log::debug!( "enabled {}, v{} is unpublished", crate_name, @@ -152,10 +157,16 @@ impl ReleaseStep { continue; }; + // HACK: `index` only supports default registry if pkg.config.publish() && pkg.config.registry().is_none() { let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version); let crate_name = pkg.meta.name.as_str(); - if !cargo::is_published(&mut index, crate_name, &version.full_version_string) { + if !cargo::is_published( + &mut index, + pkg.config.registry(), + crate_name, + &version.full_version_string, + ) { let _ = crate::ops::shell::warn(format!( "disabled by user, skipping {} v{} despite being unpublished", crate_name, version.full_version_string, @@ -195,16 +206,19 @@ impl ReleaseStep { if !pkg.config.publish() { continue; } - if pkg.config.registry().is_none() { - let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version); - let crate_name = pkg.meta.name.as_str(); - if cargo::is_published(&mut index, crate_name, &version.full_version_string) { - let _ = crate::ops::shell::error(format!( - "{} {} is already published", - crate_name, version.full_version_string - )); - double_publish = true; - } + let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version); + let crate_name = pkg.meta.name.as_str(); + if cargo::is_published( + &mut index, + pkg.config.registry(), + crate_name, + &version.full_version_string, + ) { + let _ = crate::ops::shell::error(format!( + "{} {} is already published", + crate_name, version.full_version_string + )); + double_publish = true; } } if double_publish { diff --git a/src/steps/replace.rs b/src/steps/replace.rs index b5fe47004..6a26a6d70 100644 --- a/src/steps/replace.rs +++ b/src/steps/replace.rs @@ -84,6 +84,7 @@ impl ReplaceStep { let version = &pkg.initial_version; if !crate::ops::cargo::is_published( &mut index, + pkg.config.registry(), crate_name, &version.full_version_string, ) {