Merge pull request #735 from epage/registry

fix(index): Don't talk to crates.io when using alt registry

src/ops/cargo.rs

@@ -112,18 +112,25 @@ pub fn publish(
 
 pub fn wait_for_publish(
  index: &mut crate::ops::index::CratesIoIndex,
+ registry: Option<&str>,
  name: &str,
  version: &str,
  timeout: std::time::Duration,
  dry_run: bool,
 ) -> CargoResult<()> {
  if !dry_run {
+ if registry.is_none() {
+ // HACK: `index` never reports crates as present for alternative registries
+ log::debug!("Not waiting for publish as that is only supported for crates.io; ensure you are using at least cargo v1.66 which will wait for you.");
+ return Ok(());
+ }
+
  let now = std::time::Instant::now();
  let sleep_time = std::time::Duration::from_secs(1);
  let mut logged = false;
  loop {
- index.update_krate(name);
- if is_published(index, name, version) {
+ index.update_krate(registry, name);
+ if is_published(index, registry, name, version) {
  break;
  } else if timeout < now.elapsed() {
  anyhow::bail!("timeout waiting for crate to be published");
@@ -145,10 +152,11 @@ pub fn wait_for_publish(
 
 pub fn is_published(
  index: &mut crate::ops::index::CratesIoIndex,
+ registry: Option<&str>,
  name: &str,
  version: &str,
 ) -> bool {
- match index.has_krate_version(name, version) {
+ match index.has_krate_version(registry, name, version) {
  Ok(has_krate_version) => has_krate_version.unwrap_or(false),
  Err(err) => {
  // For both http and git indices, this _might_ be an error that goes away in

src/ops/index.rs

@@ -1,53 +1,74 @@
 use tame_index::krate::IndexKrate;
 use tame_index::utils::flock::FileLock;
 
+#[derive(Default)]
 pub struct CratesIoIndex {
- index: RemoteIndex,
+ index: Option<RemoteIndex>,
  cache: std::collections::HashMap<String, Option<IndexKrate>>,
 }
 
 impl CratesIoIndex {
  #[inline]
- pub fn open() -> Result<Self, crate::error::CliError> {
- Ok(Self {
- index: RemoteIndex::open()?,
- cache: Default::default(),
- })
+ pub fn new() -> Self {
+ Self {
+ index: None,
+ cache: std::collections::HashMap::new(),
+ }
  }
 
  /// Determines if the specified crate exists in the crates.io index
  #[inline]
- pub fn has_krate(&mut self, name: &str) -> Result<bool, crate::error::CliError> {
- Ok(self.krate(name)?.map(|_| true).unwrap_or(false))
+ pub fn has_krate(
+ &mut self,
+ registry: Option<&str>,
+ name: &str,
+ ) -> Result<bool, crate::error::CliError> {
+ Ok(self.krate(registry, name)?.map(|_| true).unwrap_or(false))
  }
 
  /// Determines if the specified crate version exists in the crates.io index
  #[inline]
  pub fn has_krate_version(
  &mut self,
+ registry: Option<&str>,
  name: &str,
  version: &str,
  ) -> Result<Option<bool>, crate::error::CliError> {
- let krate = self.krate(name)?;
+ let krate = self.krate(registry, name)?;
  Ok(krate.map(|ik| ik.versions.iter().any(|iv| iv.version == version)))
  }
 
  #[inline]
- pub fn update_krate(&mut self, name: &str) {
+ pub fn update_krate(&mut self, registry: Option<&str>, name: &str) {
+ if registry.is_some() {
+ return;
+ }
+
  self.cache.remove(name);
  }
 
  pub(crate) fn krate(
  &mut self,
+ registry: Option<&str>,
  name: &str,
  ) -> Result<Option<IndexKrate>, crate::error::CliError> {
+ if let Some(registry) = registry {
+ log::trace!("Cannot connect to registry `{registry}`");
+ return Ok(None);
+ }
+
  if let Some(entry) = self.cache.get(name) {
  log::trace!("Reusing index for {name}");
  return Ok(entry.clone());
  }
 
+ if self.index.is_none() {
+ log::trace!("Connecting to index");
+ self.index = Some(RemoteIndex::open()?);
+ }
+ let index = self.index.as_mut().unwrap();
  log::trace!("Downloading index for {name}");
- let entry = self.index.krate(name)?;
+ let entry = index.krate(name)?;
  self.cache.insert(name.to_owned(), entry.clone());
  Ok(entry)
  }

src/steps/hook.rs

@@ -47,7 +47,7 @@ pub struct HookStep {
 impl HookStep {
  pub fn run(&self) -> Result<(), CliError> {
  git::git_version()?;
- let mut index = crate::ops::index::CratesIoIndex::open()?;
+ let mut index = crate::ops::index::CratesIoIndex::new();
 
  if self.dry_run {
  let _ =
@@ -88,6 +88,7 @@ impl HookStep {
  let version = &pkg.initial_version;
  if !crate::ops::cargo::is_published(
  &mut index,
+ pkg.config.registry(),
  crate_name,
  &version.full_version_string,
  ) {

src/steps/mod.rs

@@ -225,9 +225,10 @@ pub fn verify_rate_limit(
  let mut new = 0;
  let mut existing = 0;
  for pkg in pkgs {
+ // Note: these rate limits are only known for default registry
  if pkg.config.registry().is_none() && pkg.config.publish() {
  let crate_name = pkg.meta.name.as_str();
- if index.has_krate(crate_name)? {
+ if index.has_krate(None, crate_name)? {
  existing += 1;
  } else {
  new += 1;

src/steps/publish.rs

@@ -80,13 +80,14 @@ impl PublishStep {
 
  let mut pkgs = plan::plan(pkgs)?;
 
- let mut index = crate::ops::index::CratesIoIndex::open()?;
+ let mut index = crate::ops::index::CratesIoIndex::new();
  for pkg in pkgs.values_mut() {
- if pkg.config.registry().is_none() && pkg.config.release() {
+ if pkg.config.release() {
  let crate_name = pkg.meta.name.as_str();
  let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
  if crate::ops::cargo::is_published(
  &mut index,
+ pkg.config.registry(),
  crate_name,
  &version.full_version_string,
  ) {
@@ -200,33 +201,31 @@ pub fn publish(
  return Err(101.into());
  }
 
- if pkg.config.registry().is_none() {
-  let timeout = std::time::Duration::from_secs(300);
-  let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
- crate::ops::cargo::wait_for_publish(
-  index,
-  crate_name,
-  &version.full_version_string,
-  timeout,
-  dry_run,
-  )?;
-  // HACK: Even once the index is updated, there seems to be another step before the publish is fully ready.
-  // We don't have a way yet to check for that, so waiting for now in hopes everything is ready
-  if !dry_run {
-  let publish_grace_sleep = std::env::var("PUBLISH_GRACE_SLEEP")
-  .unwrap_or_else(|_| Default::default())
-  .parse()
-  .unwrap_or(0);
-  if 0 < publish_grace_sleep {
-  log::debug!(
-  "waiting an additional {} seconds for crates.io to update its indices...",
-  publish_grace_sleep
- );
-  std::thread::sleep(std::time::Duration::from_secs(publish_grace_sleep));
- }
+ let timeout = std::time::Duration::from_secs(300);
+ let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
+ crate::ops::cargo::wait_for_publish(
+ index,
+ pkg.config.registry(),
+ crate_name,
+ &version.full_version_string,
+ timeout,
+ dry_run,
+ )?;
+ // HACK: Even once the index is updated, there seems to be another step before the publish is fully ready.
+ // We don't have a way yet to check for that, so waiting for now in hopes everything is ready
+ if !dry_run {
+ let publish_grace_sleep = std::env::var("PUBLISH_GRACE_SLEEP")
+ .unwrap_or_else(|_| Default::default())
+ .parse()
+ .unwrap_or(0);
+ if 0 < publish_grace_sleep {
+ log::debug!(
+ "waiting an additional {} seconds for {} to update its indices...",
+ publish_grace_sleep,
+ pkg.config.registry().unwrap_or("crates.io")
+ );
+ std::thread::sleep(std::time::Duration::from_secs(publish_grace_sleep));
  }
- } else {
- log::debug!("not waiting for publish because the registry is not crates.io and doesn't get updated automatically");
  }
  }
 

src/steps/release.rs

@@ -46,7 +46,7 @@ pub struct ReleaseStep {
 impl ReleaseStep {
  pub fn run(&self) -> Result<(), CliError> {
  git::git_version()?;
- let mut index = crate::ops::index::CratesIoIndex::open()?;
+ let mut index = crate::ops::index::CratesIoIndex::new();
 
  if self.dry_run {
  let _ =
@@ -73,7 +73,7 @@ impl ReleaseStep {
  pkg.bump(level_or_version, self.metadata.as_deref())?;
  }
  }
- if index.has_krate(&pkg.meta.name)? {
+ if index.has_krate(pkg.config.registry(), &pkg.meta.name)? {
  // Already published, skip it. Use `cargo release owner` for one-time updates
  pkg.ensure_owners = false;
  }
@@ -101,7 +101,12 @@ impl ReleaseStep {
  && !explicitly_excluded
  {
  let version = &pkg.initial_version;
- if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
+ if !cargo::is_published(
+ &mut index,
+ pkg.config.registry(),
+ crate_name,
+ &version.full_version_string,
+ ) {
  log::debug!(
  "enabled {}, v{} is unpublished",
  crate_name,
@@ -152,10 +157,16 @@ impl ReleaseStep {
  continue;
  };
 
+ // HACK: `index` only supports default registry
  if pkg.config.publish() && pkg.config.registry().is_none() {
  let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
  let crate_name = pkg.meta.name.as_str();
- if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
+ if !cargo::is_published(
+ &mut index,
+ pkg.config.registry(),
+ crate_name,
+ &version.full_version_string,
+ ) {
  let _ = crate::ops::shell::warn(format!(
  "disabled by user, skipping {} v{} despite being unpublished",
  crate_name, version.full_version_string,
@@ -195,16 +206,19 @@ impl ReleaseStep {
  if !pkg.config.publish() {
  continue;
  }
- if pkg.config.registry().is_none() {
- let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
- let crate_name = pkg.meta.name.as_str();
- if cargo::is_published(&mut index, crate_name, &version.full_version_string) {
- let _ = crate::ops::shell::error(format!(
- "{} {} is already published",
- crate_name, version.full_version_string
- ));
- double_publish = true;
- }
+ let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
+ let crate_name = pkg.meta.name.as_str();
+ if cargo::is_published(
+ &mut index,
+ pkg.config.registry(),
+ crate_name,
+ &version.full_version_string,
+ ) {
+ let _ = crate::ops::shell::error(format!(
+ "{} {} is already published",
+ crate_name, version.full_version_string
+ ));
+ double_publish = true;
  }
  }
  if double_publish {

src/steps/replace.rs

@@ -43,7 +43,7 @@ pub struct ReplaceStep {
 impl ReplaceStep {
  pub fn run(&self) -> Result<(), CliError> {
  git::git_version()?;
- let mut index = crate::ops::index::CratesIoIndex::open()?;
+ let mut index = crate::ops::index::CratesIoIndex::new();
 
  if self.dry_run {
  let _ =
@@ -84,6 +84,7 @@ impl ReplaceStep {
  let version = &pkg.initial_version;
  if !crate::ops::cargo::is_published(
  &mut index,
+ pkg.config.registry(),
  crate_name,
  &version.full_version_string,
  ) {