binstalk-registry: Rm rate-limit for crates.io registry (#1299)

Fixed #1295

The 1 request per second rate-limit is too strict and it makes
`cargo-binstall` very slow when resolving many crates in parallel.

Relying on the rate-limit in `binstalk_downloader::remote::Client`
should be good enough.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/binstalk-registry/Cargo.toml

@@ -27,7 +27,7 @@ serde_json = "1.0.99"
 sha2 = "0.10.7"
 tempfile = "3.5.0"
 thiserror = "1.0.40"
-tokio = { version = "1.30.0", features = ["rt", "sync", "time"], default-features = false }
+tokio = { version = "1.30.0", features = ["rt", "sync"], default-features = false }
 tracing = "0.1.37"
 url = "2.3.1"
 

crates/binstalk-registry/src/crates_io_registry.rs

@@ -4,42 +4,10 @@ use cargo_toml_workspace::cargo_toml::Manifest;
 use compact_str::{CompactString, ToCompactString};
 use semver::{Comparator, Op as ComparatorOp, Version as SemVersion, VersionReq};
 use serde::Deserialize;
-use tokio::{
-    sync::Mutex,
-    time::{interval, Duration, Interval, MissedTickBehavior},
-};
 use tracing::debug;
 
 use crate::{parse_manifest, MatchedVersion, RegistryError};
 
-#[derive(Debug)]
-pub struct CratesIoRateLimit(Mutex<Interval>);
-
-impl Default for CratesIoRateLimit {
-    fn default() -> Self {
-        let mut interval = interval(Duration::from_secs(1));
-        // If somehow one tick is delayed, then next tick should be at least
-        // 1s later than the current tick.
-        //
-        // Other MissedTickBehavior including Burst (default), which will
-        // tick as fast as possible to catch up, and Skip, which will
-        // skip the current tick for the next one.
-        //
-        // Both Burst and Skip is not the expected behavior for rate limit:
-        // ticking as fast as possible would violate crates.io crawler
-        // policy, and skipping the current one will slow down the resolution
-        // process.
-        interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
-        Self(Mutex::new(interval))
-    }
-}
-
-impl CratesIoRateLimit {
-    pub(super) async fn tick(&self) {
-        self.0.lock().await.tick().await;
-    }
-}
-
 /// Return `Some(checksum)` if the version is not yanked, otherwise `None`.
 async fn is_crate_yanked(client: &Client, url: Url) -> Result<Option<String>, RemoteError> {
     #[derive(Deserialize)]
@@ -141,11 +109,7 @@ pub async fn fetch_crate_cratesio(
     client: Client,
     name: &str,
     version_req: &VersionReq,
-    crates_io_rate_limit: &CratesIoRateLimit,
 ) -> Result<Manifest<Meta>, RegistryError> {
-    // Wait until we can make another request to crates.io
-    crates_io_rate_limit.tick().await;
-
     let url = Url::parse(&format!("https://crates.io/api/v1/crates/{name}"))?;
 
     let (version, cksum) = match version_req.comparators.as_slice() {

crates/binstalk-registry/src/lib.rs

@@ -34,7 +34,7 @@ mod git_registry;
 pub use git_registry::GitRegistry;
 
 mod crates_io_registry;
-pub use crates_io_registry::{fetch_crate_cratesio, CratesIoRateLimit};
+pub use crates_io_registry::fetch_crate_cratesio;
 
 mod sparse_registry;
 pub use sparse_registry::SparseRegistry;
@@ -100,10 +100,11 @@ impl From<CargoTomlError> for RegistryError {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 #[non_exhaustive]
 pub enum Registry {
-    CratesIo(Arc<CratesIoRateLimit>),
+    #[default]
+    CratesIo,
 
     Sparse(Arc<SparseRegistry>),
 
@@ -111,12 +112,6 @@ pub enum Registry {
     Git(GitRegistry),
 }
 
-impl Default for Registry {
-    fn default() -> Self {
-        Self::CratesIo(Default::default())
-    }
-}
-
 #[derive(Debug, ThisError)]
 #[error("Invalid registry `{src}`, {inner}")]
 pub struct InvalidRegistryError {
@@ -175,9 +170,7 @@ impl Registry {
         version_req: &VersionReq,
     ) -> Result<Manifest<Meta>, RegistryError> {
         match self {
-            Self::CratesIo(rate_limit) => {
+            Self::CratesIo => fetch_crate_cratesio(client, crate_name, version_req).await,
-                fetch_crate_cratesio(client, crate_name, version_req, rate_limit).await
-            }
             Self::Sparse(sparse_registry) => {
                 sparse_registry
                     .fetch_crate_matched(client, crate_name, version_req)