diff --git a/crates/bin/src/entry.rs b/crates/bin/src/entry.rs index ccc2d953..1eb7b61b 100644 --- a/crates/bin/src/entry.rs +++ b/crates/bin/src/entry.rs @@ -117,6 +117,7 @@ pub fn install_crates( client, gh_api_client, jobserver_client, + crates_io_rate_limit: Default::default(), }); // Destruct args before any async function to reduce size of the future diff --git a/crates/binstalk/src/drivers/crates_io.rs b/crates/binstalk/src/drivers/crates_io.rs index b758d7c3..28816597 100644 --- a/crates/binstalk/src/drivers/crates_io.rs +++ b/crates/binstalk/src/drivers/crates_io.rs @@ -13,6 +13,7 @@ use crate::{ remote::{Client, Url}, }, manifests::cargo_toml_binstall::{Meta, TarBasedFmt}, + ops::CratesIoRateLimit, }; mod vfs; @@ -48,7 +49,11 @@ pub async fn fetch_crate_cratesio( client: Client, name: &str, version_req: &VersionReq, + crates_io_rate_limit: &CratesIoRateLimit, ) -> Result, BinstallError> { + // Wait until we can make another request to crates.io + crates_io_rate_limit.tick().await; + // Fetch / update index debug!("Looking up crate information"); diff --git a/crates/binstalk/src/ops.rs b/crates/binstalk/src/ops.rs index 42f2eec3..67b33a38 100644 --- a/crates/binstalk/src/ops.rs +++ b/crates/binstalk/src/ops.rs @@ -3,6 +3,10 @@ use std::{path::PathBuf, sync::Arc}; use semver::VersionReq; +use tokio::{ + sync::Mutex, + time::{interval, Duration, Interval, MissedTickBehavior}, +}; use crate::{ fetchers::{Data, Fetcher, TargetData}, @@ -37,4 +41,32 @@ pub struct Options { pub client: Client, pub gh_api_client: GhApiClient, pub jobserver_client: LazyJobserverClient, + pub crates_io_rate_limit: CratesIoRateLimit, +} + +pub struct CratesIoRateLimit(Mutex); + +impl Default for CratesIoRateLimit { + fn default() -> Self { + let mut interval = interval(Duration::from_secs(1)); + // If somehow one tick is delayed, then next tick should be at least + // 1s later than the current tick. + // + // Other MissedTickBehavior including Burst (default), which will + // tick as fast as possible to catch up, and Skip, which will + // skip the current tick for the next one. + // + // Both Burst and Skip is not the expected behavior for rate limit: + // ticking as fast as possible would violate crates.io crawler + // policy, and skipping the current one will slow down the resolution + // process. + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + Self(Mutex::new(interval)) + } +} + +impl CratesIoRateLimit { + pub(super) async fn tick(&self) { + self.0.lock().await.tick().await; + } } diff --git a/crates/binstalk/src/ops/resolve.rs b/crates/binstalk/src/ops/resolve.rs index 8b378902..8255be87 100644 --- a/crates/binstalk/src/ops/resolve.rs +++ b/crates/binstalk/src/ops/resolve.rs @@ -354,7 +354,15 @@ impl PackageInfo { // Fetch crate via crates.io, git, or use a local manifest path let manifest = match opts.manifest_path.as_ref() { Some(manifest_path) => load_manifest_path(manifest_path)?, - None => Box::pin(fetch_crate_cratesio(client, &name, version_req)).await?, + None => { + Box::pin(fetch_crate_cratesio( + client, + &name, + version_req, + &opts.crates_io_rate_limit, + )) + .await? + } }; let Some(mut package) = manifest.package else {