actions/cargo-binstall
1
0
Fork 0
mirror of https://github.com/cargo-bins/cargo-binstall.git synced 2025-05-06 12:10:02 +00:00
Code Issues Projects Releases Packages Wiki Activity

Fix "Too many "Too Many Requests" log" (#761)

Browse source 
Fixed #747

 - Add dep compact_str v0.6.1 to binstalk-downloader
 - Impl new type `DelayRequest`
 - Handle 503/429 with wait duration > `MAX_RETRY_DURATION` by simply taking the min
 - Fix `Client::send_request_inner`: Ensure 503/429 get propagated to other requests
   
   even if the current requests reach its maximum retry and decides to
   simply return an error.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2023-02-04 11:14:53 +11:00 • committed by GitHub
parent e510511487
commit f2fc37eea5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 134 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

40
crates/binstalk-downloader/src/remote.rs
View file

@ -12,13 +12,16 @@ use reqwest::{
Request, Response, StatusCode,
};
use thiserror::Error as ThisError;
use tokio::{sync::Mutex, time::sleep};
use tokio::time::Instant;
use tower::{limit::rate::RateLimit, Service, ServiceBuilder, ServiceExt};
use tracing::{debug, info};
pub use reqwest::{tls, Error as ReqwestError, Method};
pub use url::Url;
mod delay_request;
use delay_request::DelayRequest;
const MAX_RETRY_DURATION: Duration = Duration::from_secs(120);
const MAX_RETRY_COUNT: u8 = 3;
const DEFAULT_MIN_TLS: tls::Version = tls::Version::TLS_1_2;
@ -44,7 +47,7 @@ pub struct HttpError {
#[derive(Debug)]
struct Inner {
client: reqwest::Client,
rate_limit: Mutex<RateLimit<reqwest::Client>>,
service: DelayRequest<RateLimit<reqwest::Client>>,
}
#[derive(Clone, Debug)]
@ -81,7 +84,7 @@ impl Client {
Ok(Client(Arc::new(Inner {
client: client.clone(),
rate_limit: Mutex::new(
service: DelayRequest::new(
ServiceBuilder::new()
.rate_limit(num_request.get(), per)
.service(client),
@ -107,13 +110,7 @@ impl Client {
loop {
let request = Request::new(method.clone(), url.clone());
// Reduce critical section:
// - Construct the request before locking
// - Once the rate_limit is ready, call it and obtain
// the future, then release the lock before
// polling the future, which performs network I/O that could
// take really long.
let future = self.0.rate_limit.lock().await.ready().await?.call(request);
let future = (&self.0.service).ready().await?.call(request);
let response = future.await?;
@ -124,9 +121,20 @@ impl Client {
// 503 429
StatusCode::SERVICE_UNAVAILABLE | StatusCode::TOO_MANY_REQUESTS,
Some(duration),
) if duration <= MAX_RETRY_DURATION && count < MAX_RETRY_COUNT => {
) => {
let duration = duration.min(MAX_RETRY_DURATION);
info!("Receiver status code {status}, will wait for {duration:#?} and retry");
sleep(duration).await
let deadline = Instant::now() + duration;
self.0
.service
.add_urls_to_delay(dedup([url, response.url()]), deadline);
if count >= MAX_RETRY_COUNT {
break Ok(response);
}
}
_ => break Ok(response),
}
@ -211,3 +219,11 @@ fn parse_header_retry_after(headers: &HeaderMap) -> Option<Duration> {
}
}
}
fn dedup(urls: [&Url; 2]) -> impl Iterator<Item = &Url> {
if urls[0] == urls[1] {
Some(urls[0]).into_iter().chain(None)
} else {
Some(urls[0]).into_iter().chain(Some(urls[1]))
}
}