Feature: Better retry policy in binstalk-downloader (#794)

Fixed #779 #791 

 - Retry request on timeout
 - Retry for `StatusCode::{REQUEST_TIMEOUT, GATEWAY_TIMEOUT}`
 - Add `DEFAULT_RETRY_DURATION_FOR_RATE_LIMIT` for 503/429
   if 503/429 does not give us a header or give us an invalid header on
   when to retry, we would default to
   `DEFAULT_RETRY_DURATION_FOR_RATE_LIMIT`.
 - Fix `Client::get_redirected_final_url`: Retry using `GET` on status code 400..405 + 410
 - Rename remote_exists => remote_gettable & support fallback to GET
   if HEAD fails due to status code 400..405 + 410.
 - Improve `Client::get_stream`: Include url & method in the err of the stream returned

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2023-02-13 13:43:48 +11:00 committed by GitHub
parent 1b2fb08bcb
commit 87686cb2f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 202 additions and 91 deletions

View file

@ -13,9 +13,7 @@ use url::Url;
use crate::{
errors::{BinstallError, InvalidPkgFmtError},
helpers::{
download::Download,
futures_resolver::FuturesResolver,
remote::{Client, Method},
download::Download, futures_resolver::FuturesResolver, remote::Client,
tasks::AutoAbortJoinHandle,
},
manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
@ -68,8 +66,9 @@ impl GhCrateMeta {
async move {
debug!("Checking for package at: '{url}'");
Ok((client.remote_exists(url.clone(), Method::HEAD).await?
|| client.remote_exists(url.clone(), Method::GET).await?)
Ok(client
.remote_gettable(url.clone())
.await?
.then_some((url, pkg_fmt)))
}
})

View file

@ -6,11 +6,7 @@ use url::Url;
use crate::{
errors::BinstallError,
helpers::{
download::Download,
remote::{Client, Method},
tasks::AutoAbortJoinHandle,
},
helpers::{download::Download, remote::Client, tasks::AutoAbortJoinHandle},
manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
};
@ -60,10 +56,7 @@ impl super::Fetcher for QuickInstall {
let url = self.package_url();
debug!("Checking for package at: '{url}'");
Ok(self
.client
.remote_exists(Url::parse(&url)?, Method::HEAD)
.await?)
Ok(self.client.remote_gettable(Url::parse(&url)?).await?)
})
}
@ -124,7 +117,7 @@ impl QuickInstall {
let url = Url::parse(&self.stats_url())?;
debug!("Sending installation report to quickinstall ({url})");
self.client.remote_exists(url, Method::HEAD).await?;
self.client.remote_gettable(url).await?;
Ok(())
}