Impl GhApiClient and use it in cargo-binstall to speedup resolution process (#832)

Fixed #776

 - Add new feature gh-api-client to binstalk-downloader
 - Impl new type `binstalk_downloader::remote::{RequestBuilder, Response}`
 - Impl `binstalk_downloader::gh_api_client::GhApiClient`, exposed if `cfg(feature = "gh-api-client")` and add e2e and unit tests for it
 - Use `binstalk_downloader::gh_api_client::GhApiClient` to speedup `cargo-binstall`
 - Add new option `--github-token` to supply the token for GitHub restful API, or read from env variable `GITHUB_TOKEN` if not present.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/binstalk/Cargo.toml

@@ -11,7 +11,7 @@ license = "GPL-3.0"
 
 [dependencies]
 async-trait = "0.1.64"
-binstalk-downloader = { version = "0.3.3", path = "../binstalk-downloader", default-features = false }
+binstalk-downloader = { version = "0.3.3", path = "../binstalk-downloader", default-features = false, features = ["gh-api-client"] }
 binstalk-types = { version = "0.2.1", path = "../binstalk-types" }
 cargo_toml = "0.15.2"
 command-group = { version = "2.0.1", features = ["with-tokio"] }

crates/binstalk/src/errors.rs

@@ -6,6 +6,7 @@ use std::{
 
 use binstalk_downloader::{
     download::{DownloadError, ZipError},
+    gh_api_client::GhApiError,
     remote::{Error as RemoteError, HttpError, ReqwestError},
 };
 use cargo_toml::Error as CargoTomlError;
@@ -309,6 +310,14 @@ pub enum BinstallError {
     #[diagnostic(severity(error), code(binstall::invalid_pkg_fmt))]
     InvalidPkgFmt(Box<InvalidPkgFmtError>),
 
+    /// Request to GitHub Restful API failed
+    ///
+    /// - Code: `binstall::gh_restful_api_failure`
+    /// - Exit: 96
+    #[error("Request to GitHub Restful API failed: {0}")]
+    #[diagnostic(severity(error), code(binstall::gh_restful_api_failure))]
+    GhApiErr(#[source] Box<GhApiError>),
+
     /// A wrapped error providing the context of which crate the error is about.
     #[error(transparent)]
     #[diagnostic(transparent)]
@@ -343,6 +352,7 @@ impl BinstallError {
             EmptySourceFilePath => 92,
             NoFallbackToCargoInstall => 94,
             InvalidPkgFmt(..) => 95,
+            GhApiErr(..) => 96,
             CrateContext(context) => context.err.exit_number(),
         };
 
@@ -453,3 +463,9 @@ impl From<InvalidPkgFmtError> for BinstallError {
         BinstallError::InvalidPkgFmt(Box::new(e))
     }
 }
+
+impl From<GhApiError> for BinstallError {
+    fn from(e: GhApiError) -> Self {
+        BinstallError::GhApiErr(Box::new(e))
+    }
+}

crates/binstalk/src/fetchers.rs

@@ -8,7 +8,7 @@ use url::Url;
 
 use crate::{
     errors::BinstallError,
-    helpers::{remote::Client, tasks::AutoAbortJoinHandle},
+    helpers::{gh_api_client::GhApiClient, remote::Client, tasks::AutoAbortJoinHandle},
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
 };
 
@@ -19,7 +19,12 @@ pub(crate) mod quickinstall;
 pub trait Fetcher: Send + Sync {
     /// Create a new fetcher from some data
     #[allow(clippy::new_ret_no_self)]
-    fn new(client: Client, data: Arc<Data>, target_data: Arc<TargetData>) -> Arc<dyn Fetcher>
+    fn new(
+        client: Client,
+        gh_api_client: GhApiClient,
+        data: Arc<Data>,
+        target_data: Arc<TargetData>,
+    ) -> Arc<dyn Fetcher>
     where
         Self: Sized;
 

crates/binstalk/src/fetchers/gh_crate_meta.rs

@@ -13,7 +13,10 @@ use url::Url;
 use crate::{
     errors::{BinstallError, InvalidPkgFmtError},
     helpers::{
-        download::Download, futures_resolver::FuturesResolver, remote::Client,
+        download::Download,
+        futures_resolver::FuturesResolver,
+        gh_api_client::{GhApiClient, GhReleaseArtifact, HasReleaseArtifact},
+        remote::Client,
         tasks::AutoAbortJoinHandle,
     },
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
@@ -26,6 +29,7 @@ use hosting::RepositoryHost;
 
 pub struct GhCrateMeta {
     client: Client,
+    gh_api_client: GhApiClient,
     data: Arc<Data>,
     target_data: Arc<TargetData>,
     resolution: OnceCell<(Url, PkgFmt)>,
@@ -62,10 +66,29 @@ impl GhCrateMeta {
         // go check all potential URLs at once
         urls.map(move |url| {
             let client = self.client.clone();
+            let gh_api_client = self.gh_api_client.clone();
 
             async move {
                 debug!("Checking for package at: '{url}'");
 
+                if let Some(artifact) = GhReleaseArtifact::try_extract_from_url(&url) {
+                    debug!("Using GitHub Restful API to check for existence of artifact, which will also cache the API response");
+
+                    match gh_api_client.has_release_artifact(artifact).await? {
+                        HasReleaseArtifact::Yes => return Ok(Some((url, pkg_fmt))),
+                        HasReleaseArtifact::No | HasReleaseArtifact::NoSuchRelease=> return Ok(None),
+
+                        HasReleaseArtifact::RateLimit { retry_after } => {
+                            warn!("Your GitHub API token (if any) has reached its rate limit and cannot be used again until {retry_after:?}, so we will fallback to HEAD/GET on the url.");
+                            warn!("If you did not supply the github token, consider supply one since GitHub by default limit the number of requests for unauthoized user to 60 requests per hour per origin IP address.");
+                        }
+                        HasReleaseArtifact::Unauthorized => {
+                            warn!("GitHub API somehow requires a token for the API access, so we will fallback to HEAD/GET on the url.");
+                            warn!("Please consider supplying a token to cargo-binstall to speedup resolution.");
+                        }
+                    }
+                }
+
                 Ok(client
                     .remote_gettable(url.clone())
                     .await?
@@ -79,11 +102,13 @@ impl GhCrateMeta {
 impl super::Fetcher for GhCrateMeta {
     fn new(
         client: Client,
+        gh_api_client: GhApiClient,
         data: Arc<Data>,
         target_data: Arc<TargetData>,
     ) -> Arc<dyn super::Fetcher> {
         Arc::new(Self {
             client,
+            gh_api_client,
             data,
             target_data,
             resolution: OnceCell::new(),

crates/binstalk/src/fetchers/quickinstall.rs

@@ -6,7 +6,9 @@ use url::Url;
 
 use crate::{
     errors::BinstallError,
-    helpers::{download::Download, remote::Client, tasks::AutoAbortJoinHandle},
+    helpers::{
+        download::Download, gh_api_client::GhApiClient, remote::Client, tasks::AutoAbortJoinHandle,
+    },
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
 };
 
@@ -25,6 +27,7 @@ pub struct QuickInstall {
 impl super::Fetcher for QuickInstall {
     fn new(
         client: Client,
+        _gh_api_client: GhApiClient,
         data: Arc<Data>,
         target_data: Arc<TargetData>,
     ) -> Arc<dyn super::Fetcher> {

crates/binstalk/src/helpers.rs

@@ -3,4 +3,4 @@ pub mod jobserver_client;
 pub mod signal;
 pub mod tasks;
 
-pub use binstalk_downloader::{download, remote};
+pub use binstalk_downloader::{download, gh_api_client, remote};

crates/binstalk/src/ops.rs

@@ -7,14 +7,14 @@ use semver::VersionReq;
 
 use crate::{
     fetchers::{Data, Fetcher, TargetData},
-    helpers::{jobserver_client::LazyJobserverClient, remote::Client},
+    helpers::{gh_api_client::GhApiClient, jobserver_client::LazyJobserverClient, remote::Client},
     manifests::cargo_toml_binstall::PkgOverride,
     DesiredTargets,
 };
 
 pub mod resolve;
 
-pub type Resolver = fn(Client, Arc<Data>, Arc<TargetData>) -> Arc<dyn Fetcher>;
+pub type Resolver = fn(Client, GhApiClient, Arc<Data>, Arc<TargetData>) -> Arc<dyn Fetcher>;
 
 pub struct Options {
     pub no_symlinks: bool,
@@ -35,5 +35,6 @@ pub struct Options {
     pub install_path: PathBuf,
     pub client: Client,
     pub crates_io_api_client: CratesIoApiClient,
+    pub gh_api_client: GhApiClient,
     pub jobserver_client: LazyJobserverClient,
 }

crates/binstalk/src/ops/resolve.rs

@@ -109,7 +109,12 @@ async fn resolve_inner(
             })
             .cartesian_product(resolvers)
             .map(|(target_data, f)| {
-                let fetcher = f(opts.client.clone(), data.clone(), target_data);
+                let fetcher = f(
+                    opts.client.clone(),
+                    opts.gh_api_client.clone(),
+                    data.clone(),
+                    target_data,
+                );
                 (fetcher.clone(), fetcher.find())
             }),
     );