Replace dep crates_io_api with in-house solution (#846)

It also uses `max_stable_version` in the json downloaded from https://crates.io/api/v1/crates/$name if possible, which is equivalent to the version shown on https://crates.io/crates/$name .

 - Add new feat `json` to `binstalk-downloader`
 - Impl new async fn `Response::json`
 - use `Response::json` in `GhApiClient` impl
 - Mark all err types in binstalk-downloader as `non_exhaustive`
 - Ret `remote::Error` in `remote::Certificate::{from_pem, from_der}` instead of `ReqwestError`.
 - Refactor `BinstallError`: Merge variant `Unzip`, `Reqwest` & `Http`
    into one variant `Download`.
 - Manually download and parse json from httos://crates.io/api/v1
 - Remove unused deps `crates_io_api`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/binstalk/Cargo.toml

@@ -16,7 +16,6 @@ binstalk-types = { version = "0.2.1", path = "../binstalk-types" }
 cargo_toml = "0.15.2"
 command-group = { version = "2.0.1", features = ["with-tokio"] }
 compact_str = { version = "0.7.0", features = ["serde"] }
-crates_io_api = { version = "0.8.1", default-features = false }
 detect-targets = { version = "0.1.5", path = "../detect-targets" }
 either = "1.8.1"
 home = "0.5.4"
@@ -49,7 +48,7 @@ pkg-config = ["binstalk-downloader/pkg-config"]
 
 zlib-ng = ["binstalk-downloader/zlib-ng"]
 
-rustls = ["crates_io_api/rustls", "binstalk-downloader/rustls"]
+rustls = ["binstalk-downloader/rustls"]
 native-tls = ["binstalk-downloader/native-tls"]
 
 trust-dns = ["binstalk-downloader/trust-dns"]

crates/binstalk/src/drivers.rs

@@ -1,5 +1,2 @@
-mod version;
-use version::find_version;
-
 mod crates_io;
 pub use crates_io::fetch_crate_cratesio;

crates/binstalk/src/drivers/crates_io.rs

@@ -1,8 +1,9 @@
 use std::path::PathBuf;
 
 use cargo_toml::Manifest;
-use crates_io_api::AsyncClient;
+use compact_str::CompactString;
 use semver::VersionReq;
+use serde::Deserialize;
 use tracing::debug;
 
 use crate::{
@@ -14,43 +15,98 @@ use crate::{
     manifests::cargo_toml_binstall::{Meta, TarBasedFmt},
 };
 
-use super::find_version;
-
 mod vfs;
 
 mod visitor;
 use visitor::ManifestVisitor;
 
-/// Fetch a crate Cargo.toml by name and version from crates.io
+#[derive(Deserialize)]
+struct CrateInfo {
+    #[serde(rename = "crate")]
+    inner: CrateInfoInner,
+}
+
+#[derive(Deserialize)]
+struct CrateInfoInner {
+    max_stable_version: CompactString,
+}
+
+#[derive(Deserialize)]
+struct Versions {
+    versions: Vec<Version>,
+}
+
+#[derive(Deserialize)]
+struct Version {
+    num: CompactString,
+    yanked: bool,
+}
+
+/// Find the crate by name, get its latest stable version matches `version_req`,
+/// retrieve its Cargo.toml and infer all its bins.
 pub async fn fetch_crate_cratesio(
     client: Client,
-    crates_io_api_client: &AsyncClient,
     name: &str,
     version_req: &VersionReq,
 ) -> Result<Manifest<Meta>, BinstallError> {
     // Fetch / update index
     debug!("Looking up crate information");
 
-    // Fetch online crate information
-    let base_info = crates_io_api_client.get_crate(name).await.map_err(|err| {
-        Box::new(CratesIoApiError {
-            crate_name: name.into(),
-            err,
-        })
-    })?;
+    let response = client
+        .get(Url::parse(&format!(
+            "https://crates.io/api/v1/crates/{name}"
+        ))?)
+        .send(true)
+        .await
+        .map_err(|err| {
+            BinstallError::CratesIoApi(Box::new(CratesIoApiError {
+                crate_name: name.into(),
+                err,
+            }))
+        })?;
 
-    // Locate matching version
-    let version_iter = base_info.versions.iter().filter(|v| !v.yanked);
-    let (version, version_name) = find_version(version_req, version_iter)?;
+    let version = if version_req == &VersionReq::STAR {
+        let crate_info: CrateInfo = response.json().await?;
+        crate_info.inner.max_stable_version
+    } else {
+        let response: Versions = response.json().await?;
+        response
+            .versions
+            .into_iter()
+            .filter_map(|item| {
+                if !item.yanked {
+                    // Remove leading `v` for git tags
+                    let num = if let Some(num) = item.num.strip_prefix('v') {
+                        num.into()
+                    } else {
+                        item.num
+                    };
 
-    debug!("Found information for crate version: '{}'", version.num);
+                    // Parse out version
+                    let ver = semver::Version::parse(&num).ok()?;
+
+                    // Filter by version match
+                    version_req.matches(&ver).then_some((num, ver))
+                } else {
+                    None
+                }
+            })
+            // Return highest version
+            .max_by(|(_ver_str_x, ver_x), (_ver_str_y, ver_y)| ver_x.cmp(ver_y))
+            .ok_or_else(|| BinstallError::VersionMismatch {
+                req: version_req.clone(),
+            })?
+            .0
+    };
+
+    debug!("Found information for crate version: '{version}'");
 
     // Download crate to temporary dir (crates.io or git?)
-    let crate_url = format!("https://crates.io/{}", version.dl_path);
+    let crate_url = format!("https://crates.io/api/v1/crates/{name}/{version}/download");
 
     debug!("Fetching crate from: {crate_url} and extracting Cargo.toml from it");
 
-    let manifest_dir_path: PathBuf = format!("{name}-{version_name}").into();
+    let manifest_dir_path: PathBuf = format!("{name}-{version}").into();
 
     let mut manifest_visitor = ManifestVisitor::new(manifest_dir_path);
 

crates/binstalk/src/drivers/version.rs

@@ -1,46 +0,0 @@
-use semver::VersionReq;
-
-use crate::errors::BinstallError;
-
-pub(super) trait Version {
-    /// Return `None` on error.
-    fn get_version(&self) -> Option<semver::Version>;
-}
-
-impl<T: Version> Version for &T {
-    fn get_version(&self) -> Option<semver::Version> {
-        (*self).get_version()
-    }
-}
-
-impl Version for crates_io_api::Version {
-    fn get_version(&self) -> Option<semver::Version> {
-        // Remove leading `v` for git tags
-        let ver_str = match self.num.strip_prefix('v') {
-            Some(v) => v,
-            None => &self.num,
-        };
-
-        // Parse out version
-        semver::Version::parse(ver_str).ok()
-    }
-}
-
-pub(super) fn find_version<Item: Version, VersionIter: Iterator<Item = Item>>(
-    version_req: &VersionReq,
-    version_iter: VersionIter,
-) -> Result<(Item, semver::Version), BinstallError> {
-    version_iter
-        // Filter for matching versions
-        .filter_map(|item| {
-            let ver = item.get_version()?;
-
-            // Filter by version match
-            version_req.matches(&ver).then_some((item, ver))
-        })
-        // Return highest version
-        .max_by(|(_item_x, ver_x), (_item_y, ver_y)| ver_x.cmp(ver_y))
-        .ok_or_else(|| BinstallError::VersionMismatch {
-            req: version_req.clone(),
-        })
-}

crates/binstalk/src/errors.rs

@@ -5,9 +5,7 @@ use std::{
 };
 
 use binstalk_downloader::{
-    download::{DownloadError, ZipError},
-    gh_api_client::GhApiError,
-    remote::{Error as RemoteError, HttpError, ReqwestError},
+    download::DownloadError, gh_api_client::GhApiError, remote::Error as RemoteError,
 };
 use cargo_toml::Error as CargoTomlError;
 use compact_str::CompactString;
@@ -22,7 +20,7 @@ use tracing::{error, warn};
 pub struct CratesIoApiError {
     pub crate_name: CompactString,
     #[source]
-    pub err: crates_io_api::Error,
+    pub err: RemoteError,
 }
 
 #[derive(Debug, Error)]
@@ -84,14 +82,6 @@ pub enum BinstallError {
     #[diagnostic(severity(error), code(binstall::url_parse))]
     UrlParse(#[from] url::ParseError),
 
-    /// An error while unzipping a file.
-    ///
-    /// - Code: `binstall::unzip`
-    /// - Exit: 66
-    #[error("Failed to extract zipfile: {0}")]
-    #[diagnostic(severity(error), code(binstall::unzip))]
-    Unzip(#[from] ZipError),
-
     /// A rendering error in a template.
     ///
     /// - Code: `binstall::template`
@@ -100,26 +90,13 @@ pub enum BinstallError {
     #[diagnostic(severity(error), code(binstall::template))]
     Template(Box<TinyTemplateError>),
 
-    /// A generic error from our HTTP client, reqwest.
+    /// Failed to download or failed to decode the body.
     ///
-    /// Errors resulting from HTTP fetches are handled with [`BinstallError::Http`] instead.
-    ///
-    /// - Code: `binstall::reqwest`
+    /// - Code: `binstall::download`
     /// - Exit: 68
-    #[error("Reqwest error: {0}")]
-    #[diagnostic(severity(error), code(binstall::reqwest))]
-    Reqwest(#[from] ReqwestError),
-
-    /// An HTTP request failed.
-    ///
-    /// This includes both connection/transport failures and when the HTTP status of the response
-    /// is not as expected.
-    ///
-    /// - Code: `binstall::http`
-    /// - Exit: 69
     #[error(transparent)]
-    #[diagnostic(severity(error), code(binstall::http))]
-    Http(#[from] Box<HttpError>),
+    #[diagnostic(severity(error), code(binstall::download))]
+    Download(#[from] DownloadError),
 
     /// A subprocess failed.
     ///
@@ -331,10 +308,8 @@ impl BinstallError {
             TaskJoinError(_) => 17,
             UserAbort => 32,
             UrlParse(_) => 65,
-            Unzip(_) => 66,
             Template(_) => 67,
-            Reqwest(_) => 68,
-            Http { .. } => 69,
+            Download(_) => 68,
             SubProcess { .. } => 70,
             Io(_) => 74,
             CratesIoApi { .. } => 76,
@@ -425,24 +400,7 @@ impl From<BinstallError> for io::Error {
 
 impl From<RemoteError> for BinstallError {
     fn from(e: RemoteError) -> Self {
-        use RemoteError::*;
-
-        match e {
-            Reqwest(reqwest_error) => reqwest_error.into(),
-            Http(http_error) => http_error.into(),
-        }
-    }
-}
-
-impl From<DownloadError> for BinstallError {
-    fn from(e: DownloadError) -> Self {
-        use DownloadError::*;
-
-        match e {
-            Unzip(zip_error) => zip_error.into(),
-            Remote(remote_error) => remote_error.into(),
-            Io(io_error) => io_error.into(),
-        }
+        DownloadError::from(e).into()
     }
 }
 

crates/binstalk/src/ops.rs

@@ -2,7 +2,6 @@
 
 use std::{path::PathBuf, sync::Arc};
 
-use crates_io_api::AsyncClient as CratesIoApiClient;
 use semver::VersionReq;
 
 use crate::{
@@ -34,7 +33,6 @@ pub struct Options {
     pub temp_dir: PathBuf,
     pub install_path: PathBuf,
     pub client: Client,
-    pub crates_io_api_client: CratesIoApiClient,
     pub gh_api_client: GhApiClient,
     pub jobserver_client: LazyJobserverClient,
 }

crates/binstalk/src/ops/resolve.rs

@@ -8,7 +8,6 @@ use std::{
 
 use cargo_toml::Manifest;
 use compact_str::{CompactString, ToCompactString};
-use crates_io_api::AsyncClient as CratesIoApiClient;
 use itertools::Itertools;
 use maybe_owned::MaybeOwned;
 use semver::{Version, VersionReq};
@@ -72,8 +71,7 @@ async fn resolve_inner(
         crate_name.name,
         curr_version,
         &version_req,
-        opts.client.clone(),
-        &opts.crates_io_api_client).await?
+        opts.client.clone()).await?
     else {
         return Ok(Resolution::AlreadyUpToDate)
     };
@@ -350,20 +348,11 @@ impl PackageInfo {
         curr_version: Option<Version>,
         version_req: &VersionReq,
         client: Client,
-        crates_io_api_client: &CratesIoApiClient,
     ) -> Result<Option<Self>, BinstallError> {
         // Fetch crate via crates.io, git, or use a local manifest path
         let manifest = match opts.manifest_path.as_ref() {
             Some(manifest_path) => load_manifest_path(manifest_path)?,
-            None => {
-                Box::pin(fetch_crate_cratesio(
-                    client,
-                    crates_io_api_client,
-                    &name,
-                    version_req,
-                ))
-                .await?
-            }
+            None => Box::pin(fetch_crate_cratesio(client, &name, version_req)).await?,
         };
 
         let Some(mut package) = manifest.package else {