Replace dep crates_io_api with in-house solution (#846)

It also uses `max_stable_version` in the json downloaded from https://crates.io/api/v1/crates/$name if possible, which is equivalent to the version shown on https://crates.io/crates/$name .

 - Add new feat `json` to `binstalk-downloader`
 - Impl new async fn `Response::json`
 - use `Response::json` in `GhApiClient` impl
 - Mark all err types in binstalk-downloader as `non_exhaustive`
 - Ret `remote::Error` in `remote::Certificate::{from_pem, from_der}` instead of `ReqwestError`.
 - Refactor `BinstallError`: Merge variant `Unzip`, `Reqwest` & `Http`
    into one variant `Download`.
 - Manually download and parse json from httos://crates.io/api/v1
 - Remove unused deps `crates_io_api`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2023-03-02 18:25:34 +11:00 committed by GitHub
parent c00d648dac
commit 8eee318ccd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 120 additions and 244 deletions

87
Cargo.lock generated
View file

@ -158,7 +158,6 @@ dependencies = [
"cargo_toml",
"command-group",
"compact_str",
"crates_io_api",
"detect-targets",
"either",
"home",
@ -343,7 +342,6 @@ dependencies = [
"binstalk-manifests",
"clap",
"compact_str",
"crates_io_api",
"dirs",
"embed-resource",
"file-format",
@ -398,17 +396,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
dependencies = [
"num-integer",
"num-traits",
"serde",
]
[[package]]
name = "clap"
version = "4.1.8"
@ -497,24 +484,6 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "crates_io_api"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fdfaac64c5eb7a33a5b895ffbaf6f1146721b6cd4c889010d19c8a1c1cae562"
dependencies = [
"chrono",
"futures",
"log",
"reqwest",
"serde",
"serde_derive",
"serde_json",
"serde_path_to_error",
"tokio",
"url",
]
[[package]]
name = "crc32fast"
version = "1.3.2"
@ -733,21 +702,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "futures"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.26"
@ -755,7 +709,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@ -764,17 +717,6 @@ version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
[[package]]
name = "futures-executor"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.26"
@ -820,7 +762,6 @@ version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
@ -1408,25 +1349,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
@ -2021,15 +1943,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341"
dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.1"

View file

@ -26,7 +26,6 @@ binstalk = { path = "../binstalk", version = "0.8.0", default-features = false }
binstalk-manifests = { path = "../binstalk-manifests", version = "0.3.0" }
clap = { version = "4.1.8", features = ["derive", "env"] }
compact_str = "0.7.0"
crates_io_api = { version = "0.8.1", default-features = false }
dirs = "4.0.0"
file-format = { version = "0.14.0", default-features = false }
fs-lock = { version = "0.1.0", path = "../fs-lock" }

View file

@ -22,7 +22,6 @@ use binstalk::{
},
};
use binstalk_manifests::cargo_toml_binstall::PkgOverride;
use crates_io_api::AsyncClient as CratesIoApiClient;
use file_format::FileFormat;
use log::LevelFilter;
use miette::{miette, Result, WrapErr};
@ -90,10 +89,6 @@ pub async fn install_crates(args: Args, jobserver_client: LazyJobserverClient) -
let gh_api_client = GhApiClient::new(client.clone(), args.github_token);
// Build crates.io api client
let crates_io_api_client =
CratesIoApiClient::with_http_client(client.get_inner().clone(), Duration::from_millis(100));
// Create binstall_opts
let binstall_opts = Arc::new(ops::Options {
no_symlinks: args.no_symlinks,
@ -113,7 +108,6 @@ pub async fn install_crates(args: Args, jobserver_client: LazyJobserverClient) -
temp_dir: temp_dir.path().to_owned(),
install_path,
client,
crates_io_api_client,
gh_api_client,
jobserver_client,
});

View file

@ -74,7 +74,8 @@ zstd-thin = ["zstd/thin"]
cross-lang-fat-lto = ["zstd/fat-lto"]
gh-api-client = ["serde", "serde_json"]
gh-api-client = ["json"]
json = ["serde", "serde_json"]
[package.metadata.docs.rs]
all-features = true

View file

@ -23,6 +23,7 @@ mod zip_extraction;
pub use zip_extraction::ZipError;
#[derive(Debug, ThisError)]
#[non_exhaustive]
pub enum DownloadError {
#[error("Failed to extract zipfile: {0}")]
Unzip(#[from] ZipError),

View file

@ -11,7 +11,7 @@ use tokio::sync::OnceCell;
use crate::remote;
mod request;
pub use request::{GhApiError, JsonError};
pub use request::GhApiError;
/// default retry duration if x-ratelimit-reset is not found in response header
const DEFAULT_RETRY_DURATION: Duration = Duration::from_secs(3);

View file

@ -8,22 +8,17 @@ use std::{
use compact_str::CompactString;
use serde::Deserialize;
use serde_json::from_slice as json_from_slice;
use thiserror::Error as ThisError;
use url::Url;
pub use serde_json::Error as JsonError;
use super::{remote, GhRelease};
#[derive(ThisError, Debug)]
#[non_exhaustive]
pub enum GhApiError {
#[error("IO Error: {0}")]
Io(#[from] io::Error),
#[error("Failed to parse json: {0}")]
Json(#[from] JsonError),
#[error("Remote Error: {0}")]
Remote(#[from] remote::Error),
@ -129,7 +124,5 @@ pub(super) async fn fetch_release_artifacts(
return Ok(FetchReleaseRet::ReleaseNotFound);
}
let bytes = response.error_for_status()?.bytes().await?;
Ok(FetchReleaseRet::Artifacts(json_from_slice(&bytes)?))
Ok(FetchReleaseRet::Artifacts(response.json().await?))
}

View file

@ -28,6 +28,9 @@ pub use certificate::Certificate;
mod request_builder;
pub use request_builder::{RequestBuilder, Response};
#[cfg(feature = "json")]
pub use request_builder::JsonError;
const MAX_RETRY_DURATION: Duration = Duration::from_secs(120);
const MAX_RETRY_COUNT: u8 = 3;
const DEFAULT_RETRY_DURATION_FOR_RATE_LIMIT: Duration = Duration::from_millis(200);
@ -35,12 +38,17 @@ const RETRY_DURATION_FOR_TIMEOUT: Duration = Duration::from_millis(200);
const DEFAULT_MIN_TLS: tls::Version = tls::Version::TLS_1_2;
#[derive(Debug, ThisError)]
#[non_exhaustive]
pub enum Error {
#[error("Reqwest error: {0}")]
Reqwest(#[from] reqwest::Error),
#[error(transparent)]
Http(Box<HttpError>),
#[cfg(feature = "json")]
#[error("Failed to parse http response body as Json: {0}")]
Json(#[from] JsonError),
}
#[derive(Debug, ThisError)]

View file

@ -1,18 +1,22 @@
use reqwest::tls;
use super::ReqwestError;
use super::Error;
#[derive(Clone, Debug)]
pub struct Certificate(pub(super) tls::Certificate);
impl Certificate {
/// Create a Certificate from a binary DER encoded certificate
pub fn from_der(der: impl AsRef<[u8]>) -> Result<Self, ReqwestError> {
tls::Certificate::from_der(der.as_ref()).map(Self)
pub fn from_der(der: impl AsRef<[u8]>) -> Result<Self, Error> {
tls::Certificate::from_der(der.as_ref())
.map(Self)
.map_err(Error::from)
}
/// Create a Certificate from a PEM encoded certificate
pub fn from_pem(pem: impl AsRef<[u8]>) -> Result<Self, ReqwestError> {
tls::Certificate::from_pem(pem.as_ref()).map(Self)
pub fn from_pem(pem: impl AsRef<[u8]>) -> Result<Self, Error> {
tls::Certificate::from_pem(pem.as_ref())
.map(Self)
.map_err(Error::from)
}
}

View file

@ -6,6 +6,9 @@ use reqwest::Method;
use super::{header, Client, Error, HttpError, StatusCode, Url};
#[cfg(feature = "json")]
pub use serde_json::Error as JsonError;
#[derive(Debug)]
pub struct RequestBuilder {
pub(super) client: Client,
@ -96,4 +99,13 @@ impl Response {
pub fn headers(&self) -> &header::HeaderMap {
self.inner.headers()
}
#[cfg(feature = "json")]
pub async fn json<T>(self) -> Result<T, Error>
where
T: serde::de::DeserializeOwned,
{
let bytes = self.error_for_status()?.bytes().await?;
Ok(serde_json::from_slice(&bytes)?)
}
}

View file

@ -16,7 +16,6 @@ binstalk-types = { version = "0.2.1", path = "../binstalk-types" }
cargo_toml = "0.15.2"
command-group = { version = "2.0.1", features = ["with-tokio"] }
compact_str = { version = "0.7.0", features = ["serde"] }
crates_io_api = { version = "0.8.1", default-features = false }
detect-targets = { version = "0.1.5", path = "../detect-targets" }
either = "1.8.1"
home = "0.5.4"
@ -49,7 +48,7 @@ pkg-config = ["binstalk-downloader/pkg-config"]
zlib-ng = ["binstalk-downloader/zlib-ng"]
rustls = ["crates_io_api/rustls", "binstalk-downloader/rustls"]
rustls = ["binstalk-downloader/rustls"]
native-tls = ["binstalk-downloader/native-tls"]
trust-dns = ["binstalk-downloader/trust-dns"]

View file

@ -1,5 +1,2 @@
mod version;
use version::find_version;
mod crates_io;
pub use crates_io::fetch_crate_cratesio;

View file

@ -1,8 +1,9 @@
use std::path::PathBuf;
use cargo_toml::Manifest;
use crates_io_api::AsyncClient;
use compact_str::CompactString;
use semver::VersionReq;
use serde::Deserialize;
use tracing::debug;
use crate::{
@ -14,43 +15,98 @@ use crate::{
manifests::cargo_toml_binstall::{Meta, TarBasedFmt},
};
use super::find_version;
mod vfs;
mod visitor;
use visitor::ManifestVisitor;
/// Fetch a crate Cargo.toml by name and version from crates.io
#[derive(Deserialize)]
struct CrateInfo {
#[serde(rename = "crate")]
inner: CrateInfoInner,
}
#[derive(Deserialize)]
struct CrateInfoInner {
max_stable_version: CompactString,
}
#[derive(Deserialize)]
struct Versions {
versions: Vec<Version>,
}
#[derive(Deserialize)]
struct Version {
num: CompactString,
yanked: bool,
}
/// Find the crate by name, get its latest stable version matches `version_req`,
/// retrieve its Cargo.toml and infer all its bins.
pub async fn fetch_crate_cratesio(
client: Client,
crates_io_api_client: &AsyncClient,
name: &str,
version_req: &VersionReq,
) -> Result<Manifest<Meta>, BinstallError> {
// Fetch / update index
debug!("Looking up crate information");
// Fetch online crate information
let base_info = crates_io_api_client.get_crate(name).await.map_err(|err| {
Box::new(CratesIoApiError {
crate_name: name.into(),
err,
})
})?;
let response = client
.get(Url::parse(&format!(
"https://crates.io/api/v1/crates/{name}"
))?)
.send(true)
.await
.map_err(|err| {
BinstallError::CratesIoApi(Box::new(CratesIoApiError {
crate_name: name.into(),
err,
}))
})?;
// Locate matching version
let version_iter = base_info.versions.iter().filter(|v| !v.yanked);
let (version, version_name) = find_version(version_req, version_iter)?;
let version = if version_req == &VersionReq::STAR {
let crate_info: CrateInfo = response.json().await?;
crate_info.inner.max_stable_version
} else {
let response: Versions = response.json().await?;
response
.versions
.into_iter()
.filter_map(|item| {
if !item.yanked {
// Remove leading `v` for git tags
let num = if let Some(num) = item.num.strip_prefix('v') {
num.into()
} else {
item.num
};
debug!("Found information for crate version: '{}'", version.num);
// Parse out version
let ver = semver::Version::parse(&num).ok()?;
// Filter by version match
version_req.matches(&ver).then_some((num, ver))
} else {
None
}
})
// Return highest version
.max_by(|(_ver_str_x, ver_x), (_ver_str_y, ver_y)| ver_x.cmp(ver_y))
.ok_or_else(|| BinstallError::VersionMismatch {
req: version_req.clone(),
})?
.0
};
debug!("Found information for crate version: '{version}'");
// Download crate to temporary dir (crates.io or git?)
let crate_url = format!("https://crates.io/{}", version.dl_path);
let crate_url = format!("https://crates.io/api/v1/crates/{name}/{version}/download");
debug!("Fetching crate from: {crate_url} and extracting Cargo.toml from it");
let manifest_dir_path: PathBuf = format!("{name}-{version_name}").into();
let manifest_dir_path: PathBuf = format!("{name}-{version}").into();
let mut manifest_visitor = ManifestVisitor::new(manifest_dir_path);

View file

@ -1,46 +0,0 @@
use semver::VersionReq;
use crate::errors::BinstallError;
pub(super) trait Version {
/// Return `None` on error.
fn get_version(&self) -> Option<semver::Version>;
}
impl<T: Version> Version for &T {
fn get_version(&self) -> Option<semver::Version> {
(*self).get_version()
}
}
impl Version for crates_io_api::Version {
fn get_version(&self) -> Option<semver::Version> {
// Remove leading `v` for git tags
let ver_str = match self.num.strip_prefix('v') {
Some(v) => v,
None => &self.num,
};
// Parse out version
semver::Version::parse(ver_str).ok()
}
}
pub(super) fn find_version<Item: Version, VersionIter: Iterator<Item = Item>>(
version_req: &VersionReq,
version_iter: VersionIter,
) -> Result<(Item, semver::Version), BinstallError> {
version_iter
// Filter for matching versions
.filter_map(|item| {
let ver = item.get_version()?;
// Filter by version match
version_req.matches(&ver).then_some((item, ver))
})
// Return highest version
.max_by(|(_item_x, ver_x), (_item_y, ver_y)| ver_x.cmp(ver_y))
.ok_or_else(|| BinstallError::VersionMismatch {
req: version_req.clone(),
})
}

View file

@ -5,9 +5,7 @@ use std::{
};
use binstalk_downloader::{
download::{DownloadError, ZipError},
gh_api_client::GhApiError,
remote::{Error as RemoteError, HttpError, ReqwestError},
download::DownloadError, gh_api_client::GhApiError, remote::Error as RemoteError,
};
use cargo_toml::Error as CargoTomlError;
use compact_str::CompactString;
@ -22,7 +20,7 @@ use tracing::{error, warn};
pub struct CratesIoApiError {
pub crate_name: CompactString,
#[source]
pub err: crates_io_api::Error,
pub err: RemoteError,
}
#[derive(Debug, Error)]
@ -84,14 +82,6 @@ pub enum BinstallError {
#[diagnostic(severity(error), code(binstall::url_parse))]
UrlParse(#[from] url::ParseError),
/// An error while unzipping a file.
///
/// - Code: `binstall::unzip`
/// - Exit: 66
#[error("Failed to extract zipfile: {0}")]
#[diagnostic(severity(error), code(binstall::unzip))]
Unzip(#[from] ZipError),
/// A rendering error in a template.
///
/// - Code: `binstall::template`
@ -100,26 +90,13 @@ pub enum BinstallError {
#[diagnostic(severity(error), code(binstall::template))]
Template(Box<TinyTemplateError>),
/// A generic error from our HTTP client, reqwest.
/// Failed to download or failed to decode the body.
///
/// Errors resulting from HTTP fetches are handled with [`BinstallError::Http`] instead.
///
/// - Code: `binstall::reqwest`
/// - Code: `binstall::download`
/// - Exit: 68
#[error("Reqwest error: {0}")]
#[diagnostic(severity(error), code(binstall::reqwest))]
Reqwest(#[from] ReqwestError),
/// An HTTP request failed.
///
/// This includes both connection/transport failures and when the HTTP status of the response
/// is not as expected.
///
/// - Code: `binstall::http`
/// - Exit: 69
#[error(transparent)]
#[diagnostic(severity(error), code(binstall::http))]
Http(#[from] Box<HttpError>),
#[diagnostic(severity(error), code(binstall::download))]
Download(#[from] DownloadError),
/// A subprocess failed.
///
@ -331,10 +308,8 @@ impl BinstallError {
TaskJoinError(_) => 17,
UserAbort => 32,
UrlParse(_) => 65,
Unzip(_) => 66,
Template(_) => 67,
Reqwest(_) => 68,
Http { .. } => 69,
Download(_) => 68,
SubProcess { .. } => 70,
Io(_) => 74,
CratesIoApi { .. } => 76,
@ -425,24 +400,7 @@ impl From<BinstallError> for io::Error {
impl From<RemoteError> for BinstallError {
fn from(e: RemoteError) -> Self {
use RemoteError::*;
match e {
Reqwest(reqwest_error) => reqwest_error.into(),
Http(http_error) => http_error.into(),
}
}
}
impl From<DownloadError> for BinstallError {
fn from(e: DownloadError) -> Self {
use DownloadError::*;
match e {
Unzip(zip_error) => zip_error.into(),
Remote(remote_error) => remote_error.into(),
Io(io_error) => io_error.into(),
}
DownloadError::from(e).into()
}
}

View file

@ -2,7 +2,6 @@
use std::{path::PathBuf, sync::Arc};
use crates_io_api::AsyncClient as CratesIoApiClient;
use semver::VersionReq;
use crate::{
@ -34,7 +33,6 @@ pub struct Options {
pub temp_dir: PathBuf,
pub install_path: PathBuf,
pub client: Client,
pub crates_io_api_client: CratesIoApiClient,
pub gh_api_client: GhApiClient,
pub jobserver_client: LazyJobserverClient,
}

View file

@ -8,7 +8,6 @@ use std::{
use cargo_toml::Manifest;
use compact_str::{CompactString, ToCompactString};
use crates_io_api::AsyncClient as CratesIoApiClient;
use itertools::Itertools;
use maybe_owned::MaybeOwned;
use semver::{Version, VersionReq};
@ -72,8 +71,7 @@ async fn resolve_inner(
crate_name.name,
curr_version,
&version_req,
opts.client.clone(),
&opts.crates_io_api_client).await?
opts.client.clone()).await?
else {
return Ok(Resolution::AlreadyUpToDate)
};
@ -350,20 +348,11 @@ impl PackageInfo {
curr_version: Option<Version>,
version_req: &VersionReq,
client: Client,
crates_io_api_client: &CratesIoApiClient,
) -> Result<Option<Self>, BinstallError> {
// Fetch crate via crates.io, git, or use a local manifest path
let manifest = match opts.manifest_path.as_ref() {
Some(manifest_path) => load_manifest_path(manifest_path)?,
None => {
Box::pin(fetch_crate_cratesio(
client,
crates_io_api_client,
&name,
version_req,
))
.await?
}
None => Box::pin(fetch_crate_cratesio(client, &name, version_req)).await?,
};
let Some(mut package) = manifest.package else {