actions/cargo-binstall
1
0
Fork 0
mirror of https://github.com/cargo-bins/cargo-binstall.git synced 2025-04-24 22:30:03 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor: Extract new crate binstalk-registry (#1289)

Browse source 
To speedup codegen of `binstalk` and enable it to be reused.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2023-08-13 17:16:53 +10:00 • committed by GitHub
parent 6c801a97ae
commit 623f7ff4ed
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 373 additions and 127 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crates/binstalk/Cargo.toml
View file

@ -12,8 +12,8 @@ license = "GPL-3.0-only"
[dependencies]
async-trait = "0.1.68"
atomic-file-install = { version = "0.0.0", path = "../atomic-file-install" }
base16 = "0.2.1"
binstalk-downloader = { version = "0.7.0", path = "../binstalk-downloader", default-features = false, features = ["gh-api-client"] }
binstalk-registry = { version = "0.0.0", path = "../binstalk-registry" }
binstalk-types = { version = "0.5.0", path = "../binstalk-types" }
cargo-toml-workspace = { version = "0.0.0", path = "../cargo-toml-workspace" }
command-group = { version = "2.1.0", features = ["with-tokio"] }
@ -30,9 +30,6 @@ miette = "5.9.0"
normalize-path = { version = "0.2.1", path = "../normalize-path" }
once_cell = "1.18.0"
semver = { version = "1.0.17", features = ["serde"] }
serde = { version = "1.0.163", features = ["derive"] }
serde_json = "1.0.99"
sha2 = "0.10.7"
strum = "0.25.0"
target-lexicon = { version = "0.12.11", features = ["std"] }
tempfile = "3.5.0"
@ -40,16 +37,12 @@ thiserror = "1.0.40"
tokio = { version = "1.30.0", features = ["rt", "process", "sync"], default-features = false }
tracing = "0.1.37"
url = { version = "2.3.1", features = ["serde"] }
xz2 = "0.1.7"
[dev-dependencies]
toml_edit = { version = "0.19.11", features = ["serde"] }
[features]
default = ["static", "rustls", "git"]
git = ["binstalk-downloader/git"]
git-max-perf = ["binstalk-downloader/git-max-perf"]
git = ["binstalk-registry/git"]
git-max-perf = ["git", "binstalk-downloader/git-max-perf"]
static = ["binstalk-downloader/static"]
pkg-config = ["binstalk-downloader/pkg-config"]

8
crates/binstalk/src/drivers.rs
View file

@ -1,8 +0,0 @@
mod registry;
pub use registry::{
fetch_crate_cratesio, CratesIoRateLimit, InvalidRegistryError, Registry, RegistryError,
SparseRegistry,
};
#[cfg(feature = "git")]
pub use registry::{GitRegistry, GitUrl, GitUrlParseError};

261
crates/binstalk/src/drivers/registry.rs
View file

@ -1,261 +0,0 @@
use std::{str::FromStr, sync::Arc};
use base16::DecodeError as Base16DecodeError;
use compact_str::CompactString;
use leon::{ParseError, RenderError};
use miette::Diagnostic;
use semver::VersionReq;
use serde_json::Error as JsonError;
use thiserror::Error as ThisError;
use crate::{
errors::BinstallError,
helpers::{
cargo_toml::Manifest,
remote::{Client, Error as RemoteError, Url, UrlParseError},
},
manifests::cargo_toml_binstall::Meta,
};
#[cfg(feature = "git")]
pub use crate::helpers::git::{GitUrl, GitUrlParseError};
mod vfs;
mod visitor;
mod common;
use common::*;
#[cfg(feature = "git")]
mod git_registry;
#[cfg(feature = "git")]
pub use git_registry::GitRegistry;
mod crates_io_registry;
pub use crates_io_registry::{fetch_crate_cratesio, CratesIoRateLimit};
mod sparse_registry;
pub use sparse_registry::SparseRegistry;
#[derive(Debug, ThisError, Diagnostic)]
#[diagnostic(severity(error), code(binstall::cargo_registry))]
#[non_exhaustive]
pub enum RegistryError {
#[error(transparent)]
Remote(#[from] RemoteError),
#[error("{0} is not found")]
#[diagnostic(
help("Check that the crate name you provided is correct.\nYou can also search for a matching crate at: https://lib.rs/search?q={0}")
)]
NotFound(CompactString),
#[error(transparent)]
Json(#[from] JsonError),
#[error("Failed to parse dl config: {0}")]
ParseDlConfig(#[from] ParseError),
#[error("Failed to render dl config: {0}")]
RenderDlConfig(#[from] RenderError),
#[error("Failed to parse checksum encoded in hex: {0}")]
InvalidHex(#[from] Base16DecodeError),
#[error("Expected checksum `{expected}`, actual checksum `{actual}`")]
UnmatchedChecksum { expected: String, actual: String },
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum Registry {
CratesIo(Arc<CratesIoRateLimit>),
Sparse(Arc<SparseRegistry>),
#[cfg(feature = "git")]
Git(GitRegistry),
}
impl Default for Registry {
fn default() -> Self {
Self::CratesIo(Default::default())
}
}
#[derive(Debug, ThisError)]
#[error("Invalid registry `{src}`, {inner}")]
pub struct InvalidRegistryError {
src: CompactString,
#[source]
inner: InvalidRegistryErrorInner,
}
#[derive(Debug, ThisError)]
enum InvalidRegistryErrorInner {
#[cfg(feature = "git")]
#[error("failed to parse git url {0}")]
GitUrlParseErr(#[from] Box<GitUrlParseError>),
#[error("failed to parse sparse registry url: {0}")]
UrlParseErr(#[from] UrlParseError),
#[error("expected protocol http(s), actual protocl {0}")]
InvalidScheme(CompactString),
#[cfg(not(feature = "git"))]
#[error("git registry not supported")]
GitRegistryNotSupported,
}
impl Registry {
fn from_str_inner(s: &str) -> Result<Self, InvalidRegistryErrorInner> {
if let Some(s) = s.strip_prefix("sparse+") {
let url = Url::parse(s)?;
let scheme = url.scheme();
if scheme != "http" && scheme != "https" {
Err(InvalidRegistryErrorInner::InvalidScheme(scheme.into()))
} else {
Ok(Self::Sparse(Arc::new(SparseRegistry::new(url))))
}
} else {
#[cfg(not(feature = "git"))]
{
Err(InvalidRegistryErrorInner::GitRegistryNotSupported)
}
#[cfg(feature = "git")]
{
let url = GitUrl::from_str(s).map_err(Box::new)?;
Ok(Self::Git(GitRegistry::new(url)))
}
}
}
/// Fetch the latest crate with `crate_name` and with version matching
/// `version_req`.
pub async fn fetch_crate_matched(
&self,
client: Client,
crate_name: &str,
version_req: &VersionReq,
) -> Result<Manifest<Meta>, BinstallError> {
match self {
Self::CratesIo(rate_limit) => {
fetch_crate_cratesio(client, crate_name, version_req, rate_limit).await
}
Self::Sparse(sparse_registry) => {
sparse_registry
.fetch_crate_matched(client, crate_name, version_req)
.await
}
#[cfg(feature = "git")]
Self::Git(git_registry) => {
git_registry
.fetch_crate_matched(client, crate_name, version_req)
.await
}
}
}
}
impl FromStr for Registry {
type Err = InvalidRegistryError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::from_str_inner(s).map_err(|inner| InvalidRegistryError {
src: s.into(),
inner,
})
}
}
#[cfg(test)]
mod test {
use std::num::NonZeroU16;
use toml_edit::ser::to_string;
use super::*;
/// Mark this as an async fn so that you won't accidentally use it in
/// sync context.
async fn create_client() -> Client {
Client::new(
concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")),
None,
NonZeroU16::new(10).unwrap(),
1.try_into().unwrap(),
[],
)
.unwrap()
}
#[tokio::test]
async fn test_crates_io_sparse_registry() {
let client = create_client().await;
let sparse_registry: Registry = "sparse+https://index.crates.io/".parse().unwrap();
assert!(
matches!(sparse_registry, Registry::Sparse(_)),
"{:?}",
sparse_registry
);
let crate_name = "cargo-binstall";
let version_req = &VersionReq::parse("=1.0.0").unwrap();
let manifest_from_sparse = sparse_registry
.fetch_crate_matched(client.clone(), crate_name, version_req)
.await
.unwrap();
let manifest_from_cratesio_api = Registry::default()
.fetch_crate_matched(client, crate_name, version_req)
.await
.unwrap();
let serialized_manifest_from_sparse = to_string(&manifest_from_sparse).unwrap();
let serialized_manifest_from_cratesio_api = to_string(&manifest_from_cratesio_api).unwrap();
assert_eq!(
serialized_manifest_from_sparse,
serialized_manifest_from_cratesio_api
);
}
#[cfg(feature = "git")]
#[tokio::test]
async fn test_crates_io_git_registry() {
let client = create_client().await;
let git_registry: Registry = "https://github.com/rust-lang/crates.io-index"
.parse()
.unwrap();
assert!(
matches!(git_registry, Registry::Git(_)),
"{:?}",
git_registry
);
let crate_name = "cargo-binstall";
let version_req = &VersionReq::parse("=1.0.0").unwrap();
let manifest_from_git = git_registry
.fetch_crate_matched(client.clone(), crate_name, version_req)
.await
.unwrap();
let manifest_from_cratesio_api = Registry::default()
.fetch_crate_matched(client, crate_name, version_req)
.await
.unwrap();
let serialized_manifest_from_git = to_string(&manifest_from_git).unwrap();
let serialized_manifest_from_cratesio_api = to_string(&manifest_from_cratesio_api).unwrap();
assert_eq!(
serialized_manifest_from_git,
serialized_manifest_from_cratesio_api
);
}
}

196
crates/binstalk/src/drivers/registry/common.rs
View file

@ -1,196 +0,0 @@
use std::borrow::Cow;
use base16::{decode as decode_base16, encode_lower as encode_base16};
use compact_str::{format_compact, CompactString, ToCompactString};
use leon::{Template, Values};
use semver::{Version, VersionReq};
use serde::Deserialize;
use serde_json::Error as JsonError;
use sha2::{Digest, Sha256};
use tracing::debug;
use crate::{
drivers::registry::{visitor::ManifestVisitor, RegistryError},
errors::BinstallError,
helpers::{
bytes::Bytes,
cargo_toml::Manifest,
download::{DataVerifier, Download},
remote::{Client, Url},
},
manifests::cargo_toml_binstall::{Meta, TarBasedFmt},
};
#[derive(Deserialize)]
pub(super) struct RegistryConfig {
pub(super) dl: CompactString,
}
struct Sha256Digest(Sha256);
impl Default for Sha256Digest {
fn default() -> Self {
Sha256Digest(Sha256::new())
}
}
impl DataVerifier for Sha256Digest {
fn update(&mut self, data: &Bytes) {
self.0.update(data);
}
}
pub(super) async fn parse_manifest(
client: Client,
crate_name: &str,
crate_url: Url,
MatchedVersion { version, cksum }: MatchedVersion,
) -> Result<Manifest<Meta>, BinstallError> {
debug!("Fetching crate from: {crate_url} and extracting Cargo.toml from it");
let mut manifest_visitor = ManifestVisitor::new(format!("{crate_name}-{version}").into());
let checksum = decode_base16(cksum.as_bytes()).map_err(RegistryError::from)?;
let mut sha256_digest = Sha256Digest::default();
Download::new_with_data_verifier(client, crate_url, &mut sha256_digest)
.and_visit_tar(TarBasedFmt::Tgz, &mut manifest_visitor)
.await?;
let digest_checksum = sha256_digest.0.finalize();
if digest_checksum.as_slice() != checksum.as_slice() {
Err(RegistryError::UnmatchedChecksum {
expected: cksum,
actual: encode_base16(digest_checksum.as_slice()),
}
.into())
} else {
manifest_visitor.load_manifest()
}
}
/// Return components of crate prefix
pub(super) fn crate_prefix_components(
crate_name: &str,
) -> Result<(CompactString, Option<CompactString>), RegistryError> {
let mut chars = crate_name.chars();
match (chars.next(), chars.next(), chars.next(), chars.next()) {
(None, None, None, None) => Err(RegistryError::NotFound(crate_name.into())),
(Some(_), None, None, None) => Ok((CompactString::new("1"), None)),
(Some(_), Some(_), None, None) => Ok((CompactString::new("2"), None)),
(Some(ch), Some(_), Some(_), None) => Ok((
CompactString::new("3"),
Some(ch.to_lowercase().to_compact_string()),
)),
(Some(a), Some(b), Some(c), Some(d)) => Ok((
format_compact!("{}{}", a.to_lowercase(), b.to_lowercase()),
Some(format_compact!("{}{}", c.to_lowercase(), d.to_lowercase())),
)),
_ => unreachable!(),
}
}
pub(super) fn render_dl_template(
dl_template: &str,
crate_name: &str,
(c1, c2): &(CompactString, Option<CompactString>),
MatchedVersion { version, cksum }: &MatchedVersion,
) -> Result<String, RegistryError> {
let template = Template::parse(dl_template)?;
if template.keys().next().is_some() {
let mut crate_prefix = c1.clone();
if let Some(c2) = c2 {
crate_prefix.push('/');
crate_prefix.push_str(c2);
}
struct Context<'a> {
crate_name: &'a str,
crate_prefix: CompactString,
crate_lowerprefix: String,
version: &'a str,
cksum: &'a str,
}
impl Values for Context<'_> {
fn get_value(&self, key: &str) -> Option<Cow<'_, str>> {
match key {
"crate" => Some(Cow::Borrowed(self.crate_name)),
"version" => Some(Cow::Borrowed(self.version)),
"prefix" => Some(Cow::Borrowed(&self.crate_prefix)),
"lowerprefix" => Some(Cow::Borrowed(&self.crate_lowerprefix)),
"sha256-checksum" => Some(Cow::Borrowed(self.cksum)),
_ => None,
}
}
}
Ok(template.render(&Context {
crate_name,
crate_lowerprefix: crate_prefix.to_lowercase(),
crate_prefix,
version,
cksum,
})?)
} else {
Ok(format!("{dl_template}/{crate_name}/{version}/download"))
}
}
#[derive(Deserialize)]
pub(super) struct RegistryIndexEntry {
vers: CompactString,
yanked: bool,
cksum: String,
}
pub(super) struct MatchedVersion {
pub(super) version: CompactString,
/// sha256 checksum encoded in base16
pub(super) cksum: String,
}
impl MatchedVersion {
pub(super) fn find(
it: &mut dyn Iterator<Item = Result<RegistryIndexEntry, JsonError>>,
version_req: &VersionReq,
) -> Result<Self, BinstallError> {
let mut ret = Option::<(Self, Version)>::None;
for res in it {
let entry = res.map_err(RegistryError::from)?;
if entry.yanked {
continue;
}
let num = entry.vers;
// Parse out version
let Ok(ver) = Version::parse(&num) else { continue };
// Filter by version match
if !version_req.matches(&ver) {
continue;
}
let matched = Self {
version: num,
cksum: entry.cksum,
};
if let Some((_, max_ver)) = &ret {
if ver > *max_ver {
ret = Some((matched, ver));
}
} else {
ret = Some((matched, ver));
}
}
ret.map(|(num, _)| num)
.ok_or_else(|| BinstallError::VersionMismatch {
req: version_req.clone(),
})
}
}

202
crates/binstalk/src/drivers/registry/crates_io_registry.rs
View file

@ -1,202 +0,0 @@
use binstalk_downloader::remote::Error as RemoteError;
use compact_str::{CompactString, ToCompactString};
use semver::{Comparator, Op as ComparatorOp, Version as SemVersion, VersionReq};
use serde::Deserialize;
use tokio::{
sync::Mutex,
time::{interval, Duration, Interval, MissedTickBehavior},
};
use tracing::debug;
use crate::{
drivers::registry::{parse_manifest, MatchedVersion, RegistryError},
errors::BinstallError,
helpers::{
cargo_toml::Manifest,
remote::{Client, Url},
},
manifests::cargo_toml_binstall::Meta,
};
#[derive(Debug)]
pub struct CratesIoRateLimit(Mutex<Interval>);
impl Default for CratesIoRateLimit {
fn default() -> Self {
let mut interval = interval(Duration::from_secs(1));
// If somehow one tick is delayed, then next tick should be at least
// 1s later than the current tick.
//
// Other MissedTickBehavior including Burst (default), which will
// tick as fast as possible to catch up, and Skip, which will
// skip the current tick for the next one.
//
// Both Burst and Skip is not the expected behavior for rate limit:
// ticking as fast as possible would violate crates.io crawler
// policy, and skipping the current one will slow down the resolution
// process.
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
Self(Mutex::new(interval))
}
}
impl CratesIoRateLimit {
pub(super) async fn tick(&self) {
self.0.lock().await.tick().await;
}
}
/// Return `Some(checksum)` if the version is not yanked, otherwise `None`.
async fn is_crate_yanked(client: &Client, url: Url) -> Result<Option<String>, RemoteError> {
#[derive(Deserialize)]
struct CrateInfo {
version: Inner,
}
#[derive(Deserialize)]
struct Inner {
yanked: bool,
checksum: String,
}
// Fetch / update index
debug!("Looking up crate information");
let info: CrateInfo = client.get(url).send(true).await?.json().await?;
let version = info.version;
Ok((!version.yanked).then_some(version.checksum))
}
async fn fetch_crate_cratesio_version_matched(
client: &Client,
url: Url,
version_req: &VersionReq,
) -> Result<Option<(CompactString, String)>, RemoteError> {
#[derive(Deserialize)]
struct CrateInfo {
#[serde(rename = "crate")]
inner: CrateInfoInner,
versions: Vec<Version>,
}
#[derive(Deserialize)]
struct CrateInfoInner {
max_stable_version: CompactString,
}
#[derive(Deserialize)]
struct Version {
num: CompactString,
yanked: bool,
checksum: String,
}
// Fetch / update index
debug!("Looking up crate information");
let crate_info: CrateInfo = client.get(url).send(true).await?.json().await?;
let version_with_checksum = if version_req == &VersionReq::STAR {
let version = crate_info.inner.max_stable_version;
crate_info
.versions
.into_iter()
.find_map(|v| (v.num.as_str() == version.as_str()).then_some(v.checksum))
.map(|checksum| (version, checksum))
} else {
crate_info
.versions
.into_iter()
.filter_map(|item| {
if !item.yanked {
// Remove leading `v` for git tags
let num = if let Some(num) = item.num.strip_prefix('v') {
num.into()
} else {
item.num
};
// Parse out version
let ver = semver::Version::parse(&num).ok()?;
// Filter by version match
version_req
.matches(&ver)
.then_some((num, ver, item.checksum))
} else {
None
}
})
// Return highest version
.max_by(
|(_ver_str_x, ver_x, _checksum_x), (_ver_str_y, ver_y, _checksum_y)| {
ver_x.cmp(ver_y)
},
)
.map(|(ver_str, _, checksum)| (ver_str, checksum))
};
Ok(version_with_checksum)
}
/// Find the crate by name, get its latest stable version matches `version_req`,
/// retrieve its Cargo.toml and infer all its bins.
pub async fn fetch_crate_cratesio(
client: Client,
name: &str,
version_req: &VersionReq,
crates_io_rate_limit: &CratesIoRateLimit,
) -> Result<Manifest<Meta>, BinstallError> {
// Wait until we can make another request to crates.io
crates_io_rate_limit.tick().await;
let url = Url::parse(&format!("https://crates.io/api/v1/crates/{name}"))?;
let (version, cksum) = match version_req.comparators.as_slice() {
[Comparator {
op: ComparatorOp::Exact,
major,
minor: Some(minor),
patch: Some(patch),
pre,
}] => {
let version = SemVersion {
major: *major,
minor: *minor,
patch: *patch,
pre: pre.clone(),
build: Default::default(),
}
.to_compact_string();
let mut url = url.clone();
url.path_segments_mut().unwrap().push(&version);
is_crate_yanked(&client, url)
.await
.map(|ret| ret.map(|checksum| (version, checksum)))
}
_ => fetch_crate_cratesio_version_matched(&client, url.clone(), version_req).await,
}
.map_err(|e| match e {
RemoteError::Http(e) if e.is_status() => RegistryError::NotFound(name.into()),
e => e.into(),
})?
.ok_or_else(|| BinstallError::VersionMismatch {
req: version_req.clone(),
})?;
debug!("Found information for crate version: '{version}'");
// Download crate to temporary dir (crates.io or git?)
let mut crate_url = url;
crate_url
.path_segments_mut()
.unwrap()
.push(&version)
.push("download");
parse_manifest(client, name, crate_url, MatchedVersion { version, cksum }).await
}

148
crates/binstalk/src/drivers/registry/git_registry.rs
View file

@ -1,148 +0,0 @@
use std::{io, path::PathBuf, sync::Arc};
use compact_str::{CompactString, ToCompactString};
use once_cell::sync::OnceCell;
use semver::VersionReq;
use serde_json::{from_slice as json_from_slice, Deserializer as JsonDeserializer};
use tempfile::TempDir;
use tokio::task::spawn_blocking;
use url::Url;
use crate::{
drivers::registry::{
crate_prefix_components, parse_manifest, render_dl_template, MatchedVersion,
RegistryConfig, RegistryError,
},
errors::BinstallError,
helpers::{
cargo_toml::Manifest,
git::{GitCancellationToken, GitUrl, Repository},
remote::Client,
},
manifests::cargo_toml_binstall::Meta,
};
#[derive(Debug)]
struct GitIndex {
_tempdir: TempDir,
repo: Repository,
dl_template: CompactString,
}
impl GitIndex {
fn new(url: GitUrl, cancellation_token: GitCancellationToken) -> Result<Self, BinstallError> {
let tempdir = TempDir::new()?;
let repo = Repository::shallow_clone_bare(
url.clone(),
tempdir.as_ref(),
Some(cancellation_token),
)?;
let config: RegistryConfig = {
let config = repo
.get_head_commit_entry_data_by_path("config.json")?
.ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
format!("config.json not found in repository `{url}`"),
)
})?;
json_from_slice(&config).map_err(RegistryError::from)?
};
Ok(Self {
_tempdir: tempdir,
repo,
dl_template: config.dl,
})
}
}
#[derive(Debug)]
struct GitRegistryInner {
url: GitUrl,
git_index: OnceCell<GitIndex>,
}
#[derive(Clone, Debug)]
pub struct GitRegistry(Arc<GitRegistryInner>);
impl GitRegistry {
pub fn new(url: GitUrl) -> Self {
Self(Arc::new(GitRegistryInner {
url,
git_index: Default::default(),
}))
}
/// WARNING: This is a blocking operation.
fn find_crate_matched_ver(
repo: &Repository,
crate_name: &str,
(c1, c2): &(CompactString, Option<CompactString>),
version_req: &VersionReq,
) -> Result<MatchedVersion, BinstallError> {
let mut path = PathBuf::with_capacity(128);
path.push(&**c1);
if let Some(c2) = c2 {
path.push(&**c2);
}
path.push(&*crate_name.to_lowercase());
let crate_versions = repo
.get_head_commit_entry_data_by_path(path)?
.ok_or_else(|| RegistryError::NotFound(crate_name.into()))?;
MatchedVersion::find(
&mut JsonDeserializer::from_slice(&crate_versions).into_iter(),
version_req,
)
}
pub async fn fetch_crate_matched(
&self,
client: Client,
name: &str,
version_req: &VersionReq,
) -> Result<Manifest<Meta>, BinstallError> {
let crate_prefix = crate_prefix_components(name)?;
let crate_name = name.to_compact_string();
let version_req = version_req.clone();
let this = self.clone();
let cancellation_token = GitCancellationToken::default();
// Cancel git operation if the future is cancelled (dropped).
let cancel_on_drop = cancellation_token.clone().cancel_on_drop();
let (matched_version, dl_url) = spawn_blocking(move || {
let GitIndex {
_tempdir: _,
repo,
dl_template,
} = this
.0
.git_index
.get_or_try_init(|| GitIndex::new(this.0.url.clone(), cancellation_token))?;
let matched_version =
Self::find_crate_matched_ver(repo, &crate_name, &crate_prefix, &version_req)?;
let url = Url::parse(&render_dl_template(
dl_template,
&crate_name,
&crate_prefix,
&matched_version,
)?)?;
Ok::<_, BinstallError>((matched_version, url))
})
.await??;
// Git operation done, disarm it
cancel_on_drop.disarm();
parse_manifest(client, name, dl_url, matched_version).await
}
}

110
crates/binstalk/src/drivers/registry/sparse_registry.rs
View file

@ -1,110 +0,0 @@
use compact_str::CompactString;
use semver::VersionReq;
use serde_json::Deserializer as JsonDeserializer;
use tokio::sync::OnceCell;
use url::Url;
use crate::{
drivers::registry::{
crate_prefix_components, parse_manifest, render_dl_template, MatchedVersion,
RegistryConfig, RegistryError,
},
errors::BinstallError,
helpers::{
cargo_toml::Manifest,
remote::{Client, Error as RemoteError},
},
manifests::cargo_toml_binstall::Meta,
};
#[derive(Debug)]
pub struct SparseRegistry {
url: Url,
dl_template: OnceCell<CompactString>,
}
impl SparseRegistry {
/// * `url` - `url.cannot_be_a_base()` must be `false`
pub fn new(url: Url) -> Self {
Self {
url,
dl_template: Default::default(),
}
}
async fn get_dl_template(&self, client: &Client) -> Result<&str, RegistryError> {
self.dl_template
.get_or_try_init(|| {
Box::pin(async {
let mut url = self.url.clone();
url.path_segments_mut().unwrap().push("config.json");
let config: RegistryConfig = client.get(url).send(true).await?.json().await?;
Ok(config.dl)
})
})
.await
.map(AsRef::as_ref)
}
/// `url` must be a valid http(s) url.
async fn find_crate_matched_ver(
client: &Client,
mut url: Url,
crate_name: &str,
(c1, c2): &(CompactString, Option<CompactString>),
version_req: &VersionReq,
) -> Result<MatchedVersion, BinstallError> {
{
let mut path = url.path_segments_mut().unwrap();
path.push(c1);
if let Some(c2) = c2 {
path.push(c2);
}
path.push(&crate_name.to_lowercase());
}
let body = client
.get(url)
.send(true)
.await
.map_err(|e| match e {
RemoteError::Http(e) if e.is_status() => RegistryError::NotFound(crate_name.into()),
e => e.into(),
})?
.bytes()
.await
.map_err(RegistryError::from)?;
MatchedVersion::find(
&mut JsonDeserializer::from_slice(&body).into_iter(),
version_req,
)
}
pub async fn fetch_crate_matched(
&self,
client: Client,
crate_name: &str,
version_req: &VersionReq,
) -> Result<Manifest<Meta>, BinstallError> {
let crate_prefix = crate_prefix_components(crate_name)?;
let dl_template = self.get_dl_template(&client).await?;
let matched_version = Self::find_crate_matched_ver(
&client,
self.url.clone(),
crate_name,
&crate_prefix,
version_req,
)
.await?;
let dl_url = Url::parse(&render_dl_template(
dl_template,
crate_name,
&crate_prefix,
&matched_version,
)?)?;
parse_manifest(client, crate_name, dl_url, matched_version).await
}
}

43
crates/binstalk/src/drivers/registry/vfs.rs
View file

@ -1,43 +0,0 @@
use std::{
collections::{hash_set::HashSet, BTreeMap},
io,
path::Path,
};
use crate::helpers::cargo_toml::AbstractFilesystem;
use normalize_path::NormalizePath;
/// This type stores the filesystem structure for the crate tarball
/// extracted in memory and can be passed to
/// `cargo_toml::Manifest::complete_from_abstract_filesystem`.
#[derive(Debug, Default)]
pub(super) struct Vfs(BTreeMap<Box<Path>, HashSet<Box<str>>>);
impl Vfs {
/// * `path` - must be canonical, must not be empty.
pub(super) fn add_path(&mut self, mut path: &Path) {
while let Some(parent) = path.parent() {
// Since path has parent, it must have a filename
let filename = path.file_name().unwrap();
// `cargo_toml`'s implementation does the same thing.
// https://docs.rs/cargo_toml/0.11.5/src/cargo_toml/afs.rs.html#24
let filename = filename.to_string_lossy();
self.0
.entry(parent.into())
.or_insert_with(|| HashSet::with_capacity(4))
.insert(filename.into());
path = parent;
}
}
}
impl AbstractFilesystem for Vfs {
fn file_names_in(&self, rel_path: &str) -> io::Result<HashSet<Box<str>>> {
let rel_path = Path::new(rel_path).normalize();
Ok(self.0.get(&*rel_path).map(Clone::clone).unwrap_or_default())
}
}

86
crates/binstalk/src/drivers/registry/visitor.rs
View file

@ -1,86 +0,0 @@
use std::path::{Path, PathBuf};
use normalize_path::NormalizePath;
use tokio::io::AsyncReadExt;
use tracing::debug;
use super::vfs::Vfs;
use crate::{
errors::BinstallError,
helpers::{
cargo_toml::{Manifest, Value},
download::{DownloadError, TarEntriesVisitor, TarEntry},
},
manifests::cargo_toml_binstall::Meta,
};
#[derive(Debug)]
pub(super) struct ManifestVisitor {
cargo_toml_content: Vec<u8>,
/// manifest_dir_path is treated as the current dir.
manifest_dir_path: PathBuf,
vfs: Vfs,
}
impl ManifestVisitor {
pub(super) fn new(manifest_dir_path: PathBuf) -> Self {
Self {
// Cargo.toml is quite large usually.
cargo_toml_content: Vec::with_capacity(2000),
manifest_dir_path,
vfs: Vfs::default(),
}
}
}
#[async_trait::async_trait]
impl TarEntriesVisitor for ManifestVisitor {
async fn visit(&mut self, entry: &mut dyn TarEntry) -> Result<(), DownloadError> {
let path = entry.path()?;
let path = path.normalize();
let path = if let Ok(path) = path.strip_prefix(&self.manifest_dir_path) {
path
} else {
// The path is outside of the curr dir (manifest dir),
// ignore it.
return Ok(());
};
if path == Path::new("Cargo.toml")
|| path == Path::new("src/main.rs")
|| path.starts_with("src/bin")
{
self.vfs.add_path(path);
}
if path == Path::new("Cargo.toml") {
// Since it is possible for the same Cargo.toml to appear
// multiple times using `tar --keep-old-files`, here we
// clear the buffer first before reading into it.
self.cargo_toml_content.clear();
self.cargo_toml_content
.reserve_exact(entry.size()?.try_into().unwrap_or(usize::MAX));
entry.read_to_end(&mut self.cargo_toml_content).await?;
}
Ok(())
}
}
impl ManifestVisitor {
/// Load binstall metadata using the extracted information stored in memory.
pub(super) fn load_manifest(self) -> Result<Manifest<Meta>, BinstallError> {
debug!("Loading manifest directly from extracted file");
// Load and parse manifest
let mut manifest = Manifest::from_slice_with_metadata(&self.cargo_toml_content)?;
// Checks vfs for binary output names
manifest.complete_from_abstract_filesystem::<Value, _>(&self.vfs, None)?;
// Return metadata
Ok(manifest)
}
}

15
crates/binstalk/src/errors.rs
View file

@ -15,10 +15,10 @@ use tokio::task;
use tracing::{error, warn};
use crate::{
drivers::{InvalidRegistryError, RegistryError},
helpers::{
cargo_toml::Error as CargoTomlError, cargo_toml_workspace::Error as LoadManifestFromWSError,
},
registry::{InvalidRegistryError, RegistryError},
};
#[derive(Debug, Error)]
@ -198,18 +198,6 @@ pub enum BinstallError {
#[diagnostic(severity(error), code(binstall::version::parse))]
VersionParse(#[from] Box<VersionParseError>),
/// No available version matches the requirements.
///
/// This may be the case when using the `--version` option.
///
/// Note that using `--version 1.2.3` is interpreted as the requirement `=1.2.3`.
///
/// - Code: `binstall::version::mismatch`
/// - Exit: 82
#[error("no version matching requirement '{req}'")]
#[diagnostic(severity(error), code(binstall::version::mismatch))]
VersionMismatch { req: semver::VersionReq },
/// The crate@version syntax was used at the same time as the --version option.
///
/// You can't do that as it's ambiguous which should apply.
@ -374,7 +362,6 @@ impl BinstallError {
CargoManifest { .. } => 78,
RegistryParseError(..) => 79,
VersionParse { .. } => 80,
VersionMismatch { .. } => 82,
SuperfluousVersionOption => 84,
UnspecifiedBinaries => 86,
NoViableTargets => 87,

2
crates/binstalk/src/helpers.rs
View file

@ -4,8 +4,8 @@ pub mod remote;
pub(crate) mod target_triple;
pub mod tasks;
pub(crate) use binstalk_downloader::download;
pub use binstalk_downloader::gh_api_client;
pub(crate) use binstalk_downloader::{bytes, download};
#[cfg(feature = "git")]
pub(crate) use binstalk_downloader::git;

2
crates/binstalk/src/lib.rs
View file

@ -1,13 +1,13 @@
#![cfg_attr(docsrs, feature(doc_auto_cfg))]
mod bins;
pub mod drivers;
pub mod errors;
pub mod fetchers;
pub mod helpers;
pub mod ops;
use atomic_file_install as fs;
pub use binstalk_registry as registry;
pub use binstalk_types as manifests;
pub use detect_targets::{get_desired_targets, DesiredTargets, TARGET};
pub use home;

2
crates/binstalk/src/ops.rs
View file

@ -5,12 +5,12 @@ use std::{path::PathBuf, sync::Arc};
use semver::VersionReq;
use crate::{
drivers::Registry,
fetchers::{Data, Fetcher, TargetData},
helpers::{
self, gh_api_client::GhApiClient, jobserver_client::LazyJobserverClient, remote::Client,
},
manifests::cargo_toml_binstall::PkgOverride,
registry::Registry,
DesiredTargets,
};