actions/cargo-binstall
1
0
Fork 0
mirror of https://github.com/cargo-bins/cargo-binstall.git synced 2025-04-24 14:28:42 +00:00
Code Issues Projects Releases Packages Wiki Activity

Optimizations plus bug fix (#541)

Browse source 
* Optimize `Download::and_extract`: Avoid dup monomorphization
* Increase buffer size for binstall_crates_v1 to `4096 * 5`
* Optimize `opts::resolve`: Avoid unnecessary `clone`s
* Fix reserve in `opts::resolve`: Do not over-reserve
* Rename field `opts::Options::resolver` => `resolvers`
* Refactor: Extract new type `resolve::PackageInfo`
    - which makes `opts::resolve_inner` easier to understand
    - reduce number of parameters required for `download_extract_and_verify` and
      `collect_bin_files`
    - reducing size of future returned by `opts::resolve_inner` by dropping
      `cargo_toml::{Manifest, Package}` as early as possible since
      `Manifest` is 3000 Bytes large while `Package` is 600 Bytes large.
* Optimize `fetchers::Data`: Use `CompactString` for field name & version
   since they are usually small enough to fit in inlined version of
   `CompactString`.
* Optimize `gh_crate_meta`: Avoid unnecessary allocation
   in `RepositoryHost::get_default_pkg_url_template`.
* Refacator: Use `Itertools::cartesian_product` in `apply_filenames_to_paths`
* Optimize `ops::resolve`: Avoid unnecessary `clone` & reduce future size
   by calling `fetcher.target_meta()` to obtain final metadata after
   downloaded and extracted the binaries.
* Optimize `ops::resolve`: Avoid unnecessary allocation
   in `download_extract_and_verify`: Replace `Itertools::join` with
   `Itertools::format` to avoid allocating the string.
* Fix disabling cargo-install fallback
* Simplify `BinFile::from_product`: Takes `&str` instead of `&Product`
   since we only need `product.name`
* Rename `BinFile::from_product` => `BinFile::new`
* Refactor: Create newtype `ops::resolve::Bin`
   so that we don't need to `unwrap()` on `Product::name`
   and reduce memory usage.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2022-11-18 10:59:35 +11:00 • committed by GitHub
parent bb1f51a739
commit 325cb5cc19
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 220 additions and 160 deletions
Download patch file Download diff file Expand all files Collapse all files

13
crates/bin/src/entry.rs
View file

@ -64,7 +64,7 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien
strategies.pop().unwrap();
}
let resolver: Vec<_> = strategies
let resolvers: Vec<_> = strategies
.into_iter()
.map(|strategy| match strategy {
Strategy::CrateMetaData => GhCrateMeta::new,
@ -192,7 +192,8 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien
cli_overrides,
desired_targets,
quiet: args.log_level == LevelFilter::Off,
resolver,
resolvers,
cargo_install_fallback,
});
let tasks: Vec<_> = if !args.dry_run && !args.no_confirm {
@ -263,13 +264,7 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien
)
.await?;
if !cargo_install_fallback
&& matches!(resolution, Resolution::InstallFromSource { .. })
{
Err(BinstallError::NoFallbackToCargoInstall)
} else {
ops::install::install(resolution, opts, jobserver_client).await
}
ops::install::install(resolution, opts, jobserver_client).await
})
})
.collect()

30
crates/binstalk-downloader/src/download.rs
View file

@ -122,22 +122,30 @@ impl Download {
path: impl AsRef<Path>,
cancellation_future: CancellationFuture,
) -> Result<(), DownloadError> {
let stream = self.client.get_stream(self.url).await?;
async fn inner(
this: Download,
fmt: PkgFmt,
path: &Path,
cancellation_future: CancellationFuture,
) -> Result<(), DownloadError> {
let stream = this.client.get_stream(this.url).await?;
let path = path.as_ref();
debug!("Downloading and extracting to: '{}'", path.display());
debug!("Downloading and extracting to: '{}'", path.display());
match fmt.decompose() {
PkgFmtDecomposed::Tar(fmt) => {
extract_tar_based_stream(stream, path, fmt, cancellation_future).await?
match fmt.decompose() {
PkgFmtDecomposed::Tar(fmt) => {
extract_tar_based_stream(stream, path, fmt, cancellation_future).await?
}
PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?,
PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?,
}
PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?,
PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?,
debug!("Download OK, extracted to: '{}'", path.display());
Ok(())
}
debug!("Download OK, extracted to: '{}'", path.display());
Ok(())
inner(self, fmt, path.as_ref(), cancellation_future).await
}
}

7
crates/binstalk-manifests/src/binstall_crates_v1.rs
View file

@ -25,6 +25,9 @@ use thiserror::Error;
use crate::{crate_info::CrateInfo, helpers::create_if_not_exist};
/// Buffer size for loading and writing binstall_crates_v1 manifest.
const BUFFER_SIZE: usize = 4096 * 5;
#[derive(Debug, Diagnostic, Error)]
#[non_exhaustive]
pub enum Error {
@ -56,7 +59,7 @@ where
}
pub fn write_to(file: &mut FileLock, iter: &mut dyn Iterator<Item = Data>) -> Result<(), Error> {
let writer = io::BufWriter::with_capacity(512, file);
let writer = io::BufWriter::with_capacity(BUFFER_SIZE, file);
let mut ser = serde_json::Serializer::new(writer);
@ -149,7 +152,7 @@ pub struct Records {
impl Records {
fn load_impl(&mut self) -> Result<(), Error> {
let reader = io::BufReader::with_capacity(1024, &mut self.file);
let reader = io::BufReader::with_capacity(BUFFER_SIZE, &mut self.file);
let stream_deser = serde_json::Deserializer::from_reader(reader).into_iter();
for res in stream_deser {

13
crates/binstalk/src/bins.rs
View file

@ -4,7 +4,6 @@ use std::{
path::{Component, Path, PathBuf},
};
use cargo_toml::Product;
use compact_str::CompactString;
use normalize_path::NormalizePath;
use serde::Serialize;
@ -74,14 +73,12 @@ pub struct BinFile {
}
impl BinFile {
pub fn from_product(
pub fn new(
data: &Data<'_>,
product: &Product,
base_name: &str,
bin_dir: &str,
no_symlinks: bool,
) -> Result<Self, BinstallError> {
let base_name = product.name.as_deref().unwrap();
let binary_ext = if data.target.contains("windows") {
".exe"
} else {
@ -99,7 +96,7 @@ impl BinFile {
};
let source = if data.meta.pkg_fmt == Some(PkgFmt::Bin) {
data.bin_path.clone()
data.bin_path.to_path_buf()
} else {
// Generate install paths
// Source path is the download dir + the generated binary path
@ -229,8 +226,8 @@ pub struct Data<'a> {
pub version: &'a str,
pub repo: Option<&'a str>,
pub meta: PkgMeta,
pub bin_path: PathBuf,
pub install_path: PathBuf,
pub bin_path: &'a Path,
pub install_path: &'a Path,
}
#[derive(Clone, Debug, Serialize)]

4
crates/binstalk/src/fetchers.rs
View file

@ -60,9 +60,9 @@ pub trait Fetcher: Send + Sync {
/// Data required to fetch a package
#[derive(Clone, Debug)]
pub struct Data {
pub name: String,
pub name: CompactString,
pub target: String,
pub version: String,
pub version: CompactString,
pub repo: Option<String>,
pub meta: PkgMeta,
}

29
crates/binstalk/src/fetchers/gh_crate_meta.rs
View file

@ -263,6 +263,7 @@ mod test {
use crate::manifests::cargo_toml_binstall::{PkgFmt, PkgMeta};
use super::{super::Data, Context};
use compact_str::ToCompactString;
use url::Url;
const DEFAULT_PKG_URL: &str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }";
@ -275,9 +276,9 @@ mod test {
fn defaults() {
let meta = PkgMeta::default();
let data = Data {
name: "cargo-binstall".to_string(),
name: "cargo-binstall".to_compact_string(),
target: "x86_64-unknown-linux-gnu".to_string(),
version: "1.2.3".to_string(),
version: "1.2.3".to_compact_string(),
repo: Some("https://github.com/ryankurte/cargo-binstall".to_string()),
meta,
};
@ -294,9 +295,9 @@ mod test {
fn no_repo() {
let meta = PkgMeta::default();
let data = Data {
name: "cargo-binstall".to_string(),
name: "cargo-binstall".to_compact_string(),
target: "x86_64-unknown-linux-gnu".to_string(),
version: "1.2.3".to_string(),
version: "1.2.3".to_compact_string(),
repo: None,
meta,
};
@ -314,9 +315,9 @@ mod test {
};
let data = Data {
name: "cargo-binstall".to_string(),
name: "cargo-binstall".to_compact_string(),
target: "x86_64-unknown-linux-gnu".to_string(),
version: "1.2.3".to_string(),
version: "1.2.3".to_compact_string(),
repo: None,
meta,
};
@ -338,9 +339,9 @@ mod test {
};
let data = Data {
name: "radio-sx128x".to_string(),
name: "radio-sx128x".to_compact_string(),
target: "x86_64-unknown-linux-gnu".to_string(),
version: "0.14.1-alpha.5".to_string(),
version: "0.14.1-alpha.5".to_compact_string(),
repo: Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()),
meta,
};
@ -360,9 +361,9 @@ mod test {
};
let data = Data {
name: "radio-sx128x".to_string(),
name: "radio-sx128x".to_compact_string(),
target: "x86_64-unknown-linux-gnu".to_string(),
version: "0.14.1-alpha.5".to_string(),
version: "0.14.1-alpha.5".to_compact_string(),
repo: Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()),
meta,
};
@ -386,9 +387,9 @@ mod test {
};
let data = Data {
name: "cargo-watch".to_string(),
name: "cargo-watch".to_compact_string(),
target: "aarch64-apple-darwin".to_string(),
version: "9.0.0".to_string(),
version: "9.0.0".to_compact_string(),
repo: Some("https://github.com/watchexec/cargo-watch".to_string()),
meta,
};
@ -409,9 +410,9 @@ mod test {
};
let data = Data {
name: "cargo-watch".to_string(),
name: "cargo-watch".to_compact_string(),
target: "aarch64-pc-windows-msvc".to_string(),
version: "9.0.0".to_string(),
version: "9.0.0".to_compact_string(),
repo: Some("https://github.com/watchexec/cargo-watch".to_string()),
meta,
};

29
crates/binstalk/src/fetchers/gh_crate_meta/hosting.rs
View file

@ -1,3 +1,4 @@
use itertools::Itertools;
use url::Url;
use crate::errors::BinstallError;
@ -53,6 +54,7 @@ impl RepositoryHost {
"{ repo }/releases/download/v{ version }",
],
&[FULL_FILENAMES, NOVERSION_FILENAMES],
"",
)),
GitLab => Some(apply_filenames_to_paths(
&[
@ -60,32 +62,31 @@ impl RepositoryHost {
"{ repo }/-/releases/v{ version }/downloads/binaries",
],
&[FULL_FILENAMES, NOVERSION_FILENAMES],
"",
)),
BitBucket => Some(apply_filenames_to_paths(
&["{ repo }/downloads"],
&[FULL_FILENAMES],
"",
)),
SourceForge => Some(apply_filenames_to_paths(
&[
"{ repo }/files/binaries/{ version }",
"{ repo }/files/binaries/v{ version }",
],
&[FULL_FILENAMES, NOVERSION_FILENAMES],
"/download",
)),
SourceForge => Some(
apply_filenames_to_paths(
&[
"{ repo }/files/binaries/{ version }",
"{ repo }/files/binaries/v{ version }",
],
&[FULL_FILENAMES, NOVERSION_FILENAMES],
)
.into_iter()
.map(|url| format!("{url}/download"))
.collect(),
),
Unknown => None,
}
}
}
fn apply_filenames_to_paths(paths: &[&str], filenames: &[&[&str]]) -> Vec<String> {
fn apply_filenames_to_paths(paths: &[&str], filenames: &[&[&str]], suffix: &str) -> Vec<String> {
filenames
.iter()
.flat_map(|fs| fs.iter())
.flat_map(|filename| paths.iter().map(move |path| format!("{path}/{filename}")))
.cartesian_product(paths.iter())
.map(|(filename, path)| format!("{path}/{filename}{suffix}"))
.collect()
}

3
crates/binstalk/src/ops.rs
View file

@ -25,5 +25,6 @@ pub struct Options {
pub cli_overrides: PkgOverride,
pub desired_targets: DesiredTargets,
pub quiet: bool,
pub resolver: Vec<Resolver>,
pub resolvers: Vec<Resolver>,
pub cargo_install_fallback: bool,
}

252
crates/binstalk/src/ops/resolve.rs
View file

@ -1,12 +1,12 @@
use std::{
borrow::Cow,
collections::BTreeSet,
collections::{BTreeMap, BTreeSet},
iter, mem,
path::{Path, PathBuf},
path::Path,
sync::Arc,
};
use cargo_toml::{Manifest, Package, Product};
use cargo_toml::Manifest;
use compact_str::{CompactString, ToCompactString};
use itertools::Itertools;
use semver::{Version, VersionReq};
@ -20,7 +20,7 @@ use crate::{
errors::BinstallError,
fetchers::{Data, Fetcher},
helpers::{remote::Client, tasks::AutoAbortJoinHandle},
manifests::cargo_toml_binstall::{Meta, PkgMeta},
manifests::cargo_toml_binstall::{Meta, PkgMeta, PkgOverride},
};
mod crate_name;
@ -128,70 +128,30 @@ async fn resolve_inner(
) -> Result<Resolution, BinstallError> {
info!("Resolving package: '{}'", crate_name);
let version_req: VersionReq = match (&crate_name.version_req, &opts.version_req) {
(Some(version), None) => version.clone(),
let version_req: VersionReq = match (crate_name.version_req, &opts.version_req) {
(Some(version), None) => version,
(None, Some(version)) => version.clone(),
(Some(_), Some(_)) => Err(BinstallError::SuperfluousVersionOption)?,
(None, None) => VersionReq::STAR,
};
// Fetch crate via crates.io, git, or use a local manifest path
// TODO: work out which of these to do based on `opts.name`
// TODO: support git-based fetches (whole repo name rather than just crate name)
let manifest = match opts.manifest_path.clone() {
Some(manifest_path) => load_manifest_path(manifest_path)?,
None => {
fetch_crate_cratesio(
client.clone(),
&crates_io_api_client,
&crate_name.name,
&version_req,
)
.await?
}
let version_req_str = version_req.to_compact_string();
let Some(package_info) = PackageInfo::resolve(opts,
crate_name.name,
curr_version,
version_req,
client.clone(),
crates_io_api_client).await?
else {
return Ok(Resolution::AlreadyUpToDate)
};
let package = manifest
.package
.ok_or_else(|| BinstallError::CargoTomlMissingPackage(crate_name.name.clone()))?;
let new_version =
Version::parse(package.version()).map_err(|err| BinstallError::VersionParse {
v: package.version().to_compact_string(),
err,
})?;
if let Some(curr_version) = curr_version {
if new_version == curr_version {
info!(
"{} v{curr_version} is already installed, use --force to override",
crate_name.name
);
return Ok(Resolution::AlreadyUpToDate);
}
}
let (mut meta, mut binaries) = (
package
.metadata
.as_ref()
.and_then(|m| m.binstall.clone())
.unwrap_or_default(),
manifest.bin,
);
binaries.retain(|product| product.name.is_some());
// Check binaries
if binaries.is_empty() {
return Err(BinstallError::UnspecifiedBinaries);
}
let desired_targets = opts.desired_targets.get().await;
let resolvers = &opts.resolvers;
let mut handles: Vec<(Arc<dyn Fetcher>, _)> = Vec::with_capacity(desired_targets.len() * 2);
let overrides = mem::take(&mut meta.overrides);
let mut handles: Vec<(Arc<dyn Fetcher>, _)> =
Vec::with_capacity(desired_targets.len() * resolvers.len());
handles.extend(
desired_targets
@ -199,20 +159,21 @@ async fn resolve_inner(
.map(|target| {
debug!("Building metadata for target: {target}");
let target_meta = meta
.merge_overrides(iter::once(&opts.cli_overrides).chain(overrides.get(target)));
let target_meta = package_info.meta.merge_overrides(
iter::once(&opts.cli_overrides).chain(package_info.overrides.get(target)),
);
debug!("Found metadata: {target_meta:?}");
Arc::new(Data {
name: package.name.clone(),
name: package_info.name.clone(),
target: target.clone(),
version: package.version().to_string(),
repo: package.repository().map(ToString::to_string),
version: package_info.version_str.clone(),
repo: package_info.repo.clone(),
meta: target_meta,
})
})
.cartesian_product(&opts.resolver)
.cartesian_product(resolvers)
.map(|(fetcher_data, f)| {
let fetcher = f(&client, &fetcher_data);
(
@ -228,7 +189,7 @@ async fn resolve_inner(
// Generate temporary binary path
let bin_path = temp_dir.join(format!(
"bin-{}-{}-{}",
crate_name.name,
package_info.name,
fetcher.target(),
fetcher.fetcher_name()
));
@ -236,9 +197,8 @@ async fn resolve_inner(
match download_extract_and_verify(
fetcher.as_ref(),
&bin_path,
&package,
&package_info,
&install_path,
&binaries,
opts.no_symlinks,
)
.await
@ -247,9 +207,9 @@ async fn resolve_inner(
if !bin_files.is_empty() {
return Ok(Resolution::Fetch {
fetcher,
new_version,
name: crate_name.name,
version_req: version_req.to_compact_string(),
new_version: package_info.version,
name: package_info.name,
version_req: version_req_str,
bin_files,
});
} else {
@ -283,29 +243,31 @@ async fn resolve_inner(
}
}
Ok(Resolution::InstallFromSource {
name: crate_name.name,
version: package.version().to_compact_string(),
})
if opts.cargo_install_fallback {
Ok(Resolution::InstallFromSource {
name: package_info.name,
version: package_info.version_str,
})
} else {
Err(BinstallError::NoFallbackToCargoInstall)
}
}
/// * `fetcher` - `fetcher.find()` must return `Ok(true)`.
/// * `binaries` - must not be empty
async fn download_extract_and_verify(
fetcher: &dyn Fetcher,
bin_path: &Path,
package: &Package<Meta>,
package_info: &PackageInfo,
install_path: &Path,
binaries: &[Product],
no_symlinks: bool,
) -> Result<Vec<bins::BinFile>, BinstallError> {
// Build final metadata
let meta = fetcher.target_meta();
// Download and extract it.
// If that fails, then ignore this fetcher.
fetcher.fetch_and_extract(bin_path).await?;
// Build final metadata
let meta = fetcher.target_meta();
#[cfg(incomplete)]
{
// Fetch and check package signature if available
@ -334,17 +296,17 @@ async fn download_extract_and_verify(
block_in_place(|| {
let bin_files = collect_bin_files(
fetcher,
package,
package_info,
meta,
binaries,
bin_path.to_path_buf(),
install_path.to_path_buf(),
bin_path,
install_path,
no_symlinks,
)?;
let name = &package.name;
let name = &package_info.name;
binaries
package_info
.binaries
.iter()
.zip(bin_files)
.filter_map(|(bin, bin_file)| {
@ -360,8 +322,8 @@ async fn download_extract_and_verify(
Some(Err(err))
} else {
// Optional, print a warning and continue.
let bin_name = bin.name.as_deref().unwrap();
let features = required_features.join(",");
let bin_name = bin.name.as_str();
let features = required_features.iter().format(",");
warn!(
"When resolving {name} bin {bin_name} is not found. \
But since it requies features {features}, this bin is ignored."
@ -375,23 +337,21 @@ async fn download_extract_and_verify(
})
}
/// * `binaries` - must not be empty
fn collect_bin_files(
fetcher: &dyn Fetcher,
package: &Package<Meta>,
package_info: &PackageInfo,
meta: PkgMeta,
binaries: &[Product],
bin_path: PathBuf,
install_path: PathBuf,
bin_path: &Path,
install_path: &Path,
no_symlinks: bool,
) -> Result<Vec<bins::BinFile>, BinstallError> {
// List files to be installed
// based on those found via Cargo.toml
let bin_data = bins::Data {
name: &package.name,
name: &package_info.name,
target: fetcher.target(),
version: package.version(),
repo: package.repository(),
version: &package_info.version_str,
repo: package_info.repo.as_deref(),
meta,
bin_path,
install_path,
@ -405,9 +365,10 @@ fn collect_bin_files(
.unwrap_or_else(|| bins::infer_bin_dir_template(&bin_data));
// Create bin_files
let bin_files = binaries
let bin_files = package_info
.binaries
.iter()
.map(|p| bins::BinFile::from_product(&bin_data, p, &bin_dir, no_symlinks))
.map(|bin| bins::BinFile::new(&bin_data, bin.name.as_str(), &bin_dir, no_symlinks))
.collect::<Result<Vec<_>, BinstallError>>()?;
let mut source_set = BTreeSet::new();
@ -423,6 +384,99 @@ fn collect_bin_files(
Ok(bin_files)
}
struct PackageInfo {
meta: PkgMeta,
binaries: Vec<Bin>,
name: CompactString,
version_str: CompactString,
version: Version,
repo: Option<String>,
overrides: BTreeMap<String, PkgOverride>,
}
struct Bin {
name: String,
required_features: Vec<String>,
}
impl PackageInfo {
/// Return `None` if already up-to-date.
async fn resolve(
opts: &Options,
name: CompactString,
curr_version: Option<Version>,
version_req: VersionReq,
client: Client,
crates_io_api_client: crates_io_api::AsyncClient,
) -> Result<Option<Self>, BinstallError> {
// Fetch crate via crates.io, git, or use a local manifest path
let manifest = match opts.manifest_path.as_ref() {
Some(manifest_path) => load_manifest_path(manifest_path)?,
None => {
fetch_crate_cratesio(client, &crates_io_api_client, &name, &version_req).await?
}
};
let Some(mut package) = manifest.package else {
return Err(BinstallError::CargoTomlMissingPackage(name));
};
let new_version_str = package.version().to_compact_string();
let new_version = match Version::parse(&new_version_str) {
Ok(new_version) => new_version,
Err(err) => {
return Err(BinstallError::VersionParse {
v: new_version_str,
err,
})
}
};
if let Some(curr_version) = curr_version {
if new_version == curr_version {
info!(
"{} v{curr_version} is already installed, use --force to override",
name
);
return Ok(None);
}
}
let (mut meta, binaries): (_, Vec<Bin>) = (
package
.metadata
.take()
.and_then(|mut m| m.binstall.take())
.unwrap_or_default(),
manifest
.bin
.into_iter()
.filter_map(|p| {
p.name.map(|name| Bin {
name,
required_features: p.required_features,
})
})
.collect(),
);
// Check binaries
if binaries.is_empty() {
Err(BinstallError::UnspecifiedBinaries)
} else {
Ok(Some(Self {
overrides: mem::take(&mut meta.overrides),
meta,
binaries,
name,
version_str: new_version_str,
version: new_version,
repo: package.repository().map(ToString::to_string),
}))
}
}
}
/// Load binstall metadata from the crate `Cargo.toml` at the provided path
pub fn load_manifest_path<P: AsRef<Path>>(
manifest_path: P,