diff --git a/crates/bin/src/entry.rs b/crates/bin/src/entry.rs index 48e50940..624d82b1 100644 --- a/crates/bin/src/entry.rs +++ b/crates/bin/src/entry.rs @@ -64,7 +64,7 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien strategies.pop().unwrap(); } - let resolver: Vec<_> = strategies + let resolvers: Vec<_> = strategies .into_iter() .map(|strategy| match strategy { Strategy::CrateMetaData => GhCrateMeta::new, @@ -192,7 +192,8 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien cli_overrides, desired_targets, quiet: args.log_level == LevelFilter::Off, - resolver, + resolvers, + cargo_install_fallback, }); let tasks: Vec<_> = if !args.dry_run && !args.no_confirm { @@ -263,13 +264,7 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien ) .await?; - if !cargo_install_fallback - && matches!(resolution, Resolution::InstallFromSource { .. }) - { - Err(BinstallError::NoFallbackToCargoInstall) - } else { - ops::install::install(resolution, opts, jobserver_client).await - } + ops::install::install(resolution, opts, jobserver_client).await }) }) .collect() diff --git a/crates/binstalk-downloader/src/download.rs b/crates/binstalk-downloader/src/download.rs index 18d41a5c..7cac3ff6 100644 --- a/crates/binstalk-downloader/src/download.rs +++ b/crates/binstalk-downloader/src/download.rs @@ -122,22 +122,30 @@ impl Download { path: impl AsRef, cancellation_future: CancellationFuture, ) -> Result<(), DownloadError> { - let stream = self.client.get_stream(self.url).await?; + async fn inner( + this: Download, + fmt: PkgFmt, + path: &Path, + cancellation_future: CancellationFuture, + ) -> Result<(), DownloadError> { + let stream = this.client.get_stream(this.url).await?; - let path = path.as_ref(); - debug!("Downloading and extracting to: '{}'", path.display()); + debug!("Downloading and extracting to: '{}'", path.display()); - match fmt.decompose() { - PkgFmtDecomposed::Tar(fmt) => { - extract_tar_based_stream(stream, path, fmt, cancellation_future).await? + match fmt.decompose() { + PkgFmtDecomposed::Tar(fmt) => { + extract_tar_based_stream(stream, path, fmt, cancellation_future).await? + } + PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?, + PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?, } - PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?, - PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?, + + debug!("Download OK, extracted to: '{}'", path.display()); + + Ok(()) } - debug!("Download OK, extracted to: '{}'", path.display()); - - Ok(()) + inner(self, fmt, path.as_ref(), cancellation_future).await } } diff --git a/crates/binstalk-manifests/src/binstall_crates_v1.rs b/crates/binstalk-manifests/src/binstall_crates_v1.rs index 3cac5e65..10393211 100644 --- a/crates/binstalk-manifests/src/binstall_crates_v1.rs +++ b/crates/binstalk-manifests/src/binstall_crates_v1.rs @@ -25,6 +25,9 @@ use thiserror::Error; use crate::{crate_info::CrateInfo, helpers::create_if_not_exist}; +/// Buffer size for loading and writing binstall_crates_v1 manifest. +const BUFFER_SIZE: usize = 4096 * 5; + #[derive(Debug, Diagnostic, Error)] #[non_exhaustive] pub enum Error { @@ -56,7 +59,7 @@ where } pub fn write_to(file: &mut FileLock, iter: &mut dyn Iterator) -> Result<(), Error> { - let writer = io::BufWriter::with_capacity(512, file); + let writer = io::BufWriter::with_capacity(BUFFER_SIZE, file); let mut ser = serde_json::Serializer::new(writer); @@ -149,7 +152,7 @@ pub struct Records { impl Records { fn load_impl(&mut self) -> Result<(), Error> { - let reader = io::BufReader::with_capacity(1024, &mut self.file); + let reader = io::BufReader::with_capacity(BUFFER_SIZE, &mut self.file); let stream_deser = serde_json::Deserializer::from_reader(reader).into_iter(); for res in stream_deser { diff --git a/crates/binstalk/src/bins.rs b/crates/binstalk/src/bins.rs index 21b58fcb..4f949405 100644 --- a/crates/binstalk/src/bins.rs +++ b/crates/binstalk/src/bins.rs @@ -4,7 +4,6 @@ use std::{ path::{Component, Path, PathBuf}, }; -use cargo_toml::Product; use compact_str::CompactString; use normalize_path::NormalizePath; use serde::Serialize; @@ -74,14 +73,12 @@ pub struct BinFile { } impl BinFile { - pub fn from_product( + pub fn new( data: &Data<'_>, - product: &Product, + base_name: &str, bin_dir: &str, no_symlinks: bool, ) -> Result { - let base_name = product.name.as_deref().unwrap(); - let binary_ext = if data.target.contains("windows") { ".exe" } else { @@ -99,7 +96,7 @@ impl BinFile { }; let source = if data.meta.pkg_fmt == Some(PkgFmt::Bin) { - data.bin_path.clone() + data.bin_path.to_path_buf() } else { // Generate install paths // Source path is the download dir + the generated binary path @@ -229,8 +226,8 @@ pub struct Data<'a> { pub version: &'a str, pub repo: Option<&'a str>, pub meta: PkgMeta, - pub bin_path: PathBuf, - pub install_path: PathBuf, + pub bin_path: &'a Path, + pub install_path: &'a Path, } #[derive(Clone, Debug, Serialize)] diff --git a/crates/binstalk/src/fetchers.rs b/crates/binstalk/src/fetchers.rs index 6e606340..a2d5e541 100644 --- a/crates/binstalk/src/fetchers.rs +++ b/crates/binstalk/src/fetchers.rs @@ -60,9 +60,9 @@ pub trait Fetcher: Send + Sync { /// Data required to fetch a package #[derive(Clone, Debug)] pub struct Data { - pub name: String, + pub name: CompactString, pub target: String, - pub version: String, + pub version: CompactString, pub repo: Option, pub meta: PkgMeta, } diff --git a/crates/binstalk/src/fetchers/gh_crate_meta.rs b/crates/binstalk/src/fetchers/gh_crate_meta.rs index 896582e4..8df5680b 100644 --- a/crates/binstalk/src/fetchers/gh_crate_meta.rs +++ b/crates/binstalk/src/fetchers/gh_crate_meta.rs @@ -263,6 +263,7 @@ mod test { use crate::manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}; use super::{super::Data, Context}; + use compact_str::ToCompactString; use url::Url; const DEFAULT_PKG_URL: &str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }"; @@ -275,9 +276,9 @@ mod test { fn defaults() { let meta = PkgMeta::default(); let data = Data { - name: "cargo-binstall".to_string(), + name: "cargo-binstall".to_compact_string(), target: "x86_64-unknown-linux-gnu".to_string(), - version: "1.2.3".to_string(), + version: "1.2.3".to_compact_string(), repo: Some("https://github.com/ryankurte/cargo-binstall".to_string()), meta, }; @@ -294,9 +295,9 @@ mod test { fn no_repo() { let meta = PkgMeta::default(); let data = Data { - name: "cargo-binstall".to_string(), + name: "cargo-binstall".to_compact_string(), target: "x86_64-unknown-linux-gnu".to_string(), - version: "1.2.3".to_string(), + version: "1.2.3".to_compact_string(), repo: None, meta, }; @@ -314,9 +315,9 @@ mod test { }; let data = Data { - name: "cargo-binstall".to_string(), + name: "cargo-binstall".to_compact_string(), target: "x86_64-unknown-linux-gnu".to_string(), - version: "1.2.3".to_string(), + version: "1.2.3".to_compact_string(), repo: None, meta, }; @@ -338,9 +339,9 @@ mod test { }; let data = Data { - name: "radio-sx128x".to_string(), + name: "radio-sx128x".to_compact_string(), target: "x86_64-unknown-linux-gnu".to_string(), - version: "0.14.1-alpha.5".to_string(), + version: "0.14.1-alpha.5".to_compact_string(), repo: Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()), meta, }; @@ -360,9 +361,9 @@ mod test { }; let data = Data { - name: "radio-sx128x".to_string(), + name: "radio-sx128x".to_compact_string(), target: "x86_64-unknown-linux-gnu".to_string(), - version: "0.14.1-alpha.5".to_string(), + version: "0.14.1-alpha.5".to_compact_string(), repo: Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()), meta, }; @@ -386,9 +387,9 @@ mod test { }; let data = Data { - name: "cargo-watch".to_string(), + name: "cargo-watch".to_compact_string(), target: "aarch64-apple-darwin".to_string(), - version: "9.0.0".to_string(), + version: "9.0.0".to_compact_string(), repo: Some("https://github.com/watchexec/cargo-watch".to_string()), meta, }; @@ -409,9 +410,9 @@ mod test { }; let data = Data { - name: "cargo-watch".to_string(), + name: "cargo-watch".to_compact_string(), target: "aarch64-pc-windows-msvc".to_string(), - version: "9.0.0".to_string(), + version: "9.0.0".to_compact_string(), repo: Some("https://github.com/watchexec/cargo-watch".to_string()), meta, }; diff --git a/crates/binstalk/src/fetchers/gh_crate_meta/hosting.rs b/crates/binstalk/src/fetchers/gh_crate_meta/hosting.rs index cf758a8f..7cb26b85 100644 --- a/crates/binstalk/src/fetchers/gh_crate_meta/hosting.rs +++ b/crates/binstalk/src/fetchers/gh_crate_meta/hosting.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use url::Url; use crate::errors::BinstallError; @@ -53,6 +54,7 @@ impl RepositoryHost { "{ repo }/releases/download/v{ version }", ], &[FULL_FILENAMES, NOVERSION_FILENAMES], + "", )), GitLab => Some(apply_filenames_to_paths( &[ @@ -60,32 +62,31 @@ impl RepositoryHost { "{ repo }/-/releases/v{ version }/downloads/binaries", ], &[FULL_FILENAMES, NOVERSION_FILENAMES], + "", )), BitBucket => Some(apply_filenames_to_paths( &["{ repo }/downloads"], &[FULL_FILENAMES], + "", + )), + SourceForge => Some(apply_filenames_to_paths( + &[ + "{ repo }/files/binaries/{ version }", + "{ repo }/files/binaries/v{ version }", + ], + &[FULL_FILENAMES, NOVERSION_FILENAMES], + "/download", )), - SourceForge => Some( - apply_filenames_to_paths( - &[ - "{ repo }/files/binaries/{ version }", - "{ repo }/files/binaries/v{ version }", - ], - &[FULL_FILENAMES, NOVERSION_FILENAMES], - ) - .into_iter() - .map(|url| format!("{url}/download")) - .collect(), - ), Unknown => None, } } } -fn apply_filenames_to_paths(paths: &[&str], filenames: &[&[&str]]) -> Vec { +fn apply_filenames_to_paths(paths: &[&str], filenames: &[&[&str]], suffix: &str) -> Vec { filenames .iter() .flat_map(|fs| fs.iter()) - .flat_map(|filename| paths.iter().map(move |path| format!("{path}/{filename}"))) + .cartesian_product(paths.iter()) + .map(|(filename, path)| format!("{path}/{filename}{suffix}")) .collect() } diff --git a/crates/binstalk/src/ops.rs b/crates/binstalk/src/ops.rs index 582d3911..805cd4d1 100644 --- a/crates/binstalk/src/ops.rs +++ b/crates/binstalk/src/ops.rs @@ -25,5 +25,6 @@ pub struct Options { pub cli_overrides: PkgOverride, pub desired_targets: DesiredTargets, pub quiet: bool, - pub resolver: Vec, + pub resolvers: Vec, + pub cargo_install_fallback: bool, } diff --git a/crates/binstalk/src/ops/resolve.rs b/crates/binstalk/src/ops/resolve.rs index 6401f55a..7999ae96 100644 --- a/crates/binstalk/src/ops/resolve.rs +++ b/crates/binstalk/src/ops/resolve.rs @@ -1,12 +1,12 @@ use std::{ borrow::Cow, - collections::BTreeSet, + collections::{BTreeMap, BTreeSet}, iter, mem, - path::{Path, PathBuf}, + path::Path, sync::Arc, }; -use cargo_toml::{Manifest, Package, Product}; +use cargo_toml::Manifest; use compact_str::{CompactString, ToCompactString}; use itertools::Itertools; use semver::{Version, VersionReq}; @@ -20,7 +20,7 @@ use crate::{ errors::BinstallError, fetchers::{Data, Fetcher}, helpers::{remote::Client, tasks::AutoAbortJoinHandle}, - manifests::cargo_toml_binstall::{Meta, PkgMeta}, + manifests::cargo_toml_binstall::{Meta, PkgMeta, PkgOverride}, }; mod crate_name; @@ -128,70 +128,30 @@ async fn resolve_inner( ) -> Result { info!("Resolving package: '{}'", crate_name); - let version_req: VersionReq = match (&crate_name.version_req, &opts.version_req) { - (Some(version), None) => version.clone(), + let version_req: VersionReq = match (crate_name.version_req, &opts.version_req) { + (Some(version), None) => version, (None, Some(version)) => version.clone(), (Some(_), Some(_)) => Err(BinstallError::SuperfluousVersionOption)?, (None, None) => VersionReq::STAR, }; - // Fetch crate via crates.io, git, or use a local manifest path - // TODO: work out which of these to do based on `opts.name` - // TODO: support git-based fetches (whole repo name rather than just crate name) - let manifest = match opts.manifest_path.clone() { - Some(manifest_path) => load_manifest_path(manifest_path)?, - None => { - fetch_crate_cratesio( - client.clone(), - &crates_io_api_client, - &crate_name.name, - &version_req, - ) - .await? - } + let version_req_str = version_req.to_compact_string(); + + let Some(package_info) = PackageInfo::resolve(opts, + crate_name.name, + curr_version, + version_req, + client.clone(), + crates_io_api_client).await? + else { + return Ok(Resolution::AlreadyUpToDate) }; - let package = manifest - .package - .ok_or_else(|| BinstallError::CargoTomlMissingPackage(crate_name.name.clone()))?; - - let new_version = - Version::parse(package.version()).map_err(|err| BinstallError::VersionParse { - v: package.version().to_compact_string(), - err, - })?; - - if let Some(curr_version) = curr_version { - if new_version == curr_version { - info!( - "{} v{curr_version} is already installed, use --force to override", - crate_name.name - ); - return Ok(Resolution::AlreadyUpToDate); - } - } - - let (mut meta, mut binaries) = ( - package - .metadata - .as_ref() - .and_then(|m| m.binstall.clone()) - .unwrap_or_default(), - manifest.bin, - ); - - binaries.retain(|product| product.name.is_some()); - - // Check binaries - if binaries.is_empty() { - return Err(BinstallError::UnspecifiedBinaries); - } - let desired_targets = opts.desired_targets.get().await; + let resolvers = &opts.resolvers; - let mut handles: Vec<(Arc, _)> = Vec::with_capacity(desired_targets.len() * 2); - - let overrides = mem::take(&mut meta.overrides); + let mut handles: Vec<(Arc, _)> = + Vec::with_capacity(desired_targets.len() * resolvers.len()); handles.extend( desired_targets @@ -199,20 +159,21 @@ async fn resolve_inner( .map(|target| { debug!("Building metadata for target: {target}"); - let target_meta = meta - .merge_overrides(iter::once(&opts.cli_overrides).chain(overrides.get(target))); + let target_meta = package_info.meta.merge_overrides( + iter::once(&opts.cli_overrides).chain(package_info.overrides.get(target)), + ); debug!("Found metadata: {target_meta:?}"); Arc::new(Data { - name: package.name.clone(), + name: package_info.name.clone(), target: target.clone(), - version: package.version().to_string(), - repo: package.repository().map(ToString::to_string), + version: package_info.version_str.clone(), + repo: package_info.repo.clone(), meta: target_meta, }) }) - .cartesian_product(&opts.resolver) + .cartesian_product(resolvers) .map(|(fetcher_data, f)| { let fetcher = f(&client, &fetcher_data); ( @@ -228,7 +189,7 @@ async fn resolve_inner( // Generate temporary binary path let bin_path = temp_dir.join(format!( "bin-{}-{}-{}", - crate_name.name, + package_info.name, fetcher.target(), fetcher.fetcher_name() )); @@ -236,9 +197,8 @@ async fn resolve_inner( match download_extract_and_verify( fetcher.as_ref(), &bin_path, - &package, + &package_info, &install_path, - &binaries, opts.no_symlinks, ) .await @@ -247,9 +207,9 @@ async fn resolve_inner( if !bin_files.is_empty() { return Ok(Resolution::Fetch { fetcher, - new_version, - name: crate_name.name, - version_req: version_req.to_compact_string(), + new_version: package_info.version, + name: package_info.name, + version_req: version_req_str, bin_files, }); } else { @@ -283,29 +243,31 @@ async fn resolve_inner( } } - Ok(Resolution::InstallFromSource { - name: crate_name.name, - version: package.version().to_compact_string(), - }) + if opts.cargo_install_fallback { + Ok(Resolution::InstallFromSource { + name: package_info.name, + version: package_info.version_str, + }) + } else { + Err(BinstallError::NoFallbackToCargoInstall) + } } /// * `fetcher` - `fetcher.find()` must return `Ok(true)`. -/// * `binaries` - must not be empty async fn download_extract_and_verify( fetcher: &dyn Fetcher, bin_path: &Path, - package: &Package, + package_info: &PackageInfo, install_path: &Path, - binaries: &[Product], no_symlinks: bool, ) -> Result, BinstallError> { - // Build final metadata - let meta = fetcher.target_meta(); - // Download and extract it. // If that fails, then ignore this fetcher. fetcher.fetch_and_extract(bin_path).await?; + // Build final metadata + let meta = fetcher.target_meta(); + #[cfg(incomplete)] { // Fetch and check package signature if available @@ -334,17 +296,17 @@ async fn download_extract_and_verify( block_in_place(|| { let bin_files = collect_bin_files( fetcher, - package, + package_info, meta, - binaries, - bin_path.to_path_buf(), - install_path.to_path_buf(), + bin_path, + install_path, no_symlinks, )?; - let name = &package.name; + let name = &package_info.name; - binaries + package_info + .binaries .iter() .zip(bin_files) .filter_map(|(bin, bin_file)| { @@ -360,8 +322,8 @@ async fn download_extract_and_verify( Some(Err(err)) } else { // Optional, print a warning and continue. - let bin_name = bin.name.as_deref().unwrap(); - let features = required_features.join(","); + let bin_name = bin.name.as_str(); + let features = required_features.iter().format(","); warn!( "When resolving {name} bin {bin_name} is not found. \ But since it requies features {features}, this bin is ignored." @@ -375,23 +337,21 @@ async fn download_extract_and_verify( }) } -/// * `binaries` - must not be empty fn collect_bin_files( fetcher: &dyn Fetcher, - package: &Package, + package_info: &PackageInfo, meta: PkgMeta, - binaries: &[Product], - bin_path: PathBuf, - install_path: PathBuf, + bin_path: &Path, + install_path: &Path, no_symlinks: bool, ) -> Result, BinstallError> { // List files to be installed // based on those found via Cargo.toml let bin_data = bins::Data { - name: &package.name, + name: &package_info.name, target: fetcher.target(), - version: package.version(), - repo: package.repository(), + version: &package_info.version_str, + repo: package_info.repo.as_deref(), meta, bin_path, install_path, @@ -405,9 +365,10 @@ fn collect_bin_files( .unwrap_or_else(|| bins::infer_bin_dir_template(&bin_data)); // Create bin_files - let bin_files = binaries + let bin_files = package_info + .binaries .iter() - .map(|p| bins::BinFile::from_product(&bin_data, p, &bin_dir, no_symlinks)) + .map(|bin| bins::BinFile::new(&bin_data, bin.name.as_str(), &bin_dir, no_symlinks)) .collect::, BinstallError>>()?; let mut source_set = BTreeSet::new(); @@ -423,6 +384,99 @@ fn collect_bin_files( Ok(bin_files) } +struct PackageInfo { + meta: PkgMeta, + binaries: Vec, + name: CompactString, + version_str: CompactString, + version: Version, + repo: Option, + overrides: BTreeMap, +} + +struct Bin { + name: String, + required_features: Vec, +} + +impl PackageInfo { + /// Return `None` if already up-to-date. + async fn resolve( + opts: &Options, + name: CompactString, + curr_version: Option, + version_req: VersionReq, + client: Client, + crates_io_api_client: crates_io_api::AsyncClient, + ) -> Result, BinstallError> { + // Fetch crate via crates.io, git, or use a local manifest path + let manifest = match opts.manifest_path.as_ref() { + Some(manifest_path) => load_manifest_path(manifest_path)?, + None => { + fetch_crate_cratesio(client, &crates_io_api_client, &name, &version_req).await? + } + }; + + let Some(mut package) = manifest.package else { + return Err(BinstallError::CargoTomlMissingPackage(name)); + }; + + let new_version_str = package.version().to_compact_string(); + let new_version = match Version::parse(&new_version_str) { + Ok(new_version) => new_version, + Err(err) => { + return Err(BinstallError::VersionParse { + v: new_version_str, + err, + }) + } + }; + + if let Some(curr_version) = curr_version { + if new_version == curr_version { + info!( + "{} v{curr_version} is already installed, use --force to override", + name + ); + return Ok(None); + } + } + + let (mut meta, binaries): (_, Vec) = ( + package + .metadata + .take() + .and_then(|mut m| m.binstall.take()) + .unwrap_or_default(), + manifest + .bin + .into_iter() + .filter_map(|p| { + p.name.map(|name| Bin { + name, + required_features: p.required_features, + }) + }) + .collect(), + ); + + // Check binaries + if binaries.is_empty() { + Err(BinstallError::UnspecifiedBinaries) + } else { + Ok(Some(Self { + overrides: mem::take(&mut meta.overrides), + meta, + binaries, + name, + version_str: new_version_str, + version: new_version, + repo: package.repository().map(ToString::to_string), + })) + } + } +} + /// Load binstall metadata from the crate `Cargo.toml` at the provided path pub fn load_manifest_path>( manifest_path: P,