From d8419ea5a2059a6a2b6b1c1c4bd6ebf0e25016ad Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 21 Jun 2023 08:05:13 +1000 Subject: [PATCH] feat: Add more variables for template (#1160) Fixed #775 - Add dep target-lexicon v0.12.7 - Add `target-{family, arch, libc, vendor}` to `package.metadata.binstall`. For `{universal, universal2}-apple-darwin`, the `target-arch` is set to `universal`. Signed-off-by: Jiahao XU --- Cargo.lock | 7 + crates/binstalk/Cargo.toml | 1 + crates/binstalk/src/bins.rs | 10 +- crates/binstalk/src/errors.rs | 16 ++ crates/binstalk/src/fetchers.rs | 5 +- crates/binstalk/src/fetchers/gh_crate_meta.rs | 201 +++++++++--------- crates/binstalk/src/fetchers/quickinstall.rs | 9 +- crates/binstalk/src/helpers.rs | 5 + crates/binstalk/src/helpers/target_triple.rs | 54 +++++ crates/binstalk/src/ops/resolve.rs | 17 +- 10 files changed, 210 insertions(+), 115 deletions(-) create mode 100644 crates/binstalk/src/helpers/target_triple.rs diff --git a/Cargo.lock b/Cargo.lock index b9a89566..7f82f6ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,6 +238,7 @@ dependencies = [ "semver", "serde", "strum", + "target-lexicon", "tempfile", "thiserror", "tokio", @@ -2237,6 +2238,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "target-lexicon" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" + [[package]] name = "tempfile" version = "3.6.0" diff --git a/crates/binstalk/Cargo.toml b/crates/binstalk/Cargo.toml index 4b926224..30e1d0a4 100644 --- a/crates/binstalk/Cargo.toml +++ b/crates/binstalk/Cargo.toml @@ -30,6 +30,7 @@ once_cell = "1.18.0" semver = { version = "1.0.17", features = ["serde"] } serde = { version = "1.0.163", features = ["derive"] } strum = "0.25.0" +target-lexicon = { version = "0.12.7", features = ["std"] } tempfile = "3.5.0" thiserror = "1.0.40" # parking_lot for `tokio::sync::OnceCell::const_new` diff --git a/crates/binstalk/src/bins.rs b/crates/binstalk/src/bins.rs index 939afb96..980069f2 100644 --- a/crates/binstalk/src/bins.rs +++ b/crates/binstalk/src/bins.rs @@ -15,7 +15,7 @@ use crate::{ atomic_install, atomic_install_noclobber, atomic_symlink_file, atomic_symlink_file_noclobber, }, - helpers::download::ExtractedFiles, + helpers::{download::ExtractedFiles, target_triple::TargetTriple}, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, }; @@ -98,6 +98,8 @@ impl BinFile { version: data.version, bin: base_name, binary_ext, + + triple: data.triple, }; let (source, archive_source_path) = if data.meta.pkg_fmt == Some(PkgFmt::Bin) { @@ -271,6 +273,7 @@ pub struct Data<'a> { pub meta: PkgMeta, pub bin_path: &'a Path, pub install_path: &'a Path, + pub triple: &'a TargetTriple, } #[derive(Clone, Debug)] @@ -283,6 +286,8 @@ struct Context<'c> { /// Filename extension on the binary, i.e. .exe on Windows, nothing otherwise pub binary_ext: &'c str, + + pub triple: &'c TargetTriple, } impl leon::Values for Context<'_> { @@ -296,7 +301,8 @@ impl leon::Values for Context<'_> { "binary-ext" => Some(Cow::Borrowed(self.binary_ext)), // Soft-deprecated alias for binary-ext "format" => Some(Cow::Borrowed(self.binary_ext)), - _ => None, + + key => self.triple.get_value(key), } } } diff --git a/crates/binstalk/src/errors.rs b/crates/binstalk/src/errors.rs index d01d6a22..bc637947 100644 --- a/crates/binstalk/src/errors.rs +++ b/crates/binstalk/src/errors.rs @@ -10,6 +10,7 @@ use binstalk_downloader::{ use cargo_toml::Error as CargoTomlError; use compact_str::CompactString; use miette::{Diagnostic, Report}; +use target_lexicon::ParseError as TargetTripleParseError; use thiserror::Error; use tokio::task; use tracing::{error, warn}; @@ -314,6 +315,14 @@ pub enum BinstallError { #[diagnostic(severity(error), code(binstall::gh_api_failure))] GhApiErr(#[source] Box), + /// Failed to parse target triple + /// + /// - Code: `binstall::target_triple_parse_error` + /// - Exit: 97 + #[error("Failed to parse target triple: {0}")] + #[diagnostic(severity(error), code(binstall::target_triple_parse_error))] + TargetTripleParseError(#[source] Box), + /// A wrapped error providing the context of which crate the error is about. #[error(transparent)] #[diagnostic(transparent)] @@ -348,6 +357,7 @@ impl BinstallError { NoFallbackToCargoInstall => 94, InvalidPkgFmt(..) => 95, GhApiErr(..) => 96, + TargetTripleParseError(..) => 97, CrateContext(context) => context.err.exit_number(), }; @@ -441,3 +451,9 @@ impl From for BinstallError { BinstallError::GhApiErr(Box::new(e)) } } + +impl From for BinstallError { + fn from(e: target_lexicon::ParseError) -> Self { + BinstallError::TargetTripleParseError(Box::new(e)) + } +} diff --git a/crates/binstalk/src/fetchers.rs b/crates/binstalk/src/fetchers.rs index 03822e7b..194a96c5 100644 --- a/crates/binstalk/src/fetchers.rs +++ b/crates/binstalk/src/fetchers.rs @@ -11,7 +11,7 @@ use crate::{ errors::BinstallError, helpers::{ download::ExtractedFiles, gh_api_client::GhApiClient, remote::Client, - tasks::AutoAbortJoinHandle, + target_triple::TargetTriple, tasks::AutoAbortJoinHandle, }, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, }; @@ -72,6 +72,8 @@ pub trait Fetcher: Send + Sync { /// Return the target for this fetcher fn target(&self) -> &str; + + fn target_data(&self) -> &Arc; } #[derive(Clone, Debug)] @@ -186,6 +188,7 @@ impl RepoInfo { #[derive(Clone, Debug)] pub struct TargetData { pub target: String, + pub triple: TargetTriple, pub meta: PkgMeta, } diff --git a/crates/binstalk/src/fetchers/gh_crate_meta.rs b/crates/binstalk/src/fetchers/gh_crate_meta.rs index aa5a3d9e..33d732c2 100644 --- a/crates/binstalk/src/fetchers/gh_crate_meta.rs +++ b/crates/binstalk/src/fetchers/gh_crate_meta.rs @@ -15,6 +15,7 @@ use crate::{ futures_resolver::FuturesResolver, gh_api_client::GhApiClient, remote::{does_url_exist, Client}, + target_triple::TargetTriple, tasks::AutoAbortJoinHandle, }, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, @@ -45,6 +46,7 @@ impl GhCrateMeta { let ctx = Context::from_data_with_repo( &self.data, &self.target_data.target, + &self.target_data.triple, ext, repo, subcrate, @@ -275,6 +277,10 @@ impl super::Fetcher for GhCrateMeta { fn target(&self) -> &str { &self.target_data.target } + + fn target_data(&self) -> &Arc { + &self.target_data + } } /// Template for constructing download paths @@ -295,6 +301,8 @@ struct Context<'c> { /// Workspace of the crate inside the repository. pub subcrate: Option<&'c str>, + + pub triple: &'c TargetTriple, } impl leon::Values for Context<'_> { @@ -316,15 +324,16 @@ impl leon::Values for Context<'_> { "subcrate" => self.subcrate.map(Cow::Borrowed), - _ => None, + key => self.triple.get_value(key), } } } impl<'c> Context<'c> { - pub(self) fn from_data_with_repo( + fn from_data_with_repo( data: &'c Data, target: &'c str, + triple: &'c TargetTriple, archive_suffix: Option<&'c str>, repo: Option<&'c str>, subcrate: Option<&'c str>, @@ -344,6 +353,7 @@ impl<'c> Context<'c> { name: &data.name, repo, target, + version: &data.version, archive_format, archive_suffix, @@ -353,18 +363,9 @@ impl<'c> Context<'c> { "" }, subcrate, - } - } - #[cfg(test)] - pub(self) fn from_data(data: &'c Data, target: &'c str, archive_format: &'c str) -> Self { - Self::from_data_with_repo( - data, - target, - Some(archive_format), - data.repo.as_deref(), - None, - ) + triple, + } } /// * `tt` - must have added a template named "pkg_url". @@ -388,132 +389,124 @@ impl<'c> Context<'c> { #[cfg(test)] mod test { - use crate::manifests::cargo_toml_binstall::PkgMeta; - use super::{super::Data, Context}; + use std::str::FromStr; + + use super::{super::Data, Context, TargetTriple}; use compact_str::ToCompactString; use url::Url; const DEFAULT_PKG_URL: &str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }"; - fn url(s: &str) -> Url { - Url::parse(s).unwrap() + fn assert_context_rendering( + data: &Data, + target: &str, + archive_format: &str, + template: &str, + expected_url: &str, + ) { + let triple = &TargetTriple::from_str(target).unwrap(); + + let ctx = Context::from_data_with_repo( + data, + target, + triple, + Some(archive_format), + data.repo.as_deref(), + None, + ); + + let expected_url = Url::parse(expected_url).unwrap(); + assert_eq!(ctx.render_url(template).unwrap(), expected_url); } #[test] fn defaults() { - let data = Data::new( - "cargo-binstall".to_compact_string(), - "1.2.3".to_compact_string(), - Some("https://github.com/ryankurte/cargo-binstall".to_string()), + assert_context_rendering( + &Data::new( + "cargo-binstall".to_compact_string(), + "1.2.3".to_compact_string(), + Some("https://github.com/ryankurte/cargo-binstall".to_string()), + ), + "x86_64-unknown-linux-gnu", + ".tgz", + DEFAULT_PKG_URL, + "https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz" ); - - let ctx = Context::from_data(&data, "x86_64-unknown-linux-gnu", ".tgz"); - assert_eq!( - ctx.render_url(DEFAULT_PKG_URL).unwrap(), - url("https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz") - ); - } - - #[test] - #[should_panic] - fn no_repo() { - let meta = PkgMeta::default(); - let data = Data::new( - "cargo-binstall".to_compact_string(), - "1.2.3".to_compact_string(), - None, - ); - - let ctx = Context::from_data(&data, "x86_64-unknown-linux-gnu", ".tgz"); - ctx.render_url(meta.pkg_url.as_deref().unwrap()).unwrap(); } #[test] fn no_repo_but_full_url() { - let pkg_url = &format!("https://example.com{}", &DEFAULT_PKG_URL[8..]); - - let data = Data::new( - "cargo-binstall".to_compact_string(), - "1.2.3".to_compact_string(), - None, - ); - - let ctx = Context::from_data(&data, "x86_64-unknown-linux-gnu", ".tgz"); - assert_eq!( - ctx.render_url(pkg_url).unwrap(), - url("https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz") + assert_context_rendering( + &Data::new( + "cargo-binstall".to_compact_string(), + "1.2.3".to_compact_string(), + None, + ), + "x86_64-unknown-linux-gnu", + ".tgz", + &format!("https://example.com{}", &DEFAULT_PKG_URL[8..]), + "https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz" ); } #[test] fn different_url() { - let pkg_url = - "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ archive-format }"; - - let data = Data::new( - "radio-sx128x".to_compact_string(), - "0.14.1-alpha.5".to_compact_string(), - Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()), - ); - - let ctx = Context::from_data(&data, "x86_64-unknown-linux-gnu", ".tgz"); - assert_eq!( - ctx.render_url(pkg_url).unwrap(), - url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz") + assert_context_rendering( + &Data::new( + "radio-sx128x".to_compact_string(), + "0.14.1-alpha.5".to_compact_string(), + Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()), + ), + "x86_64-unknown-linux-gnu", + ".tgz", + "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ archive-format }", + "https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz" ); } #[test] fn deprecated_format() { - let pkg_url = "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }"; - - let data = Data::new( - "radio-sx128x".to_compact_string(), - "0.14.1-alpha.5".to_compact_string(), - Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()), - ); - - let ctx = Context::from_data(&data, "x86_64-unknown-linux-gnu", ".tgz"); - assert_eq!( - ctx.render_url(pkg_url).unwrap(), - url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz") + assert_context_rendering( + &Data::new( + "radio-sx128x".to_compact_string(), + "0.14.1-alpha.5".to_compact_string(), + Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()), + ), + "x86_64-unknown-linux-gnu", + ".tgz", + "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }", + "https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz" ); } #[test] fn different_ext() { - let pkg_url = - "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.tar.xz"; - - let data = Data::new( - "cargo-watch".to_compact_string(), - "9.0.0".to_compact_string(), - Some("https://github.com/watchexec/cargo-watch".to_string()), - ); - - let ctx = Context::from_data(&data, "aarch64-apple-darwin", ".txz"); - assert_eq!( - ctx.render_url(pkg_url).unwrap(), - url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz") + assert_context_rendering( + &Data::new( + "cargo-watch".to_compact_string(), + "9.0.0".to_compact_string(), + Some("https://github.com/watchexec/cargo-watch".to_string()), + ), + "aarch64-apple-darwin", + ".txz", + "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.tar.xz", + "https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz" ); } #[test] fn no_archive() { - let pkg_url = "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }" - ; - - let data = Data::new( - "cargo-watch".to_compact_string(), - "9.0.0".to_compact_string(), - Some("https://github.com/watchexec/cargo-watch".to_string()), - ); - - let ctx = Context::from_data(&data, "aarch64-pc-windows-msvc", ".bin"); - assert_eq!( - ctx.render_url(pkg_url).unwrap(), - url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe") + assert_context_rendering( + &Data::new( + "cargo-watch".to_compact_string(), + "9.0.0".to_compact_string(), + Some("https://github.com/watchexec/cargo-watch".to_string()), + ), + "aarch64-pc-windows-msvc", + ".bin", + "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }", + "https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe" ); } } diff --git a/crates/binstalk/src/fetchers/quickinstall.rs b/crates/binstalk/src/fetchers/quickinstall.rs index cb071c6b..a6850f5f 100644 --- a/crates/binstalk/src/fetchers/quickinstall.rs +++ b/crates/binstalk/src/fetchers/quickinstall.rs @@ -9,6 +9,7 @@ use crate::{ helpers::{ download::{Download, ExtractedFiles}, gh_api_client::GhApiClient, + is_universal_macos, remote::{does_url_exist, Client, Method}, tasks::AutoAbortJoinHandle, }, @@ -20,10 +21,6 @@ use super::{Data, TargetData}; const BASE_URL: &str = "https://github.com/cargo-bins/cargo-quickinstall/releases/download"; const STATS_URL: &str = "https://warehouse-clerk-tmp.vercel.app/api/crate"; -fn is_universal_macos(target: &str) -> bool { - ["universal-apple-darwin", "universal2-apple-darwin"].contains(&target) -} - pub struct QuickInstall { client: Client, gh_api_client: GhApiClient, @@ -136,6 +133,10 @@ by rust officially."#, fn target(&self) -> &str { &self.target_data.target } + + fn target_data(&self) -> &Arc { + &self.target_data + } } impl QuickInstall { diff --git a/crates/binstalk/src/helpers.rs b/crates/binstalk/src/helpers.rs index bfc42460..23ed00ef 100644 --- a/crates/binstalk/src/helpers.rs +++ b/crates/binstalk/src/helpers.rs @@ -2,6 +2,11 @@ pub mod futures_resolver; pub mod jobserver_client; pub mod remote; pub mod signal; +pub mod target_triple; pub mod tasks; pub use binstalk_downloader::{download, gh_api_client}; + +pub fn is_universal_macos(target: &str) -> bool { + ["universal-apple-darwin", "universal2-apple-darwin"].contains(&target) +} diff --git a/crates/binstalk/src/helpers/target_triple.rs b/crates/binstalk/src/helpers/target_triple.rs new file mode 100644 index 00000000..c4cb10d6 --- /dev/null +++ b/crates/binstalk/src/helpers/target_triple.rs @@ -0,0 +1,54 @@ +use std::{borrow::Cow, str::FromStr}; + +use compact_str::{CompactString, ToCompactString}; +use target_lexicon::Triple; + +use crate::{errors::BinstallError, helpers::is_universal_macos}; + +#[derive(Clone, Debug)] +pub struct TargetTriple { + // TODO: Once https://github.com/bytecodealliance/target-lexicon/pull/90 + // lands, consider replacing use of CompactString with `Cow<'_, str>`. + pub target_family: CompactString, + pub target_arch: CompactString, + pub target_libc: CompactString, + pub target_vendor: CompactString, +} + +impl FromStr for TargetTriple { + type Err = BinstallError; + + fn from_str(mut s: &str) -> Result { + let is_universal_macos = is_universal_macos(s); + + if is_universal_macos { + s = "x86_64-apple-darwin"; + } + + let triple = Triple::from_str(s)?; + + Ok(Self { + target_family: triple.operating_system.to_compact_string(), + target_arch: if is_universal_macos { + "universal".to_compact_string() + } else { + triple.architecture.to_compact_string() + }, + target_libc: triple.environment.to_compact_string(), + target_vendor: triple.vendor.to_compact_string(), + }) + } +} + +impl leon::Values for TargetTriple { + fn get_value<'s>(&'s self, key: &str) -> Option> { + match key { + "target-family" => Some(Cow::Borrowed(&self.target_family)), + "target-arch" => Some(Cow::Borrowed(&self.target_arch)), + "target-libc" => Some(Cow::Borrowed(&self.target_libc)), + "target-vendor" => Some(Cow::Borrowed(&self.target_vendor)), + + _ => None, + } + } +} diff --git a/crates/binstalk/src/ops/resolve.rs b/crates/binstalk/src/ops/resolve.rs index 0c98a6f3..f38246bf 100644 --- a/crates/binstalk/src/ops/resolve.rs +++ b/crates/binstalk/src/ops/resolve.rs @@ -3,6 +3,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, iter, mem, path::Path, + str::FromStr, sync::Arc, }; @@ -21,7 +22,7 @@ use crate::{ drivers::fetch_crate_cratesio, errors::{BinstallError, VersionParseError}, fetchers::{Data, Fetcher, TargetData}, - helpers::{download::ExtractedFiles, remote::Client}, + helpers::{download::ExtractedFiles, remote::Client, target_triple::TargetTriple}, manifests::cargo_toml_binstall::{Meta, PkgMeta, PkgOverride}, }; @@ -76,7 +77,13 @@ async fn resolve_inner( return Ok(Resolution::AlreadyUpToDate) }; - let desired_targets = opts.desired_targets.get().await; + let desired_targets = opts + .desired_targets + .get() + .await + .iter() + .map(|target| TargetTriple::from_str(target).map(|triple| (triple, target))) + .collect::, _>>()?; let resolvers = &opts.resolvers; let mut handles: Vec<(Arc, _)> = @@ -90,8 +97,8 @@ async fn resolve_inner( handles.extend( desired_targets - .iter() - .map(|target| { + .into_iter() + .map(|(triple, target)| { debug!("Building metadata for target: {target}"); let target_meta = package_info.meta.merge_overrides( @@ -102,6 +109,7 @@ async fn resolve_inner( Arc::new(TargetData { target: target.clone(), + triple, meta: target_meta, }) }) @@ -295,6 +303,7 @@ fn collect_bin_files( meta, bin_path, install_path, + triple: &fetcher.target_data().triple, }; let bin_dir = bin_data