actions/cargo-binstall
1
0
Fork 0
mirror of https://github.com/cargo-bins/cargo-binstall.git synced 2025-05-15 08:20:03 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor: Extract new crate binstalk-fetchers (#1291)

Browse source 
To reduce `binstalk` codegen and enable better reuse.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2023-08-14 13:20:34 +10:00 • committed by GitHub
parent 623f7ff4ed
commit 76c72469eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 1008 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

543
crates/binstalk-fetchers/src/gh_crate_meta.rs Normal file
View file

@ -0,0 +1,543 @@
use std::{borrow::Cow, fmt, iter, marker::PhantomData, path::Path, sync::Arc};
use compact_str::{CompactString, ToCompactString};
use either::Either;
use leon::Template;
use once_cell::sync::OnceCell;
use strum::IntoEnumIterator;
use tracing::{debug, warn};
use url::Url;
use crate::{
common::*, futures_resolver::FuturesResolver, Data, FetchError, InvalidPkgFmtError, RepoInfo,
TargetDataErased,
};
pub(crate) mod hosting;
pub struct GhCrateMeta {
client: Client,
gh_api_client: GhApiClient,
data: Arc<Data>,
target_data: Arc<TargetDataErased>,
resolution: OnceCell<(Url, PkgFmt)>,
}
impl GhCrateMeta {
fn launch_baseline_find_tasks(
&self,
futures_resolver: &FuturesResolver<(Url, PkgFmt), FetchError>,
pkg_fmt: PkgFmt,
pkg_url: &Template<'_>,
repo: Option<&str>,
subcrate: Option<&str>,
) {
let render_url = |ext| {
let ctx = Context::from_data_with_repo(
&self.data,
&self.target_data.target,
&self.target_data.target_related_info,
ext,
repo,
subcrate,
);
match ctx.render_url_with_compiled_tt(pkg_url) {
Ok(url) => Some(url),
Err(err) => {
warn!("Failed to render url for {ctx:#?}: {err}");
None
}
}
};
let is_windows = self.target_data.target.contains("windows");
let urls = if pkg_url.has_any_of_keys(&["format", "archive-format", "archive-suffix"]) {
// build up list of potential URLs
Either::Left(
pkg_fmt
.extensions(is_windows)
.iter()
.filter_map(|ext| render_url(Some(ext))),
)
} else {
Either::Right(render_url(None).into_iter())
};
// go check all potential URLs at once
futures_resolver.extend(urls.map(move |url| {
let client = self.client.clone();
let gh_api_client = self.gh_api_client.clone();
async move {
Ok(does_url_exist(client, gh_api_client, &url)
.await?
.then_some((url, pkg_fmt)))
}
}));
}
}
#[async_trait::async_trait]
impl super::Fetcher for GhCrateMeta {
fn new(
client: Client,
gh_api_client: GhApiClient,
data: Arc<Data>,
target_data: Arc<TargetDataErased>,
) -> Arc<dyn super::Fetcher> {
Arc::new(Self {
client,
gh_api_client,
data,
target_data,
resolution: OnceCell::new(),
})
}
fn find(self: Arc<Self>) -> JoinHandle<Result<bool, FetchError>> {
tokio::spawn(async move {
let info = self.data.get_repo_info(&self.client).await?.as_ref();
let repo = info.map(|info| &info.repo);
let subcrate = info.and_then(|info| info.subcrate.as_deref());
let mut pkg_fmt = self.target_data.meta.pkg_fmt;
let pkg_urls = if let Some(pkg_url) = self.target_data.meta.pkg_url.as_deref() {
let template = Template::parse(pkg_url)?;
if pkg_fmt.is_none()
&& !template.has_any_of_keys(&["format", "archive-format", "archive-suffix"])
{
// The crate does not specify the pkg-fmt, yet its pkg-url
// template doesn't contains format, archive-format or
// archive-suffix which is required for automatically
// deducing the pkg-fmt.
//
// We will attempt to guess the pkg-fmt there, but this is
// just a best-effort
pkg_fmt = PkgFmt::guess_pkg_format(pkg_url);
let crate_name = &self.data.name;
let version = &self.data.version;
let target = &self.target_data.target;
if pkg_fmt.is_none() {
return Err(InvalidPkgFmtError {
crate_name: crate_name.clone(),
version: version.clone(),
target: target.into(),
pkg_url: pkg_url.into(),
reason:
&"pkg-fmt is not specified, yet pkg-url does not contain format, \
archive-format or archive-suffix which is required for automatically deducing pkg-fmt",
}
.into());
}
warn!(
"Crate {crate_name}@{version} on target {target} does not specify pkg-fmt \
but its pkg-url also does not contain key format, archive-format or \
archive-suffix.\nbinstall was able to guess that from pkg-url, but \
just note that it could be wrong:\npkg-fmt=\"{pkg_fmt}\", pkg-url=\"{pkg_url}\"",
pkg_fmt = pkg_fmt.unwrap(),
);
}
Either::Left(iter::once(template))
} else if let Some(RepoInfo {
repo,
repository_host,
..
}) = info
{
if let Some(pkg_urls) = repository_host.get_default_pkg_url_template() {
let has_subcrate = subcrate.is_some();
Either::Right(
pkg_urls
.map(Template::cast)
// If subcrate is Some, then all templates will be included.
// Otherwise, only templates without key "subcrate" will be
// included.
.filter(move |template| has_subcrate || !template.has_key("subcrate")),
)
} else {
warn!(
concat!(
"Unknown repository {}, cargo-binstall cannot provide default pkg_url for it.\n",
"Please ask the upstream to provide it for target {}."
),
repo, self.target_data.target
);
return Ok(false);
}
} else {
warn!(
concat!(
"Package does not specify repository, cargo-binstall cannot provide default pkg_url for it.\n",
"Please ask the upstream to provide it for target {}."
),
self.target_data.target
);
return Ok(false);
};
// Convert Option<Url> to Option<String> to reduce size of future.
let repo = repo.map(|u| u.as_str().trim_end_matches('/'));
// Use reference to self to fix error of closure
// launch_baseline_find_tasks which moves `this`
let this = &self;
let pkg_fmts = if let Some(pkg_fmt) = pkg_fmt {
Either::Left(iter::once(pkg_fmt))
} else {
Either::Right(PkgFmt::iter())
};
let resolver = FuturesResolver::default();
// Iterate over pkg_urls first to avoid String::clone.
for pkg_url in pkg_urls {
// Clone iter pkg_fmts to ensure all pkg_fmts is
// iterated over for each pkg_url, which is
// basically cartesian product.
// |
for pkg_fmt in pkg_fmts.clone() {
this.launch_baseline_find_tasks(&resolver, pkg_fmt, &pkg_url, repo, subcrate);
}
}
if let Some((url, pkg_fmt)) = resolver.resolve().await? {
debug!("Winning URL is {url}, with pkg_fmt {pkg_fmt}");
self.resolution.set((url, pkg_fmt)).unwrap(); // find() is called first
Ok(true)
} else {
Ok(false)
}
})
}
async fn fetch_and_extract(&self, dst: &Path) -> Result<ExtractedFiles, FetchError> {
let (url, pkg_fmt) = self.resolution.get().unwrap(); // find() is called first
debug!(
"Downloading package from: '{url}' dst:{} fmt:{pkg_fmt:?}",
dst.display()
);
Ok(Download::new(self.client.clone(), url.clone())
.and_extract(*pkg_fmt, dst)
.await?)
}
fn pkg_fmt(&self) -> PkgFmt {
self.resolution.get().unwrap().1
}
fn target_meta(&self) -> PkgMeta {
let mut meta = self.target_data.meta.clone();
meta.pkg_fmt = Some(self.pkg_fmt());
meta
}
fn source_name(&self) -> CompactString {
self.resolution
.get()
.map(|(url, _pkg_fmt)| {
if let Some(domain) = url.domain() {
domain.to_compact_string()
} else if let Some(host) = url.host_str() {
host.to_compact_string()
} else {
url.to_compact_string()
}
})
.unwrap_or_else(|| "invalid url".into())
}
fn fetcher_name(&self) -> &'static str {
"GhCrateMeta"
}
fn is_third_party(&self) -> bool {
false
}
fn target(&self) -> &str {
&self.target_data.target
}
fn target_data(&self) -> &Arc<TargetDataErased> {
&self.target_data
}
}
/// Template for constructing download paths
#[derive(Clone)]
struct Context<'c> {
name: &'c str,
repo: Option<&'c str>,
target: &'c str,
version: &'c str,
/// Archive format e.g. tar.gz, zip
archive_format: Option<&'c str>,
archive_suffix: Option<&'c str>,
/// Filename extension on the binary, i.e. .exe on Windows, nothing otherwise
binary_ext: &'c str,
/// Workspace of the crate inside the repository.
subcrate: Option<&'c str>,
target_related_info: &'c dyn leon::Values,
}
impl fmt::Debug for Context<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
#[allow(dead_code)]
#[derive(Debug)]
struct Context<'c> {
name: &'c str,
repo: Option<&'c str>,
target: &'c str,
version: &'c str,
archive_format: Option<&'c str>,
archive_suffix: Option<&'c str>,
binary_ext: &'c str,
subcrate: Option<&'c str>,
target_related_info: PhantomData<&'c dyn leon::Values>,
}
fmt::Debug::fmt(
&Context {
name: self.name,
repo: self.repo,
target: self.target,
version: self.version,
archive_format: self.archive_format,
archive_suffix: self.archive_suffix,
binary_ext: self.binary_ext,
subcrate: self.subcrate,
target_related_info: PhantomData,
},
f,
)
}
}
impl leon::Values for Context<'_> {
fn get_value<'s>(&'s self, key: &str) -> Option<Cow<'s, str>> {
match key {
"name" => Some(Cow::Borrowed(self.name)),
"repo" => self.repo.map(Cow::Borrowed),
"target" => Some(Cow::Borrowed(self.target)),
"version" => Some(Cow::Borrowed(self.version)),
"archive-format" => self.archive_format.map(Cow::Borrowed),
// Soft-deprecated alias for archive-format
"format" => self.archive_format.map(Cow::Borrowed),
"archive-suffix" => self.archive_suffix.map(Cow::Borrowed),
"binary-ext" => Some(Cow::Borrowed(self.binary_ext)),
"subcrate" => self.subcrate.map(Cow::Borrowed),
key => self.target_related_info.get_value(key),
}
}
}
impl<'c> Context<'c> {
fn from_data_with_repo(
data: &'c Data,
target: &'c str,
target_related_info: &'c dyn leon::Values,
archive_suffix: Option<&'c str>,
repo: Option<&'c str>,
subcrate: Option<&'c str>,
) -> Self {
let archive_format = archive_suffix.map(|archive_suffix| {
if archive_suffix.is_empty() {
// Empty archive_suffix means PkgFmt::Bin
"bin"
} else {
debug_assert!(archive_suffix.starts_with('.'), "{archive_suffix}");
&archive_suffix[1..]
}
});
Self {
name: &data.name,
repo,
target,
version: &data.version,
archive_format,
archive_suffix,
binary_ext: if target.contains("windows") {
".exe"
} else {
""
},
subcrate,
target_related_info,
}
}
/// * `tt` - must have added a template named "pkg_url".
fn render_url_with_compiled_tt(&self, tt: &Template<'_>) -> Result<Url, FetchError> {
debug!("Render {tt:#?} using context: {self:?}");
Ok(Url::parse(&tt.render(self)?)?)
}
#[cfg(test)]
fn render_url(&self, template: &str) -> Result<Url, FetchError> {
debug!("Render {template} using context in render_url: {self:?}");
let tt = Template::parse(template)?;
self.render_url_with_compiled_tt(&tt)
}
}
#[cfg(test)]
mod test {
use super::{super::Data, Context};
use compact_str::ToCompactString;
use url::Url;
const DEFAULT_PKG_URL: &str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }";
fn assert_context_rendering(
data: &Data,
target: &str,
archive_format: &str,
template: &str,
expected_url: &str,
) {
// The template provided doesn't need this, so just returning None
// is OK.
let target_info = leon::vals(|_| None);
let ctx = Context::from_data_with_repo(
data,
target,
&target_info,
Some(archive_format),
data.repo.as_deref(),
None,
);
let expected_url = Url::parse(expected_url).unwrap();
assert_eq!(ctx.render_url(template).unwrap(), expected_url);
}
#[test]
fn defaults() {
assert_context_rendering(
&Data::new(
"cargo-binstall".to_compact_string(),
"1.2.3".to_compact_string(),
Some("https://github.com/ryankurte/cargo-binstall".to_string()),
),
"x86_64-unknown-linux-gnu",
".tgz",
DEFAULT_PKG_URL,
"https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz"
);
}
#[test]
fn no_repo_but_full_url() {
assert_context_rendering(
&Data::new(
"cargo-binstall".to_compact_string(),
"1.2.3".to_compact_string(),
None,
),
"x86_64-unknown-linux-gnu",
".tgz",
&format!("https://example.com{}", &DEFAULT_PKG_URL[8..]),
"https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz"
);
}
#[test]
fn different_url() {
assert_context_rendering(
&Data::new(
"radio-sx128x".to_compact_string(),
"0.14.1-alpha.5".to_compact_string(),
Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()),
),
"x86_64-unknown-linux-gnu",
".tgz",
"{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ archive-format }",
"https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz"
);
}
#[test]
fn deprecated_format() {
assert_context_rendering(
&Data::new(
"radio-sx128x".to_compact_string(),
"0.14.1-alpha.5".to_compact_string(),
Some("https://github.com/rust-iot/rust-radio-sx128x".to_string()),
),
"x86_64-unknown-linux-gnu",
".tgz",
"{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }",
"https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz"
);
}
#[test]
fn different_ext() {
assert_context_rendering(
&Data::new(
"cargo-watch".to_compact_string(),
"9.0.0".to_compact_string(),
Some("https://github.com/watchexec/cargo-watch".to_string()),
),
"aarch64-apple-darwin",
".txz",
"{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.tar.xz",
"https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz"
);
}
#[test]
fn no_archive() {
assert_context_rendering(
&Data::new(
"cargo-watch".to_compact_string(),
"9.0.0".to_compact_string(),
Some("https://github.com/watchexec/cargo-watch".to_string()),
),
"aarch64-pc-windows-msvc",
".bin",
"{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }",
"https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe"
);
}
}