diff --git a/SUPPORT.md b/SUPPORT.md index 7c2c871e..5b77791d 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -4,7 +4,10 @@ `binstall` works with existing CI-built binary outputs, with configuration via `[package.metadata.binstall]` keys in the relevant crate manifest. When configuring `binstall` you can test against a local manifest with `--manifest-path=PATH` argument to use the crate and manifest at the provided `PATH`, skipping crate discovery and download. -To get started, add a `[package.metadata.binstall]` section to your `Cargo.toml`. As an example, the default configuration would be: +To get started, check the [default](#Defaults) first, only add a `[package.metadata.binstall]` section +to your `Cargo.toml` if the default does not work for you. + +As an example, the configuration would be like this: ```toml [package.metadata.binstall] @@ -40,18 +43,93 @@ pkg-fmt = "zip" ### Defaults -By default `binstall` is setup to work with github releases, and expects to find: +By default, `binstall` will try all supported package format and would have `bin-dir` set to +`"{ name }-{ target }-v{ version }/{ bin }{ binary-ext }"` (where `bin` is the cargo binary name and +`binary-ext` is `.exe` on windows and empty on other platforms). -- an archive named `{ name }-{ target }-v{ version }.{ archive-format }` - - so that this does not overwrite different targets or versions when manually downloaded -- located at `{ repo }/releases/download/v{ version }/` - - compatible with github tags / releases -- containing a folder named `{ name }-{ target }-v{ version }` - - so that prior binary files are not overwritten when manually executing `tar -xvf ...` -- containing binary files in the form `{ bin }{ binary-ext }` (where `bin` is the cargo binary name and `binary-ext` is `.exe` on windows and empty on other platforms) +All binaries must contain a folder named `{ name }-{ target }-v{ version }` (so that prior binary +files are not overwritten when manually executing `tar -xvf ...`). + +The default value for `pkg-url` will depend on the repository of the package. + +It is setup to work with github releases, gitlab releases, bitbucket downloads +and source forge downloads. + +#### Github + +For github, the `pkg-url` is set to + +```rust +[ + "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }", + "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }", + "{ repo }/releases/download/v{ version }/{ name }-{ version }-{ target }.{ archive-format }", + "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }", +] +``` + +The first 3 versions does not overwrite different targets or versions when manually downloaded. + +All `pkg-url` templates download binaries located at `{ repo }/releases/download/v{ version }/`, which +is compatible with github tags / releases. If your package already uses this approach, you shouldn't need to set anything. +#### GitLab + +For gitlab, the `pkg-url` is set to + +```rust +[ + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }-v{ version }.{ archive-format }", + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-v{ version }-{ target }.{ archive-format }", + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ version }-{ target }.{ archive-format }", + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }.{ archive-format }", +] +``` + +This will attempt to find the release assets with `filepath` set to +`binaries/{ name }-{ target }.{ archive-format }` + +Note that this uses the [Permanent links to release assets](https://gitlab.kitware.com/help/user/project/releases/index#permanent-links-to-latest-release-assets) feature of gitlab, it requires you to +create an asset as a link with a `filepath`, which can be set only using gitlab api as of the writing. + +#### BitBucket + +For bitbucket, the `pkg-url` is set to + +```rust +[ + "{ repo }/downloads/{ name }-{ target }-v{ version }.{ archive-format }", + "{ repo }/downloads/{ name }-v{ version }-{ target }.{ archive-format }", + "{ repo }/downloads/{ name }-{ version }-{ target }.{ archive-format }", +] +``` + +To setup the package for binstall, upload the binary into bitbucket downloads page of your project, +with its name set to be `{ name }-{ target }-v{ version }.{ archive-format }`. + +#### SourceForge + +For source forge, the `pkg-url` is set to + +```rust +[ + "{ repo }/files/binaries/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }/download", + "{ repo }/files/binaries/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }/download", + "{ repo }/files/binaries/v{ version }/{ name }-{ version }-{ target }.{ archive-format }/download", + "{ repo }/files/binaries/v{ version }/{ name }-{ target }.{ archive-format }/download", +] +``` + +To setup the package for binstall, upload the binary to the file page of your project, +under the directory `binaries/v{ version }` with the filename `{ name }-{ target }.{ archive-format }`. + +#### Others + +For all other situations, `binstall` does not provide a default `pkg-url` and you need to manually +specify it. + ### QuickInstall [QuickInstall](https://github.com/alsuren/cargo-quickinstall) is an unofficial repository of prebuilt binaries for Crates, and `binstall` has built-in support for it! If your crate is built by QuickInstall, it will already work with `binstall`. However, binaries as configured above take precedence when they exist. diff --git a/crates/bin/Cargo.toml b/crates/bin/Cargo.toml index d6f416ca..836d4ba3 100644 --- a/crates/bin/Cargo.toml +++ b/crates/bin/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" license = "GPL-3.0" [package.metadata.binstall] -pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }" bin-dir = "{ bin }{ binary-ext }" [package.metadata.binstall.overrides.x86_64-pc-windows-msvc] diff --git a/crates/lib/src/fetchers/gh_crate_meta.rs b/crates/lib/src/fetchers/gh_crate_meta.rs index 897327df..267af11b 100644 --- a/crates/lib/src/fetchers/gh_crate_meta.rs +++ b/crates/lib/src/fetchers/gh_crate_meta.rs @@ -1,4 +1,4 @@ -use std::{path::Path, sync::Arc}; +use std::{borrow::Cow, path::Path, sync::Arc}; use compact_str::{CompactString, ToCompactString}; use log::{debug, warn}; @@ -11,12 +11,19 @@ use url::Url; use crate::{ errors::BinstallError, - helpers::{download::download_and_extract, remote::remote_exists, tasks::AutoAbortJoinHandle}, + helpers::{ + download::download_and_extract, + remote::{get_redirected_final_url, remote_exists}, + tasks::AutoAbortJoinHandle, + }, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, }; use super::Data; +mod hosting; +use hosting::GitHostingServices; + pub struct GhCrateMeta { client: Client, data: Arc, @@ -26,14 +33,16 @@ pub struct GhCrateMeta { type BaselineFindTask = AutoAbortJoinHandle, BinstallError>>; impl GhCrateMeta { - fn launch_baseline_find_tasks( - &self, + fn launch_baseline_find_tasks<'a>( + &'a self, pkg_fmt: PkgFmt, - ) -> impl Iterator + '_ { + pkg_url: &'a str, + repo: Option<&'a str>, + ) -> impl Iterator + 'a { // build up list of potential URLs - let urls = pkg_fmt.extensions().iter().filter_map(|ext| { - let ctx = Context::from_data(&self.data, ext); - match ctx.render_url(&self.data.meta.pkg_url) { + let urls = pkg_fmt.extensions().iter().filter_map(move |ext| { + let ctx = Context::from_data_with_repo(&self.data, ext, repo); + match ctx.render_url(pkg_url) { Ok(url) => Some(url), Err(err) => { warn!("Failed to render url for {ctx:#?}: {err:#?}"); @@ -68,11 +77,54 @@ impl super::Fetcher for GhCrateMeta { } async fn find(&self) -> Result { + let repo = if let Some(repo) = self.data.repo.as_deref() { + Some(get_redirected_final_url(&self.client, Url::parse(repo)?).await?) + } else { + None + }; + + let pkg_urls = if let Some(pkg_url) = self.data.meta.pkg_url.as_deref() { + Cow::Owned(vec![pkg_url]) + } else if let Some(repo) = repo.as_ref() { + if let Some(pkg_urls) = + GitHostingServices::guess_git_hosting_services(repo)?.get_default_pkg_url_template() + { + Cow::Borrowed(pkg_urls) + } else { + warn!( + concat!( + "Unknown repository {}, cargo-binstall cannot provide default pkg_url for it.\n", + "Please ask the upstream to provide it for target {}." + ), + repo, self.data.target + ); + + return Ok(false); + } + } else { + warn!( + concat!( + "Package does not specify repository, cargo-binstall cannot provide default pkg_url for it.\n", + "Please ask the upstream to provide it for target {}." + ), + self.data.target + ); + + return Ok(false); + }; + + let repo = repo.as_ref().map(Url::as_str); + let launch_baseline_find_tasks = |pkg_fmt| { + pkg_urls + .iter() + .flat_map(move |pkg_url| self.launch_baseline_find_tasks(pkg_fmt, pkg_url, repo)) + }; + let handles: Vec<_> = if let Some(pkg_fmt) = self.data.meta.pkg_fmt { - self.launch_baseline_find_tasks(pkg_fmt).collect() + launch_baseline_find_tasks(pkg_fmt).collect() } else { PkgFmt::iter() - .flat_map(|pkg_fmt| self.launch_baseline_find_tasks(pkg_fmt)) + .flat_map(launch_baseline_find_tasks) .collect() }; @@ -148,10 +200,14 @@ struct Context<'c> { } impl<'c> Context<'c> { - pub(self) fn from_data(data: &'c Data, archive_format: &'c str) -> Self { + pub(self) fn from_data_with_repo( + data: &'c Data, + archive_format: &'c str, + repo: Option<&'c str>, + ) -> Self { Self { name: &data.name, - repo: data.repo.as_deref(), + repo, target: &data.target, version: &data.version, format: archive_format, @@ -164,6 +220,11 @@ impl<'c> Context<'c> { } } + #[cfg(test)] + pub(self) fn from_data(data: &'c Data, archive_format: &'c str) -> Self { + Self::from_data_with_repo(data, archive_format, data.repo.as_deref()) + } + pub(self) fn render_url(&self, template: &str) -> Result { debug!("Render {template:?} using context: {:?}", self); @@ -180,6 +241,8 @@ mod test { use super::{super::Data, Context}; use url::Url; + const DEFAULT_PKG_URL: &str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }"; + fn url(s: &str) -> Url { Url::parse(s).unwrap() } @@ -197,7 +260,7 @@ mod test { let ctx = Context::from_data(&data, "tgz"); assert_eq!( - ctx.render_url(&data.meta.pkg_url).unwrap(), + ctx.render_url(DEFAULT_PKG_URL).unwrap(), url("https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz") ); } @@ -215,13 +278,14 @@ mod test { }; let ctx = Context::from_data(&data, "tgz"); - ctx.render_url(&data.meta.pkg_url).unwrap(); + ctx.render_url(data.meta.pkg_url.as_deref().unwrap()) + .unwrap(); } #[test] fn no_repo_but_full_url() { let meta = PkgMeta { - pkg_url: format!("https://example.com{}", PkgMeta::default().pkg_url), + pkg_url: Some(format!("https://example.com{DEFAULT_PKG_URL}")), ..Default::default() }; @@ -235,7 +299,7 @@ mod test { let ctx = Context::from_data(&data, "tgz"); assert_eq!( - ctx.render_url(&data.meta.pkg_url).unwrap(), + ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(), url("https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz") ); } @@ -243,9 +307,9 @@ mod test { #[test] fn different_url() { let meta = PkgMeta { - pkg_url: + pkg_url: Some( "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ archive-format }" - .into(), + .to_string()), ..Default::default() }; @@ -259,7 +323,7 @@ mod test { let ctx = Context::from_data(&data, "tgz"); assert_eq!( - ctx.render_url(&data.meta.pkg_url).unwrap(), + ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(), url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz") ); } @@ -267,7 +331,7 @@ mod test { #[test] fn deprecated_format() { let meta = PkgMeta { - pkg_url: "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }".into(), + pkg_url: Some("{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }".to_string()), ..Default::default() }; @@ -281,7 +345,7 @@ mod test { let ctx = Context::from_data(&data, "tgz"); assert_eq!( - ctx.render_url(&data.meta.pkg_url).unwrap(), + ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(), url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz") ); } @@ -289,9 +353,10 @@ mod test { #[test] fn different_ext() { let meta = PkgMeta { - pkg_url: + pkg_url: Some( "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.tar.xz" - .into(), + .to_string(), + ), pkg_fmt: Some(PkgFmt::Txz), ..Default::default() }; @@ -306,7 +371,7 @@ mod test { let ctx = Context::from_data(&data, "txz"); assert_eq!( - ctx.render_url(&data.meta.pkg_url).unwrap(), + ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(), url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz") ); } @@ -314,7 +379,7 @@ mod test { #[test] fn no_archive() { let meta = PkgMeta { - pkg_url: "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }".into(), + pkg_url: Some("{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }".to_string()), pkg_fmt: Some(PkgFmt::Bin), ..Default::default() }; @@ -329,7 +394,7 @@ mod test { let ctx = Context::from_data(&data, "bin"); assert_eq!( - ctx.render_url(&data.meta.pkg_url).unwrap(), + ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(), url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe") ); } diff --git a/crates/lib/src/fetchers/gh_crate_meta/hosting.rs b/crates/lib/src/fetchers/gh_crate_meta/hosting.rs new file mode 100644 index 00000000..265fab79 --- /dev/null +++ b/crates/lib/src/fetchers/gh_crate_meta/hosting.rs @@ -0,0 +1,56 @@ +use url::Url; + +use crate::errors::BinstallError; + +#[derive(Copy, Clone, Debug)] +pub enum GitHostingServices { + GitHub, + GitLab, + BitBucket, + SourceForge, + Unknown, +} +impl GitHostingServices { + pub fn guess_git_hosting_services(repo: &Url) -> Result { + use GitHostingServices::*; + + match repo.domain() { + Some(domain) if domain.starts_with("github") => Ok(GitHub), + Some(domain) if domain.starts_with("gitlab") => Ok(GitLab), + Some(domain) if domain == "bitbucket.org" => Ok(BitBucket), + Some(domain) if domain == "sourceforge.net" => Ok(SourceForge), + _ => Ok(Unknown), + } + } + + pub fn get_default_pkg_url_template(self) -> Option<&'static [&'static str]> { + use GitHostingServices::*; + + match self { + GitHub => Some(&[ + "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }", + "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }", + "{ repo }/releases/download/v{ version }/{ name }-{ version }-{ target }.{ archive-format }", + "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }", + ]), + GitLab => Some(&[ + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }-v{ version }.{ archive-format }", + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-v{ version }-{ target }.{ archive-format }", + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ version }-{ target }.{ archive-format }", + "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }.{ archive-format }", + ]), + BitBucket => Some(&[ + "{ repo }/downloads/{ name }-{ target }-v{ version }.{ archive-format }", + "{ repo }/downloads/{ name }-v{ version }-{ target }.{ archive-format }", + "{ repo }/downloads/{ name }-{ version }-{ target }.{ archive-format }", + ]), + SourceForge => Some(&[ + "{ repo }/files/binaries/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }/download", + "{ repo }/files/binaries/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }/download", + "{ repo }/files/binaries/v{ version }/{ name }-{ version }-{ target }.{ archive-format }/download", + "{ repo }/files/binaries/v{ version }/{ name }-{ target }.{ archive-format }/download", + ]), + Unknown => None, + } + } +} diff --git a/crates/lib/src/helpers/remote.rs b/crates/lib/src/helpers/remote.rs index 042ae7ce..e609015c 100644 --- a/crates/lib/src/helpers/remote.rs +++ b/crates/lib/src/helpers/remote.rs @@ -42,6 +42,19 @@ pub async fn remote_exists( Ok(req.status().is_success()) } +pub async fn get_redirected_final_url(client: &Client, url: Url) -> Result { + let method = Method::HEAD; + + let req = client + .request(method.clone(), url.clone()) + .send() + .await + .and_then(Response::error_for_status) + .map_err(|err| BinstallError::Http { method, url, err })?; + + Ok(req.url().clone()) +} + pub(crate) async fn create_request( client: &Client, url: Url, diff --git a/crates/lib/src/manifests/cargo_toml_binstall.rs b/crates/lib/src/manifests/cargo_toml_binstall.rs index 8f2748f3..5f04cf5d 100644 --- a/crates/lib/src/manifests/cargo_toml_binstall.rs +++ b/crates/lib/src/manifests/cargo_toml_binstall.rs @@ -11,10 +11,6 @@ pub use package_formats::*; mod package_formats; -/// Default package path template (may be overridden in package Cargo.toml) -pub const DEFAULT_PKG_URL: &str = - "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }"; - /// Default binary name template (may be overridden in package Cargo.toml) pub const DEFAULT_BIN_DIR: &str = "{ name }-{ target }-v{ version }/{ bin }{ binary-ext }"; @@ -34,7 +30,7 @@ pub struct Meta { #[serde(rename_all = "kebab-case", default)] pub struct PkgMeta { /// URL template for package downloads - pub pkg_url: String, + pub pkg_url: Option, /// Format for package downloads pub pkg_fmt: Option, @@ -52,7 +48,7 @@ pub struct PkgMeta { impl Default for PkgMeta { fn default() -> Self { Self { - pkg_url: DEFAULT_PKG_URL.to_string(), + pkg_url: None, pkg_fmt: None, bin_dir: DEFAULT_BIN_DIR.to_string(), pub_key: None, @@ -75,7 +71,7 @@ impl PkgMeta { /// Merge configuration overrides into object pub fn merge(&mut self, pkg_override: &PkgOverride) { if let Some(o) = &pkg_override.pkg_url { - self.pkg_url = o.clone(); + self.pkg_url = Some(o.clone()); } if let Some(o) = &pkg_override.pkg_fmt { self.pkg_fmt = Some(*o); diff --git a/crates/lib/tests/parse-meta.rs b/crates/lib/tests/parse-meta.rs index 7433974b..4517943c 100644 --- a/crates/lib/tests/parse-meta.rs +++ b/crates/lib/tests/parse-meta.rs @@ -15,7 +15,7 @@ fn parse_meta() { assert_eq!(&package.name, "cargo-binstall-test"); assert_eq!( - &meta.pkg_url, + meta.pkg_url.as_deref().unwrap(), "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }" );