Support other git hosting services (#312)

* Impl new mod `hosting` for detecting git hosting services
* Refactor: Make `guess_git_hosting_services` associated fn of `GitHostingService`
* Set default value of `PkgMeta::pkg_url` to `None`
* Impl new method `get_redirected_final_url`
* Use `get_redirected_final_url` in `GhCrateMeta::find` to make `guess_git_hosting_services` more accurate.
* Use redirected `repo` in `GhCrateMeta::launch_baseline_find_tasks`
* Refactor `<GhCrateMeta as Fetcher>::find`
* Mod `get_default_pkg_url_template` to ret `&[&str]`
* Add more default `pkg-url` templates
* Rm `pkg-url` in `bin/Cargo.toml`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

SUPPORT.md

@@ -4,7 +4,10 @@
 `binstall` works with existing CI-built binary outputs, with configuration via `[package.metadata.binstall]` keys in the relevant crate manifest.
 When configuring `binstall` you can test against a local manifest with `--manifest-path=PATH` argument to use the crate and manifest at the provided `PATH`, skipping crate discovery and download.
 
-To get started, add a `[package.metadata.binstall]` section to your `Cargo.toml`. As an example, the default configuration would be:
+To get started, check the [default](#Defaults) first, only add a `[package.metadata.binstall]` section
+to your `Cargo.toml` if the default does not work for you.
+
+As an example, the configuration would be like this:
 
 ```toml
 [package.metadata.binstall]
@@ -40,18 +43,93 @@ pkg-fmt = "zip"
 
 ### Defaults
 
-By default `binstall` is setup to work with github releases, and expects to find:
+By default, `binstall` will try all supported package format and would have `bin-dir` set to
+`"{ name }-{ target }-v{ version }/{ bin }{ binary-ext }"` (where `bin` is the cargo binary name and
+`binary-ext` is `.exe` on windows and empty on other platforms).
 
-- an archive named `{ name }-{ target }-v{ version }.{ archive-format }`
+All binaries must contain a folder named `{ name }-{ target }-v{ version }` (so that prior binary
-  - so that this does not overwrite different targets or versions when manually downloaded
+files are not overwritten when manually executing `tar -xvf ...`).
-- located at `{ repo }/releases/download/v{ version }/`
+
-  - compatible with github tags / releases
+The default value for `pkg-url` will depend on the repository of the package.
-- containing a folder named `{ name }-{ target }-v{ version }`
+
-  - so that prior binary files are not overwritten when manually executing `tar -xvf ...`
+It is setup to work with github releases, gitlab releases, bitbucket downloads
-- containing binary files in the form `{ bin }{ binary-ext }` (where `bin` is the cargo binary name and `binary-ext` is `.exe` on windows and empty on other platforms)
+and source forge downloads.
+
+#### Github
+
+For github, the `pkg-url` is set to
+
+```rust
+[
+    "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }",
+    "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }",
+    "{ repo }/releases/download/v{ version }/{ name }-{ version }-{ target }.{ archive-format }",
+    "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }",
+]
+```
+
+The first 3 versions does not overwrite different targets or versions when manually downloaded.
+
+All `pkg-url` templates download binaries located at `{ repo }/releases/download/v{ version }/`, which
+is compatible with github tags / releases.
 
 If your package already uses this approach, you shouldn't need to set anything.
 
+#### GitLab
+
+For gitlab, the `pkg-url` is set to
+
+```rust
+[
+    "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }-v{ version }.{ archive-format }",
+    "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-v{ version }-{ target }.{ archive-format }",
+    "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ version }-{ target }.{ archive-format }",
+    "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }.{ archive-format }",
+]
+```
+
+This will attempt to find the release assets with `filepath` set to
+`binaries/{ name }-{ target }.{ archive-format }`
+
+Note that this uses the [Permanent links to release assets](https://gitlab.kitware.com/help/user/project/releases/index#permanent-links-to-latest-release-assets) feature of gitlab, it requires you to
+create an asset as a link with a `filepath`, which can be set only using gitlab api as of the writing.
+
+#### BitBucket
+
+For bitbucket, the `pkg-url` is set to
+
+```rust
+[
+    "{ repo }/downloads/{ name }-{ target }-v{ version }.{ archive-format }",
+    "{ repo }/downloads/{ name }-v{ version }-{ target }.{ archive-format }",
+    "{ repo }/downloads/{ name }-{ version }-{ target }.{ archive-format }",
+]
+```
+
+To setup the package for binstall, upload the binary into bitbucket downloads page of your project,
+with its name set to be `{ name }-{ target }-v{ version }.{ archive-format }`.
+
+#### SourceForge
+
+For source forge, the `pkg-url` is set to
+
+```rust
+[
+    "{ repo }/files/binaries/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }/download",
+    "{ repo }/files/binaries/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }/download",
+    "{ repo }/files/binaries/v{ version }/{ name }-{ version }-{ target }.{ archive-format }/download",
+    "{ repo }/files/binaries/v{ version }/{ name }-{ target }.{ archive-format }/download",
+]
+```
+
+To setup the package for binstall, upload the binary to the file page of your project,
+under the directory `binaries/v{ version }` with the filename `{ name }-{ target }.{ archive-format }`.
+
+#### Others
+
+For all other situations, `binstall` does not provide a default `pkg-url` and you need to manually
+specify it.
+
 ### QuickInstall
 
 [QuickInstall](https://github.com/alsuren/cargo-quickinstall) is an unofficial repository of prebuilt binaries for Crates, and `binstall` has built-in support for it! If your crate is built by QuickInstall, it will already work with `binstall`. However, binaries as configured above take precedence when they exist.

crates/bin/Cargo.toml

@@ -10,7 +10,6 @@ edition = "2021"
 license = "GPL-3.0"
 
 [package.metadata.binstall]
-pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }"
 bin-dir = "{ bin }{ binary-ext }"
 
 [package.metadata.binstall.overrides.x86_64-pc-windows-msvc]

crates/lib/src/fetchers/gh_crate_meta.rs

@@ -1,4 +1,4 @@
-use std::{path::Path, sync::Arc};
+use std::{borrow::Cow, path::Path, sync::Arc};
 
 use compact_str::{CompactString, ToCompactString};
 use log::{debug, warn};
@@ -11,12 +11,19 @@ use url::Url;
 
 use crate::{
     errors::BinstallError,
-    helpers::{download::download_and_extract, remote::remote_exists, tasks::AutoAbortJoinHandle},
+    helpers::{
+        download::download_and_extract,
+        remote::{get_redirected_final_url, remote_exists},
+        tasks::AutoAbortJoinHandle,
+    },
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
 };
 
 use super::Data;
 
+mod hosting;
+use hosting::GitHostingServices;
+
 pub struct GhCrateMeta {
     client: Client,
     data: Arc<Data>,
@@ -26,14 +33,16 @@ pub struct GhCrateMeta {
 type BaselineFindTask = AutoAbortJoinHandle<Result<Option<(Url, PkgFmt)>, BinstallError>>;
 
 impl GhCrateMeta {
-    fn launch_baseline_find_tasks(
+    fn launch_baseline_find_tasks<'a>(
-        &self,
+        &'a self,
         pkg_fmt: PkgFmt,
-    ) -> impl Iterator<Item = BaselineFindTask> + '_ {
+        pkg_url: &'a str,
+        repo: Option<&'a str>,
+    ) -> impl Iterator<Item = BaselineFindTask> + 'a {
         // build up list of potential URLs
-        let urls = pkg_fmt.extensions().iter().filter_map(|ext| {
+        let urls = pkg_fmt.extensions().iter().filter_map(move |ext| {
-            let ctx = Context::from_data(&self.data, ext);
+            let ctx = Context::from_data_with_repo(&self.data, ext, repo);
-            match ctx.render_url(&self.data.meta.pkg_url) {
+            match ctx.render_url(pkg_url) {
                 Ok(url) => Some(url),
                 Err(err) => {
                     warn!("Failed to render url for {ctx:#?}: {err:#?}");
@@ -68,11 +77,54 @@ impl super::Fetcher for GhCrateMeta {
     }
 
     async fn find(&self) -> Result<bool, BinstallError> {
+        let repo = if let Some(repo) = self.data.repo.as_deref() {
+            Some(get_redirected_final_url(&self.client, Url::parse(repo)?).await?)
+        } else {
+            None
+        };
+
+        let pkg_urls = if let Some(pkg_url) = self.data.meta.pkg_url.as_deref() {
+            Cow::Owned(vec![pkg_url])
+        } else if let Some(repo) = repo.as_ref() {
+            if let Some(pkg_urls) =
+                GitHostingServices::guess_git_hosting_services(repo)?.get_default_pkg_url_template()
+            {
+                Cow::Borrowed(pkg_urls)
+            } else {
+                warn!(
+                    concat!(
+                        "Unknown repository {}, cargo-binstall cannot provide default pkg_url for it.\n",
+                        "Please ask the upstream to provide it for target {}."
+                    ),
+                    repo, self.data.target
+                );
+
+                return Ok(false);
+            }
+        } else {
+            warn!(
+                concat!(
+                    "Package does not specify repository, cargo-binstall cannot provide default pkg_url for it.\n",
+                    "Please ask the upstream to provide it for target {}."
+                ),
+                self.data.target
+            );
+
+            return Ok(false);
+        };
+
+        let repo = repo.as_ref().map(Url::as_str);
+        let launch_baseline_find_tasks = |pkg_fmt| {
+            pkg_urls
+                .iter()
+                .flat_map(move |pkg_url| self.launch_baseline_find_tasks(pkg_fmt, pkg_url, repo))
+        };
+
         let handles: Vec<_> = if let Some(pkg_fmt) = self.data.meta.pkg_fmt {
-            self.launch_baseline_find_tasks(pkg_fmt).collect()
+            launch_baseline_find_tasks(pkg_fmt).collect()
         } else {
             PkgFmt::iter()
-                .flat_map(|pkg_fmt| self.launch_baseline_find_tasks(pkg_fmt))
+                .flat_map(launch_baseline_find_tasks)
                 .collect()
         };
 
@@ -148,10 +200,14 @@ struct Context<'c> {
 }
 
 impl<'c> Context<'c> {
-    pub(self) fn from_data(data: &'c Data, archive_format: &'c str) -> Self {
+    pub(self) fn from_data_with_repo(
+        data: &'c Data,
+        archive_format: &'c str,
+        repo: Option<&'c str>,
+    ) -> Self {
         Self {
             name: &data.name,
-            repo: data.repo.as_deref(),
+            repo,
             target: &data.target,
             version: &data.version,
             format: archive_format,
@@ -164,6 +220,11 @@ impl<'c> Context<'c> {
         }
     }
 
+    #[cfg(test)]
+    pub(self) fn from_data(data: &'c Data, archive_format: &'c str) -> Self {
+        Self::from_data_with_repo(data, archive_format, data.repo.as_deref())
+    }
+
     pub(self) fn render_url(&self, template: &str) -> Result<Url, BinstallError> {
         debug!("Render {template:?} using context: {:?}", self);
 
@@ -180,6 +241,8 @@ mod test {
     use super::{super::Data, Context};
     use url::Url;
 
+    const DEFAULT_PKG_URL: &str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }";
+
     fn url(s: &str) -> Url {
         Url::parse(s).unwrap()
     }
@@ -197,7 +260,7 @@ mod test {
 
         let ctx = Context::from_data(&data, "tgz");
         assert_eq!(
-            ctx.render_url(&data.meta.pkg_url).unwrap(),
+            ctx.render_url(DEFAULT_PKG_URL).unwrap(),
             url("https://github.com/ryankurte/cargo-binstall/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz")
         );
     }
@@ -215,13 +278,14 @@ mod test {
         };
 
         let ctx = Context::from_data(&data, "tgz");
-        ctx.render_url(&data.meta.pkg_url).unwrap();
+        ctx.render_url(data.meta.pkg_url.as_deref().unwrap())
+            .unwrap();
     }
 
     #[test]
     fn no_repo_but_full_url() {
         let meta = PkgMeta {
-            pkg_url: format!("https://example.com{}", PkgMeta::default().pkg_url),
+            pkg_url: Some(format!("https://example.com{DEFAULT_PKG_URL}")),
             ..Default::default()
         };
 
@@ -235,7 +299,7 @@ mod test {
 
         let ctx = Context::from_data(&data, "tgz");
         assert_eq!(
-            ctx.render_url(&data.meta.pkg_url).unwrap(),
+            ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(),
             url("https://example.com/releases/download/v1.2.3/cargo-binstall-x86_64-unknown-linux-gnu-v1.2.3.tgz")
         );
     }
@@ -243,9 +307,9 @@ mod test {
     #[test]
     fn different_url() {
         let meta = PkgMeta {
-            pkg_url:
+            pkg_url: Some(
             "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ archive-format }"
-                .into(),
+                .to_string()),
             ..Default::default()
         };
 
@@ -259,7 +323,7 @@ mod test {
 
         let ctx = Context::from_data(&data, "tgz");
         assert_eq!(
-            ctx.render_url(&data.meta.pkg_url).unwrap(),
+            ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(),
             url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz")
         );
     }
@@ -267,7 +331,7 @@ mod test {
     #[test]
     fn deprecated_format() {
         let meta = PkgMeta {
-            pkg_url: "{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }".into(),
+            pkg_url: Some("{ repo }/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.{ format }".to_string()),
             ..Default::default()
         };
 
@@ -281,7 +345,7 @@ mod test {
 
         let ctx = Context::from_data(&data, "tgz");
         assert_eq!(
-            ctx.render_url(&data.meta.pkg_url).unwrap(),
+            ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(),
             url("https://github.com/rust-iot/rust-radio-sx128x/releases/download/v0.14.1-alpha.5/sx128x-util-x86_64-unknown-linux-gnu-v0.14.1-alpha.5.tgz")
         );
     }
@@ -289,9 +353,10 @@ mod test {
     #[test]
     fn different_ext() {
         let meta = PkgMeta {
-            pkg_url:
+            pkg_url: Some(
                 "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.tar.xz"
-                    .into(),
+                    .to_string(),
+            ),
             pkg_fmt: Some(PkgFmt::Txz),
             ..Default::default()
         };
@@ -306,7 +371,7 @@ mod test {
 
         let ctx = Context::from_data(&data, "txz");
         assert_eq!(
-            ctx.render_url(&data.meta.pkg_url).unwrap(),
+            ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(),
             url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-apple-darwin.tar.xz")
         );
     }
@@ -314,7 +379,7 @@ mod test {
     #[test]
     fn no_archive() {
         let meta = PkgMeta {
-            pkg_url: "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }".into(),
+            pkg_url: Some("{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }{ binary-ext }".to_string()),
             pkg_fmt: Some(PkgFmt::Bin),
             ..Default::default()
         };
@@ -329,7 +394,7 @@ mod test {
 
         let ctx = Context::from_data(&data, "bin");
         assert_eq!(
-            ctx.render_url(&data.meta.pkg_url).unwrap(),
+            ctx.render_url(data.meta.pkg_url.as_deref().unwrap()).unwrap(),
             url("https://github.com/watchexec/cargo-watch/releases/download/v9.0.0/cargo-watch-v9.0.0-aarch64-pc-windows-msvc.exe")
         );
     }

crates/lib/src/fetchers/gh_crate_meta/hosting.rs

@@ -0,0 +1,56 @@
+use url::Url;
+
+use crate::errors::BinstallError;
+
+#[derive(Copy, Clone, Debug)]
+pub enum GitHostingServices {
+    GitHub,
+    GitLab,
+    BitBucket,
+    SourceForge,
+    Unknown,
+}
+impl GitHostingServices {
+    pub fn guess_git_hosting_services(repo: &Url) -> Result<Self, BinstallError> {
+        use GitHostingServices::*;
+
+        match repo.domain() {
+            Some(domain) if domain.starts_with("github") => Ok(GitHub),
+            Some(domain) if domain.starts_with("gitlab") => Ok(GitLab),
+            Some(domain) if domain == "bitbucket.org" => Ok(BitBucket),
+            Some(domain) if domain == "sourceforge.net" => Ok(SourceForge),
+            _ => Ok(Unknown),
+        }
+    }
+
+    pub fn get_default_pkg_url_template(self) -> Option<&'static [&'static str]> {
+        use GitHostingServices::*;
+
+        match self {
+            GitHub => Some(&[
+                "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }",
+                "{ repo }/releases/download/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }",
+                "{ repo }/releases/download/v{ version }/{ name }-{ version }-{ target }.{ archive-format }",
+                "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }",
+            ]),
+            GitLab => Some(&[
+                "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }-v{ version }.{ archive-format }",
+                "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-v{ version }-{ target }.{ archive-format }",
+                "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ version }-{ target }.{ archive-format }",
+                "{ repo }/-/releases/v{ version }/downloads/binaries/{ name }-{ target }.{ archive-format }",
+            ]),
+            BitBucket => Some(&[
+                "{ repo }/downloads/{ name }-{ target }-v{ version }.{ archive-format }",
+                "{ repo }/downloads/{ name }-v{ version }-{ target }.{ archive-format }",
+                "{ repo }/downloads/{ name }-{ version }-{ target }.{ archive-format }",
+            ]),
+            SourceForge => Some(&[
+                "{ repo }/files/binaries/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }/download",
+                "{ repo }/files/binaries/v{ version }/{ name }-v{ version }-{ target }.{ archive-format }/download",
+                "{ repo }/files/binaries/v{ version }/{ name }-{ version }-{ target }.{ archive-format }/download",
+                "{ repo }/files/binaries/v{ version }/{ name }-{ target }.{ archive-format }/download",
+            ]),
+            Unknown  => None,
+        }
+    }
+}

crates/lib/src/helpers/remote.rs

@@ -42,6 +42,19 @@ pub async fn remote_exists(
     Ok(req.status().is_success())
 }
 
+pub async fn get_redirected_final_url(client: &Client, url: Url) -> Result<Url, BinstallError> {
+    let method = Method::HEAD;
+
+    let req = client
+        .request(method.clone(), url.clone())
+        .send()
+        .await
+        .and_then(Response::error_for_status)
+        .map_err(|err| BinstallError::Http { method, url, err })?;
+
+    Ok(req.url().clone())
+}
+
 pub(crate) async fn create_request(
     client: &Client,
     url: Url,

crates/lib/src/manifests/cargo_toml_binstall.rs

@@ -11,10 +11,6 @@ pub use package_formats::*;
 
 mod package_formats;
 
-/// Default package path template (may be overridden in package Cargo.toml)
-pub const DEFAULT_PKG_URL: &str =
-    "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ archive-format }";
-
 /// Default binary name template (may be overridden in package Cargo.toml)
 pub const DEFAULT_BIN_DIR: &str = "{ name }-{ target }-v{ version }/{ bin }{ binary-ext }";
 
@@ -34,7 +30,7 @@ pub struct Meta {
 #[serde(rename_all = "kebab-case", default)]
 pub struct PkgMeta {
     /// URL template for package downloads
-    pub pkg_url: String,
+    pub pkg_url: Option<String>,
 
     /// Format for package downloads
     pub pkg_fmt: Option<PkgFmt>,
@@ -52,7 +48,7 @@ pub struct PkgMeta {
 impl Default for PkgMeta {
     fn default() -> Self {
         Self {
-            pkg_url: DEFAULT_PKG_URL.to_string(),
+            pkg_url: None,
             pkg_fmt: None,
             bin_dir: DEFAULT_BIN_DIR.to_string(),
             pub_key: None,
@@ -75,7 +71,7 @@ impl PkgMeta {
     /// Merge configuration overrides into object
     pub fn merge(&mut self, pkg_override: &PkgOverride) {
         if let Some(o) = &pkg_override.pkg_url {
-            self.pkg_url = o.clone();
+            self.pkg_url = Some(o.clone());
         }
         if let Some(o) = &pkg_override.pkg_fmt {
             self.pkg_fmt = Some(*o);

crates/lib/tests/parse-meta.rs

@@ -15,7 +15,7 @@ fn parse_meta() {
     assert_eq!(&package.name, "cargo-binstall-test");
 
     assert_eq!(
-        &meta.pkg_url,
+        meta.pkg_url.as_deref().unwrap(),
         "{ repo }/releases/download/v{ version }/{ name }-{ target }.{ archive-format }"
     );