feat: Support install directly from git repo (#1162)

Fixed #3

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/binstalk/Cargo.toml

@@ -18,6 +18,8 @@ command-group = { version = "2.1.0", features = ["with-tokio"] }
 compact_str = { version = "0.7.0", features = ["serde"] }
 detect-targets = { version = "0.1.7", path = "../detect-targets" }
 either = "1.8.1"
+gix = { version = "0.47.0", features = ["blocking-http-transport-reqwest-rust-tls"], optional = true }
+glob = "0.3.1"
 home = "0.5.5"
 itertools = "0.11.0"
 jobslot = { version = "0.2.11", features = ["tokio"] }
@@ -43,7 +45,9 @@ xz2 = "0.1.7"
 windows = { version = "0.48.0", features = ["Win32_Storage_FileSystem", "Win32_Foundation"] }
 
 [features]
-default = ["static", "rustls"]
+default = ["static", "rustls", "git"]
+
+git = ["dep:gix"]
 
 static = ["binstalk-downloader/static"]
 pkg-config = ["binstalk-downloader/pkg-config"]
@@ -57,3 +61,6 @@ trust-dns = ["binstalk-downloader/trust-dns"]
 
 zstd-thin = ["binstalk-downloader/zstd-thin"]
 cross-lang-fat-lto = ["binstalk-downloader/cross-lang-fat-lto"]
+
+[package.metadata.docs.rs]
+rustdoc-args = ["--cfg", "docsrs"]

crates/binstalk/src/errors.rs

@@ -15,6 +15,8 @@ use thiserror::Error;
 use tokio::task;
 use tracing::{error, warn};
 
+use crate::helpers::cargo_toml_workspace::LoadManifestFromWSError;
+
 #[derive(Debug, Error)]
 #[error("crates.io API error for {crate_name}: {err}")]
 pub struct CratesIoApiError {
@@ -323,6 +325,23 @@ pub enum BinstallError {
     #[diagnostic(severity(error), code(binstall::target_triple_parse_error))]
     TargetTripleParseError(#[source] Box<TargetTripleParseError>),
 
+    /// Failed to shallow clone git repository
+    ///
+    /// - Code: `binstall::git`
+    /// - Exit: 98
+    #[cfg(feature = "git")]
+    #[error("Failed to shallow clone git repository: {0}")]
+    #[diagnostic(severity(error), code(binstall::git))]
+    GitError(#[from] crate::helpers::git::GitError),
+
+    /// Failed to load manifest from workspace
+    ///
+    /// - Code: `binstall::load_manifest_from_workspace`
+    /// - Exit: 99
+    #[error(transparent)]
+    #[diagnostic(severity(error), code(binstall::load_manifest_from_workspace))]
+    LoadManifestFromWSError(#[from] Box<LoadManifestFromWSError>),
+
     /// A wrapped error providing the context of which crate the error is about.
     #[error(transparent)]
     #[diagnostic(transparent)]
@@ -358,6 +377,9 @@ impl BinstallError {
             InvalidPkgFmt(..) => 95,
             GhApiErr(..) => 96,
             TargetTripleParseError(..) => 97,
+            #[cfg(feature = "git")]
+            GitError(_) => 98,
+            LoadManifestFromWSError(_) => 99,
             CrateContext(context) => context.err.exit_number(),
         };
 

crates/binstalk/src/helpers.rs

@@ -1,4 +1,7 @@
+pub mod cargo_toml_workspace;
 pub mod futures_resolver;
+#[cfg(feature = "git")]
+pub mod git;
 pub mod jobserver_client;
 pub mod remote;
 pub mod signal;

crates/binstalk/src/helpers/cargo_toml_workspace.rs

@@ -0,0 +1,263 @@
+use std::{
+    io, mem,
+    path::{Path, PathBuf},
+};
+
+use cargo_toml::{Error as CargoTomlError, Manifest};
+use compact_str::CompactString;
+use glob::PatternError;
+use normalize_path::NormalizePath;
+use thiserror::Error as ThisError;
+use tracing::{debug, instrument, warn};
+
+use crate::{errors::BinstallError, manifests::cargo_toml_binstall::Meta};
+
+/// Load binstall metadata `Cargo.toml` from workspace at the provided path
+///
+/// WARNING: This is a blocking operation.
+///
+///  * `workspace_path` - should be a directory
+pub fn load_manifest_from_workspace(
+    workspace_path: impl AsRef<Path>,
+    crate_name: impl AsRef<str>,
+) -> Result<Manifest<Meta>, BinstallError> {
+    fn inner(workspace_path: &Path, crate_name: &str) -> Result<Manifest<Meta>, BinstallError> {
+        load_manifest_from_workspace_inner(workspace_path, crate_name).map_err(|inner| {
+            Box::new(LoadManifestFromWSError {
+                workspace_path: workspace_path.into(),
+                crate_name: crate_name.into(),
+                inner,
+            })
+            .into()
+        })
+    }
+
+    inner(workspace_path.as_ref(), crate_name.as_ref())
+}
+
+#[derive(Debug, ThisError)]
+#[error("Failed to load {crate_name} from {}: {inner}", workspace_path.display())]
+pub struct LoadManifestFromWSError {
+    workspace_path: Box<Path>,
+    crate_name: CompactString,
+    #[source]
+    inner: LoadManifestFromWSErrorInner,
+}
+
+#[derive(Debug, ThisError)]
+enum LoadManifestFromWSErrorInner {
+    #[error("Invalid pattern in workspace.members or workspace.exclude: {0}")]
+    PatternError(#[from] PatternError),
+
+    #[error("Invalid pattern `{0}`: It must be relative and point within current dir")]
+    InvalidPatternError(CompactString),
+
+    #[error("Failed to parse cargo manifest: {0}")]
+    CargoManifest(#[from] CargoTomlError),
+
+    #[error("I/O error: {0}")]
+    Io(#[from] io::Error),
+
+    #[error("Not found")]
+    NotFound,
+}
+
+#[instrument]
+fn load_manifest_from_workspace_inner(
+    workspace_path: &Path,
+    crate_name: &str,
+) -> Result<Manifest<Meta>, LoadManifestFromWSErrorInner> {
+    debug!(
+        "Loading manifest of crate {crate_name} from workspace: {}",
+        workspace_path.display()
+    );
+
+    let mut workspace_paths = vec![workspace_path.to_owned()];
+
+    while let Some(workspace_path) = workspace_paths.pop() {
+        let p = workspace_path.join("Cargo.toml");
+        let manifest = Manifest::<Meta>::from_path_with_metadata(&p)?;
+
+        let name = manifest.package.as_ref().map(|p| &*p.name);
+        debug!(
+            "Loading from {}, manifest.package.name = {:#?}",
+            p.display(),
+            name
+        );
+
+        if name == Some(crate_name) {
+            return Ok(manifest);
+        }
+
+        if let Some(ws) = manifest.workspace {
+            let excludes = ws.exclude;
+            let members = ws.members;
+
+            if members.is_empty() {
+                continue;
+            }
+
+            let exclude_patterns = excludes
+                .into_iter()
+                .map(|pat| Pattern::new(&pat))
+                .collect::<Result<Vec<_>, _>>()?;
+
+            for member in members {
+                for path in Pattern::new(&member)?.glob_dirs(&workspace_path)? {
+                    if !exclude_patterns
+                        .iter()
+                        .any(|exclude| exclude.matches_with_trailing(&path))
+                    {
+                        workspace_paths.push(workspace_path.join(path));
+                    }
+                }
+            }
+        }
+    }
+
+    Err(LoadManifestFromWSErrorInner::NotFound)
+}
+
+struct Pattern(Vec<glob::Pattern>);
+
+impl Pattern {
+    fn new(pat: &str) -> Result<Self, LoadManifestFromWSErrorInner> {
+        Path::new(pat)
+            .try_normalize()
+            .ok_or_else(|| LoadManifestFromWSErrorInner::InvalidPatternError(pat.into()))?
+            .iter()
+            .map(|c| glob::Pattern::new(c.to_str().unwrap()))
+            .collect::<Result<Vec<_>, _>>()
+            .map_err(Into::into)
+            .map(Self)
+    }
+
+    /// * `glob_path` - path to dir to glob for
+    /// return paths relative to `glob_path`.
+    fn glob_dirs(&self, glob_path: &Path) -> Result<Vec<PathBuf>, LoadManifestFromWSErrorInner> {
+        let mut paths = vec![PathBuf::new()];
+
+        for pattern in &self.0 {
+            if paths.is_empty() {
+                break;
+            }
+
+            for path in mem::take(&mut paths) {
+                let p = glob_path.join(&path);
+                let res = p.read_dir();
+                if res.is_err() && !p.is_dir() {
+                    continue;
+                }
+                drop(p);
+
+                for res in res? {
+                    let entry = res?;
+
+                    let is_dir = entry
+                        .file_type()
+                        .map(|file_type| file_type.is_dir() || file_type.is_symlink())
+                        .unwrap_or(false);
+                    if !is_dir {
+                        continue;
+                    }
+
+                    let filename = entry.file_name();
+                    if filename != "." // Ignore current dir
+                        && filename != ".." // Ignore parent dir
+                        && pattern.matches(&filename.to_string_lossy())
+                    {
+                        paths.push(path.join(filename));
+                    }
+                }
+            }
+        }
+
+        Ok(paths)
+    }
+
+    /// Return `true` if `path` matches the pattern.
+    /// It will still return `true` even if there are some trailing components.
+    fn matches_with_trailing(&self, path: &Path) -> bool {
+        let mut iter = path.iter().map(|os_str| os_str.to_string_lossy());
+        for pattern in &self.0 {
+            match iter.next() {
+                Some(s) if pattern.matches(&s) => (),
+                _ => return false,
+            }
+        }
+        true
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::fs::create_dir_all as mkdir;
+
+    use tempfile::TempDir;
+
+    use super::*;
+
+    #[test]
+    fn test_glob_dirs() {
+        let pattern = Pattern::new("*/*/q/*").unwrap();
+        let tempdir = TempDir::new().unwrap();
+
+        mkdir(tempdir.as_ref().join("a/b/c/efe")).unwrap();
+        mkdir(tempdir.as_ref().join("a/b/q/ww")).unwrap();
+        mkdir(tempdir.as_ref().join("d/233/q/d")).unwrap();
+
+        let mut paths = pattern.glob_dirs(tempdir.as_ref()).unwrap();
+        paths.sort_unstable();
+        assert_eq!(
+            paths,
+            vec![PathBuf::from("a/b/q/ww"), PathBuf::from("d/233/q/d")]
+        );
+    }
+
+    #[test]
+    fn test_matches_with_trailing() {
+        let pattern = Pattern::new("*/*/q/*").unwrap();
+
+        assert!(pattern.matches_with_trailing(Path::new("a/b/q/d/")));
+        assert!(pattern.matches_with_trailing(Path::new("a/b/q/d")));
+        assert!(pattern.matches_with_trailing(Path::new("a/b/q/d/234")));
+        assert!(pattern.matches_with_trailing(Path::new("a/234/q/d/234")));
+
+        assert!(!pattern.matches_with_trailing(Path::new("")));
+        assert!(!pattern.matches_with_trailing(Path::new("a/")));
+        assert!(!pattern.matches_with_trailing(Path::new("a/234")));
+        assert!(!pattern.matches_with_trailing(Path::new("a/234/q")));
+    }
+
+    #[test]
+    fn test_load() {
+        let p = Path::new(env!("CARGO_MANIFEST_DIR"))
+            .parent()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .join("e2e-tests/manifests/workspace");
+
+        let manifest = load_manifest_from_workspace(&p, "cargo-binstall").unwrap();
+        let package = manifest.package.unwrap();
+        assert_eq!(package.name, "cargo-binstall");
+        assert_eq!(package.version.as_ref().unwrap(), "0.12.0");
+        assert_eq!(manifest.bin.len(), 1);
+        assert_eq!(manifest.bin[0].name.as_deref().unwrap(), "cargo-binstall");
+        assert_eq!(manifest.bin[0].path.as_deref().unwrap(), "src/main.rs");
+
+        let err = load_manifest_from_workspace_inner(&p, "cargo-binstall2").unwrap_err();
+        assert!(
+            matches!(err, LoadManifestFromWSErrorInner::NotFound),
+            "{:#?}",
+            err
+        );
+
+        let manifest = load_manifest_from_workspace(&p, "cargo-watch").unwrap();
+        let package = manifest.package.unwrap();
+        assert_eq!(package.name, "cargo-watch");
+        assert_eq!(package.version.as_ref().unwrap(), "8.4.0");
+        assert_eq!(manifest.bin.len(), 1);
+        assert_eq!(manifest.bin[0].name.as_deref().unwrap(), "cargo-watch");
+    }
+}

crates/binstalk/src/helpers/git.rs

@@ -0,0 +1,89 @@
+use std::{num::NonZeroU32, path::Path, str::FromStr, sync::atomic::AtomicBool};
+
+use compact_str::CompactString;
+use gix::{clone, create, open, remote, Url};
+use thiserror::Error as ThisError;
+use tracing::debug;
+
+mod progress_tracing;
+use progress_tracing::TracingProgress;
+
+#[derive(Debug, ThisError)]
+#[non_exhaustive]
+pub enum GitError {
+    #[error("Failed to prepare for fetch: {0}")]
+    PrepareFetchError(#[source] Box<clone::Error>),
+
+    #[error("Failed to fetch: {0}")]
+    FetchError(#[source] Box<clone::fetch::Error>),
+
+    #[error("Failed to checkout: {0}")]
+    CheckOutError(#[source] Box<clone::checkout::main_worktree::Error>),
+}
+
+impl From<clone::Error> for GitError {
+    fn from(e: clone::Error) -> Self {
+        Self::PrepareFetchError(Box::new(e))
+    }
+}
+
+impl From<clone::fetch::Error> for GitError {
+    fn from(e: clone::fetch::Error) -> Self {
+        Self::FetchError(Box::new(e))
+    }
+}
+
+impl From<clone::checkout::main_worktree::Error> for GitError {
+    fn from(e: clone::checkout::main_worktree::Error) -> Self {
+        Self::CheckOutError(Box::new(e))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct GitUrl(Url);
+
+impl FromStr for GitUrl {
+    type Err = gix::url::parse::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Url::try_from(s).map(Self)
+    }
+}
+
+#[derive(Debug)]
+pub struct Repository(gix::Repository);
+
+impl Repository {
+    /// WARNING: This is a blocking operation, if you want to use it in
+    /// async context then you must wrap the call in [`tokio::task::spawn_blocking`].
+    ///
+    /// WARNING: This function must be called after tokio runtime is initialized.
+    pub fn shallow_clone(url: GitUrl, path: &Path) -> Result<Self, GitError> {
+        let url_bstr = url.0.to_bstring();
+        let url_str = String::from_utf8_lossy(&url_bstr);
+
+        debug!("Shallow cloning {url_str} to {}", path.display());
+
+        let mut progress = TracingProgress::new(CompactString::new("Cloning"));
+
+        Ok(Self(
+            clone::PrepareFetch::new(
+                url.0,
+                path,
+                create::Kind::WithWorktree,
+                create::Options {
+                    destination_must_be_empty: true,
+                    ..Default::default()
+                },
+                open::Options::isolated(),
+            )?
+            .with_shallow(remote::fetch::Shallow::DepthAtRemote(
+                NonZeroU32::new(1).unwrap(),
+            ))
+            .fetch_then_checkout(&mut progress, &AtomicBool::new(false))?
+            .0
+            .main_worktree(&mut progress, &AtomicBool::new(false))?
+            .0,
+        ))
+    }
+}

crates/binstalk/src/helpers/git/progress_tracing.rs

@@ -0,0 +1,144 @@
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+use compact_str::{format_compact, CompactString};
+use gix::progress::{
+    prodash::messages::MessageLevel, Id, Progress, Step, StepShared, Unit, UNKNOWN,
+};
+use tokio::time;
+use tracing::{error, info};
+
+pub(super) struct TracingProgress {
+    name: CompactString,
+    id: Id,
+    max: Option<usize>,
+    unit: Option<Unit>,
+    step: usize,
+    trigger: Arc<AtomicBool>,
+}
+
+const EMIT_LOG_EVERY_S: f32 = 0.5;
+const SEP: &str = "::";
+
+impl TracingProgress {
+    /// Create a new instanCompactce from `name`.
+    pub fn new(name: CompactString) -> Self {
+        let trigger = Arc::new(AtomicBool::new(true));
+        tokio::spawn({
+            let mut interval = time::interval(Duration::from_secs_f32(EMIT_LOG_EVERY_S));
+            interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
+
+            let trigger = Arc::clone(&trigger);
+            async move {
+                while Arc::strong_count(&trigger) > 1 {
+                    trigger.store(true, Ordering::Relaxed);
+
+                    interval.tick().await;
+                }
+            }
+        });
+        Self {
+            name,
+            id: UNKNOWN,
+            max: None,
+            step: 0,
+            unit: None,
+            trigger,
+        }
+    }
+}
+
+impl Progress for TracingProgress {
+    type SubProgress = TracingProgress;
+
+    fn add_child(&mut self, name: impl Into<String>) -> Self::SubProgress {
+        self.add_child_with_id(name, UNKNOWN)
+    }
+
+    fn add_child_with_id(&mut self, name: impl Into<String>, id: Id) -> Self::SubProgress {
+        Self {
+            name: format_compact!("{}{}{}", self.name, SEP, Into::<String>::into(name)),
+            id,
+            step: 0,
+            max: None,
+            unit: None,
+            trigger: Arc::clone(&self.trigger),
+        }
+    }
+
+    fn init(&mut self, max: Option<usize>, unit: Option<Unit>) {
+        self.max = max;
+        self.unit = unit;
+    }
+
+    fn set(&mut self, step: usize) {
+        self.step = step;
+        if self.trigger.swap(false, Ordering::Relaxed) {
+            match (self.max, &self.unit) {
+                (max, Some(unit)) => {
+                    info!("{} → {}", self.name, unit.display(step, max, None))
+                }
+                (Some(max), None) => info!("{} → {} / {}", self.name, step, max),
+                (None, None) => info!("{} → {}", self.name, step),
+            }
+        }
+    }
+
+    fn unit(&self) -> Option<Unit> {
+        self.unit.clone()
+    }
+
+    fn max(&self) -> Option<usize> {
+        self.max
+    }
+
+    fn set_max(&mut self, max: Option<Step>) -> Option<Step> {
+        let prev = self.max;
+        self.max = max;
+        prev
+    }
+
+    fn step(&self) -> usize {
+        self.step
+    }
+
+    fn inc_by(&mut self, step: usize) {
+        self.set(self.step + step)
+    }
+
+    fn set_name(&mut self, name: impl Into<String>) {
+        let name = name.into();
+        self.name = self
+            .name
+            .split("::")
+            .next()
+            .map(|parent| format_compact!("{}{}{}", parent.to_owned(), SEP, name))
+            .unwrap_or_else(|| name.into());
+    }
+
+    fn name(&self) -> Option<String> {
+        self.name.split(SEP).nth(1).map(ToOwned::to_owned)
+    }
+
+    fn id(&self) -> Id {
+        self.id
+    }
+
+    fn message(&self, level: MessageLevel, message: impl Into<String>) {
+        let message: String = message.into();
+        match level {
+            MessageLevel::Info => info!("ℹ{} → {}", self.name, message),
+            MessageLevel::Failure => error!("𐄂{} → {}", self.name, message),
+            MessageLevel::Success => info!("✓{} → {}", self.name, message),
+        }
+    }
+
+    fn counter(&self) -> Option<StepShared> {
+        None
+    }
+}

crates/binstalk/src/lib.rs

@@ -1,3 +1,5 @@
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+
 pub mod bins;
 pub mod drivers;
 pub mod errors;

crates/binstalk/src/ops.rs

@@ -10,7 +10,9 @@ use tokio::{
 
 use crate::{
     fetchers::{Data, Fetcher, TargetData},
-    helpers::{gh_api_client::GhApiClient, jobserver_client::LazyJobserverClient, remote::Client},
+    helpers::{
+        self, gh_api_client::GhApiClient, jobserver_client::LazyJobserverClient, remote::Client,
+    },
     manifests::cargo_toml_binstall::PkgOverride,
     DesiredTargets,
 };
@@ -19,6 +21,13 @@ pub mod resolve;
 
 pub type Resolver = fn(Client, GhApiClient, Arc<Data>, Arc<TargetData>) -> Arc<dyn Fetcher>;
 
+#[non_exhaustive]
+pub enum CargoTomlFetchOverride {
+    #[cfg(feature = "git")]
+    Git(helpers::git::GitUrl),
+    Path(PathBuf),
+}
+
 pub struct Options {
     pub no_symlinks: bool,
     pub dry_run: bool,
@@ -28,7 +37,7 @@ pub struct Options {
     pub no_track: bool,
 
     pub version_req: Option<VersionReq>,
-    pub manifest_path: Option<PathBuf>,
+    pub cargo_toml_fetch_override: Option<CargoTomlFetchOverride>,
     pub cli_overrides: PkgOverride,
 
     pub desired_targets: DesiredTargets,

crates/binstalk/src/ops/resolve.rs

@@ -13,17 +13,18 @@ use itertools::Itertools;
 use leon::Template;
 use maybe_owned::MaybeOwned;
 use semver::{Version, VersionReq};
-use tokio::task::block_in_place;
+use tempfile::TempDir;
+use tokio::task::{block_in_place, spawn_blocking};
 use tracing::{debug, info, instrument, warn};
 
-use super::Options;
 use crate::{
     bins,
     drivers::fetch_crate_cratesio,
     errors::{BinstallError, VersionParseError},
     fetchers::{Data, Fetcher, TargetData},
-    helpers::{download::ExtractedFiles, remote::Client, target_triple::TargetTriple},
+    helpers::{self, download::ExtractedFiles, remote::Client, target_triple::TargetTriple},
     manifests::cargo_toml_binstall::{Meta, PkgMeta, PkgOverride},
+    ops::{CargoTomlFetchOverride, Options},
 };
 
 mod crate_name;
@@ -359,9 +360,24 @@ impl PackageInfo {
         version_req: &VersionReq,
         client: Client,
     ) -> Result<Option<Self>, BinstallError> {
+        use CargoTomlFetchOverride::*;
+
         // Fetch crate via crates.io, git, or use a local manifest path
-        let manifest = match opts.manifest_path.as_ref() {
-            Some(manifest_path) => load_manifest_path(manifest_path)?,
+        let manifest = match opts.cargo_toml_fetch_override.as_ref() {
+            Some(Path(manifest_path)) => load_manifest_path(manifest_path)?,
+            #[cfg(feature = "git")]
+            Some(Git(git_url)) => {
+                let git_url = git_url.clone();
+                let name = name.clone();
+
+                spawn_blocking(move || {
+                    let dir = TempDir::new()?;
+                    helpers::git::Repository::shallow_clone(git_url, dir.as_ref())?;
+
+                    helpers::cargo_toml_workspace::load_manifest_from_workspace(dir.as_ref(), &name)
+                })
+                .await??
+            }
             None => {
                 Box::pin(fetch_crate_cratesio(
                     client,