Split crates and clean up structure of codebase (#294)

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/lib/src/ops/install.rs

@@ -0,0 +1,190 @@
+use std::{path::PathBuf, process, sync::Arc};
+
+use cargo_toml::Package;
+use compact_str::CompactString;
+use log::{debug, error, info};
+use tokio::{process::Command, task::block_in_place};
+
+use super::{resolve::Resolution, Options};
+use crate::{
+    bins,
+    errors::BinstallError,
+    fetchers::Fetcher,
+    helpers::jobserver_client::LazyJobserverClient,
+    manifests::{
+        cargo_toml_binstall::Meta,
+        crate_info::{CrateInfo, CrateSource},
+    },
+};
+
+pub async fn install(
+    resolution: Resolution,
+    opts: Arc<Options>,
+    jobserver_client: LazyJobserverClient,
+) -> Result<Option<CrateInfo>, BinstallError> {
+    match resolution {
+        Resolution::AlreadyUpToDate => Ok(None),
+        Resolution::Fetch {
+            fetcher,
+            package,
+            name,
+            version_req,
+            bin_path,
+            bin_files,
+        } => {
+            let current_version =
+                package
+                    .version
+                    .parse()
+                    .map_err(|err| BinstallError::VersionParse {
+                        v: package.version,
+                        err,
+                    })?;
+            let target = fetcher.target().into();
+
+            install_from_package(fetcher, opts, bin_path, bin_files)
+                .await
+                .map(|option| {
+                    option.map(|bins| CrateInfo {
+                        name,
+                        version_req,
+                        current_version,
+                        source: CrateSource::cratesio_registry(),
+                        target,
+                        bins,
+                        other: Default::default(),
+                    })
+                })
+        }
+        Resolution::InstallFromSource { package } => {
+            let desired_targets = opts.desired_targets.get().await;
+            let target = desired_targets
+                .first()
+                .ok_or(BinstallError::NoViableTargets)?;
+
+            if !opts.dry_run {
+                install_from_source(package, target, jobserver_client, opts.quiet, opts.force)
+                    .await
+                    .map(|_| None)
+            } else {
+                info!(
+                    "Dry-run: running `cargo install {} --version {} --target {target}`",
+                    package.name, package.version
+                );
+                Ok(None)
+            }
+        }
+    }
+}
+
+async fn install_from_package(
+    fetcher: Arc<dyn Fetcher>,
+    opts: Arc<Options>,
+    bin_path: PathBuf,
+    bin_files: Vec<bins::BinFile>,
+) -> Result<Option<Vec<CompactString>>, BinstallError> {
+    // Download package
+    if opts.dry_run {
+        info!("Dry run, not downloading package");
+    } else {
+        fetcher.fetch_and_extract(&bin_path).await?;
+    }
+
+    #[cfg(incomplete)]
+    {
+        // Fetch and check package signature if available
+        if let Some(pub_key) = meta.as_ref().map(|m| m.pub_key.clone()).flatten() {
+            debug!("Found public key: {pub_key}");
+
+            // Generate signature file URL
+            let mut sig_ctx = ctx.clone();
+            sig_ctx.format = "sig".to_string();
+            let sig_url = sig_ctx.render(&pkg_url)?;
+
+            debug!("Fetching signature file: {sig_url}");
+
+            // Download signature file
+            let sig_path = temp_dir.join(format!("{pkg_name}.sig"));
+            download(&sig_url, &sig_path).await?;
+
+            // TODO: do the signature check
+            unimplemented!()
+        } else {
+            warn!("No public key found, package signature could not be validated");
+        }
+    }
+
+    if opts.dry_run {
+        info!("Dry run, not proceeding");
+        return Ok(None);
+    }
+
+    info!("Installing binaries...");
+    block_in_place(|| {
+        for file in &bin_files {
+            file.install_bin()?;
+        }
+
+        // Generate symlinks
+        if !opts.no_symlinks {
+            for file in &bin_files {
+                file.install_link()?;
+            }
+        }
+
+        Ok(Some(
+            bin_files.into_iter().map(|bin| bin.base_name).collect(),
+        ))
+    })
+}
+
+async fn install_from_source(
+    package: Package<Meta>,
+    target: &str,
+    lazy_jobserver_client: LazyJobserverClient,
+    quiet: bool,
+    force: bool,
+) -> Result<(), BinstallError> {
+    let jobserver_client = lazy_jobserver_client.get().await?;
+
+    debug!(
+        "Running `cargo install {} --version {} --target {target}`",
+        package.name, package.version
+    );
+    let mut command = process::Command::new("cargo");
+    jobserver_client.configure(&mut command);
+
+    let mut cmd = Command::from(command);
+
+    cmd.arg("install")
+        .arg(package.name)
+        .arg("--version")
+        .arg(package.version)
+        .arg("--target")
+        .arg(&*target);
+
+    if quiet {
+        cmd.arg("--quiet");
+    }
+
+    if force {
+        cmd.arg("--force");
+    }
+
+    let command_string = format!("{:?}", cmd);
+
+    let mut child = cmd.spawn()?;
+    debug!("Spawned command pid={:?}", child.id());
+
+    let status = child.wait().await?;
+    if status.success() {
+        info!("Cargo finished successfully");
+        Ok(())
+    } else {
+        error!("Cargo errored! {status:?}");
+        Err(BinstallError::SubProcess {
+            command: command_string,
+            status,
+        })
+    }
+}

crates/lib/src/ops/resolve.rs

@@ -0,0 +1,304 @@
+use std::{
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use cargo_toml::{Manifest, Package, Product};
+use compact_str::{CompactString, ToCompactString};
+use log::{debug, info, warn};
+use reqwest::Client;
+use semver::{Version, VersionReq};
+use tokio::task::block_in_place;
+
+use super::Options;
+use crate::{
+    bins,
+    drivers::fetch_crate_cratesio,
+    errors::BinstallError,
+    fetchers::{Data, Fetcher, GhCrateMeta, MultiFetcher, QuickInstall},
+    manifests::cargo_toml_binstall::{Meta, PkgMeta},
+};
+
+mod crate_name;
+#[doc(inline)]
+pub use crate_name::CrateName;
+mod version_ext;
+#[doc(inline)]
+pub use version_ext::VersionReqExt;
+
+pub enum Resolution {
+    Fetch {
+        fetcher: Arc<dyn Fetcher>,
+        package: Package<Meta>,
+        name: CompactString,
+        version_req: CompactString,
+        bin_path: PathBuf,
+        bin_files: Vec<bins::BinFile>,
+    },
+    InstallFromSource {
+        package: Package<Meta>,
+    },
+    AlreadyUpToDate,
+}
+impl Resolution {
+    fn print(&self, opts: &Options) {
+        match self {
+            Resolution::Fetch {
+                fetcher, bin_files, ..
+            } => {
+                let fetcher_target = fetcher.target();
+                // Prompt user for confirmation
+                debug!(
+                    "Found a binary install source: {} ({fetcher_target})",
+                    fetcher.source_name()
+                );
+
+                if fetcher.is_third_party() {
+                    warn!(
+                        "The package will be downloaded from third-party source {}",
+                        fetcher.source_name()
+                    );
+                } else {
+                    info!(
+                        "The package will be downloaded from {}",
+                        fetcher.source_name()
+                    );
+                }
+
+                info!("This will install the following binaries:");
+                for file in bin_files {
+                    info!("  - {}", file.preview_bin());
+                }
+
+                if !opts.no_symlinks {
+                    info!("And create (or update) the following symlinks:");
+                    for file in bin_files {
+                        info!("  - {}", file.preview_link());
+                    }
+                }
+            }
+            Resolution::InstallFromSource { .. } => {
+                warn!("The package will be installed from source (with cargo)",)
+            }
+            Resolution::AlreadyUpToDate => (),
+        }
+    }
+}
+
+pub async fn resolve(
+    opts: Arc<Options>,
+    crate_name: CrateName,
+    curr_version: Option<Version>,
+    temp_dir: Arc<Path>,
+    install_path: Arc<Path>,
+    client: Client,
+    crates_io_api_client: crates_io_api::AsyncClient,
+) -> Result<Resolution, BinstallError> {
+    let crate_name_name = crate_name.name.clone();
+    resolve_inner(
+        opts,
+        crate_name,
+        curr_version,
+        temp_dir,
+        install_path,
+        client,
+        crates_io_api_client,
+    )
+    .await
+    .map_err(|err| err.crate_context(crate_name_name))
+}
+
+async fn resolve_inner(
+    opts: Arc<Options>,
+    crate_name: CrateName,
+    curr_version: Option<Version>,
+    temp_dir: Arc<Path>,
+    install_path: Arc<Path>,
+    client: Client,
+    crates_io_api_client: crates_io_api::AsyncClient,
+) -> Result<Resolution, BinstallError> {
+    info!("Resolving package: '{}'", crate_name);
+
+    let version_req: VersionReq = match (&crate_name.version_req, &opts.version_req) {
+        (Some(version), None) => version.clone(),
+        (None, Some(version)) => version.clone(),
+        (Some(_), Some(_)) => Err(BinstallError::SuperfluousVersionOption)?,
+        (None, None) => VersionReq::STAR,
+    };
+
+    // Fetch crate via crates.io, git, or use a local manifest path
+    // TODO: work out which of these to do based on `opts.name`
+    // TODO: support git-based fetches (whole repo name rather than just crate name)
+    let manifest = match opts.manifest_path.clone() {
+        Some(manifest_path) => load_manifest_path(manifest_path)?,
+        None => {
+            fetch_crate_cratesio(
+                &client,
+                &crates_io_api_client,
+                &crate_name.name,
+                &version_req,
+            )
+            .await?
+        }
+    };
+
+    let package = manifest.package.unwrap();
+
+    if let Some(curr_version) = curr_version {
+        let new_version =
+            Version::parse(&package.version).map_err(|err| BinstallError::VersionParse {
+                v: package.version.clone(),
+                err,
+            })?;
+
+        if new_version == curr_version {
+            info!(
+                "{} v{curr_version} is already installed, use --force to override",
+                crate_name.name
+            );
+            return Ok(Resolution::AlreadyUpToDate);
+        }
+    }
+
+    let (mut meta, binaries) = (
+        package
+            .metadata
+            .as_ref()
+            .and_then(|m| m.binstall.clone())
+            .unwrap_or_default(),
+        manifest.bin,
+    );
+
+    let mut fetchers = MultiFetcher::default();
+
+    let desired_targets = opts.desired_targets.get().await;
+
+    for target in desired_targets {
+        debug!("Building metadata for target: {target}");
+        let mut target_meta = meta.clone();
+
+        // Merge any overrides
+        if let Some(o) = target_meta.overrides.get(target).cloned() {
+            target_meta.merge(&o);
+        }
+
+        target_meta.merge(&opts.cli_overrides);
+        debug!("Found metadata: {target_meta:?}");
+
+        let fetcher_data = Data {
+            name: package.name.clone(),
+            target: target.clone(),
+            version: package.version.clone(),
+            repo: package.repository.clone(),
+            meta: target_meta,
+        };
+
+        fetchers.add(GhCrateMeta::new(&client, &fetcher_data).await);
+        fetchers.add(QuickInstall::new(&client, &fetcher_data).await);
+    }
+
+    let resolution = match fetchers.first_available().await {
+        Some(fetcher) => {
+            // Build final metadata
+            let fetcher_target = fetcher.target();
+            if let Some(o) = meta.overrides.get(&fetcher_target.to_owned()).cloned() {
+                meta.merge(&o);
+            }
+            meta.merge(&opts.cli_overrides);
+
+            // Generate temporary binary path
+            let bin_path = temp_dir.join(format!("bin-{}", crate_name.name));
+            debug!("Using temporary binary path: {}", bin_path.display());
+
+            let bin_files = collect_bin_files(
+                fetcher.as_ref(),
+                &package,
+                meta,
+                binaries,
+                bin_path.clone(),
+                install_path.to_path_buf(),
+            )?;
+
+            Resolution::Fetch {
+                fetcher,
+                package,
+                name: crate_name.name,
+                version_req: version_req.to_compact_string(),
+                bin_path,
+                bin_files,
+            }
+        }
+        None => Resolution::InstallFromSource { package },
+    };
+
+    resolution.print(&opts);
+
+    Ok(resolution)
+}
+
+fn collect_bin_files(
+    fetcher: &dyn Fetcher,
+    package: &Package<Meta>,
+    mut meta: PkgMeta,
+    binaries: Vec<Product>,
+    bin_path: PathBuf,
+    install_path: PathBuf,
+) -> Result<Vec<bins::BinFile>, BinstallError> {
+    // Update meta
+    if fetcher.source_name() == "QuickInstall" {
+        // TODO: less of a hack?
+        meta.bin_dir = "{ bin }{ binary-ext }".to_string();
+    }
+
+    // Check binaries
+    if binaries.is_empty() {
+        return Err(BinstallError::UnspecifiedBinaries);
+    }
+
+    // List files to be installed
+    // based on those found via Cargo.toml
+    let bin_data = bins::Data {
+        name: package.name.clone(),
+        target: fetcher.target().to_string(),
+        version: package.version.clone(),
+        repo: package.repository.clone(),
+        meta,
+        bin_path,
+        install_path,
+    };
+
+    // Create bin_files
+    let bin_files = binaries
+        .iter()
+        .map(|p| bins::BinFile::from_product(&bin_data, p))
+        .collect::<Result<Vec<_>, BinstallError>>()?;
+
+    Ok(bin_files)
+}
+
+/// Load binstall metadata from the crate `Cargo.toml` at the provided path
+pub fn load_manifest_path<P: AsRef<Path>>(
+    manifest_path: P,
+) -> Result<Manifest<Meta>, BinstallError> {
+    block_in_place(|| {
+        let manifest_path = manifest_path.as_ref();
+        let manifest_path = if manifest_path.is_dir() {
+            manifest_path.join("Cargo.toml")
+        } else if manifest_path.is_file() {
+            manifest_path.into()
+        } else {
+            return Err(BinstallError::CargoManifestPath);
+        };
+
+        debug!(
+            "Reading manifest at local path: {}",
+            manifest_path.display()
+        );
+
+        // Load and parse manifest (this checks file system for binary output names)
+        let manifest = Manifest::<Meta>::from_path_with_metadata(manifest_path)?;
+
+        // Return metadata
+        Ok(manifest)
+    })
+}

crates/lib/src/ops/resolve/crate_name.rs

@@ -0,0 +1,112 @@
+use std::{fmt, str::FromStr};
+
+use compact_str::CompactString;
+use itertools::Itertools;
+use semver::{Error, VersionReq};
+
+use super::version_ext::VersionReqExt;
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct CrateName {
+    pub name: CompactString,
+    pub version_req: Option<VersionReq>,
+}
+
+impl fmt::Display for CrateName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.name)?;
+
+        if let Some(version) = &self.version_req {
+            write!(f, "@{version}")?;
+        }
+
+        Ok(())
+    }
+}
+
+impl FromStr for CrateName {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(if let Some((name, version)) = s.split_once('@') {
+            CrateName {
+                name: name.into(),
+                version_req: Some(VersionReq::parse_from_cli(version)?),
+            }
+        } else {
+            CrateName {
+                name: s.into(),
+                version_req: None,
+            }
+        })
+    }
+}
+
+impl CrateName {
+    pub fn dedup(crate_names: &[Self]) -> impl Iterator<Item = Self> {
+        let mut crate_names = crate_names.to_vec();
+        crate_names.sort_by(|x, y| x.name.cmp(&y.name));
+        crate_names.into_iter().coalesce(|previous, current| {
+            if previous.name == current.name {
+                Ok(current)
+            } else {
+                Err((previous, current))
+            }
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    macro_rules! assert_dedup {
+        ([ $( ( $input_name:expr, $input_version:expr ) ),*  ], [ $( ( $output_name:expr, $output_version:expr ) ),*  ]) => {
+            let input_crate_names = [$( CrateName {
+                name: $input_name.into(),
+                version_req: Some($input_version.parse().unwrap())
+            }, )*];
+
+            let mut output_crate_names: Vec<CrateName> = vec![$( CrateName {
+                name: $output_name.into(), version_req: Some($output_version.parse().unwrap())
+            }, )*];
+            output_crate_names.sort_by(|x, y| x.name.cmp(&y.name));
+
+            let crate_names: Vec<_> = CrateName::dedup(&input_crate_names).collect();
+            assert_eq!(crate_names, output_crate_names);
+        };
+    }
+
+    #[test]
+    fn test_dedup() {
+        // Base case 0: Empty input
+        assert_dedup!([], []);
+
+        // Base case 1: With only one input
+        assert_dedup!([("a", "1")], [("a", "1")]);
+
+        // Base Case 2: Only has duplicate names
+        assert_dedup!([("a", "1"), ("a", "2")], [("a", "2")]);
+
+        // Complex Case 0: Having two crates
+        assert_dedup!(
+            [("a", "10"), ("b", "3"), ("a", "0"), ("b", "0"), ("a", "1")],
+            [("a", "1"), ("b", "0")]
+        );
+
+        // Complex Case 1: Having three crates
+        assert_dedup!(
+            [
+                ("d", "1.1"),
+                ("a", "10"),
+                ("b", "3"),
+                ("d", "230"),
+                ("a", "0"),
+                ("b", "0"),
+                ("a", "1"),
+                ("d", "23")
+            ],
+            [("a", "1"), ("b", "0"), ("d", "23")]
+        );
+    }
+}

crates/lib/src/ops/resolve/version_ext.rs

@@ -0,0 +1,99 @@
+use compact_str::format_compact;
+use semver::{Prerelease, Version, VersionReq};
+
+/// Extension trait for [`VersionReq`].
+pub trait VersionReqExt {
+    /// Return `true` if `self.matches(version)` returns `true`
+    /// and the `version` is the latest one acceptable by `self`.
+    fn is_latest_compatible(&self, version: &Version) -> bool;
+
+    /// Parse from CLI option.
+    ///
+    /// Notably, a bare version is treated as if preceded by `=`, not by `^` as in Cargo.toml
+    /// dependencies.
+    fn parse_from_cli(str: &str) -> Result<Self, semver::Error>
+    where
+        Self: Sized;
+}
+
+impl VersionReqExt for VersionReq {
+    fn is_latest_compatible(&self, version: &Version) -> bool {
+        if !self.matches(version) {
+            return false;
+        }
+
+        // Test if bumping patch will be accepted
+        let bumped_version = Version::new(version.major, version.minor, version.patch + 1);
+
+        if self.matches(&bumped_version) {
+            return false;
+        }
+
+        // Test if bumping prerelease will be accepted if version has one.
+        let pre = &version.pre;
+        if !pre.is_empty() {
+            // Bump pre by appending random number to the end.
+            let bumped_pre = format_compact!("{}.1", pre.as_str());
+
+            let bumped_version = Version {
+                major: version.major,
+                minor: version.minor,
+                patch: version.patch,
+                pre: Prerelease::new(&bumped_pre).unwrap(),
+                build: Default::default(),
+            };
+
+            if self.matches(&bumped_version) {
+                return false;
+            }
+        }
+
+        true
+    }
+
+    fn parse_from_cli(version: &str) -> Result<Self, semver::Error> {
+        if version
+            .chars()
+            .next()
+            .map(|ch| ch.is_ascii_digit())
+            .unwrap_or(false)
+        {
+            format_compact!("={version}").parse()
+        } else {
+            version.parse()
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test() {
+        // Test star
+        assert!(!VersionReq::STAR.is_latest_compatible(&Version::parse("0.0.1").unwrap()));
+        assert!(!VersionReq::STAR.is_latest_compatible(&Version::parse("0.1.1").unwrap()));
+        assert!(!VersionReq::STAR.is_latest_compatible(&Version::parse("0.1.1-alpha").unwrap()));
+
+        // Test ^x.y.z
+        assert!(!VersionReq::parse("^0.1")
+            .unwrap()
+            .is_latest_compatible(&Version::parse("0.1.99").unwrap()));
+
+        // Test =x.y.z
+        assert!(VersionReq::parse("=0.1.0")
+            .unwrap()
+            .is_latest_compatible(&Version::parse("0.1.0").unwrap()));
+
+        // Test =x.y.z-alpha
+        assert!(VersionReq::parse("=0.1.0-alpha")
+            .unwrap()
+            .is_latest_compatible(&Version::parse("0.1.0-alpha").unwrap()));
+
+        // Test >=x.y.z-alpha
+        assert!(!VersionReq::parse(">=0.1.0-alpha")
+            .unwrap()
+            .is_latest_compatible(&Version::parse("0.1.0-alpha").unwrap()));
+    }
+}