Refactor: Extract new crate binstalk-registry (#1289)

To speedup codegen of `binstalk` and enable it to be reused.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/binstalk-registry/src/common.rs

@@ -0,0 +1,192 @@
+use std::borrow::Cow;
+
+use base16::{decode as decode_base16, encode_lower as encode_base16};
+use binstalk_downloader::{
+    bytes::Bytes,
+    download::{DataVerifier, Download},
+    remote::{Client, Url},
+};
+use binstalk_types::cargo_toml_binstall::{Meta, TarBasedFmt};
+use cargo_toml_workspace::cargo_toml::Manifest;
+use compact_str::{format_compact, CompactString, ToCompactString};
+use leon::{Template, Values};
+use semver::{Version, VersionReq};
+use serde::Deserialize;
+use serde_json::Error as JsonError;
+use sha2::{Digest, Sha256};
+use tracing::debug;
+
+use crate::{visitor::ManifestVisitor, RegistryError};
+
+#[derive(Deserialize)]
+pub(super) struct RegistryConfig {
+    pub(super) dl: CompactString,
+}
+
+struct Sha256Digest(Sha256);
+
+impl Default for Sha256Digest {
+    fn default() -> Self {
+        Sha256Digest(Sha256::new())
+    }
+}
+
+impl DataVerifier for Sha256Digest {
+    fn update(&mut self, data: &Bytes) {
+        self.0.update(data);
+    }
+}
+
+pub(super) async fn parse_manifest(
+    client: Client,
+    crate_name: &str,
+    crate_url: Url,
+    MatchedVersion { version, cksum }: MatchedVersion,
+) -> Result<Manifest<Meta>, RegistryError> {
+    debug!("Fetching crate from: {crate_url} and extracting Cargo.toml from it");
+
+    let mut manifest_visitor = ManifestVisitor::new(format!("{crate_name}-{version}").into());
+
+    let checksum = decode_base16(cksum.as_bytes()).map_err(RegistryError::from)?;
+    let mut sha256_digest = Sha256Digest::default();
+
+    Download::new_with_data_verifier(client, crate_url, &mut sha256_digest)
+        .and_visit_tar(TarBasedFmt::Tgz, &mut manifest_visitor)
+        .await?;
+
+    let digest_checksum = sha256_digest.0.finalize();
+
+    if digest_checksum.as_slice() != checksum.as_slice() {
+        Err(RegistryError::UnmatchedChecksum {
+            expected: cksum.into(),
+            actual: encode_base16(digest_checksum.as_slice()).into(),
+        })
+    } else {
+        manifest_visitor.load_manifest()
+    }
+}
+
+/// Return components of crate prefix
+pub(super) fn crate_prefix_components(
+    crate_name: &str,
+) -> Result<(CompactString, Option<CompactString>), RegistryError> {
+    let mut chars = crate_name.chars();
+
+    match (chars.next(), chars.next(), chars.next(), chars.next()) {
+        (None, None, None, None) => Err(RegistryError::NotFound(crate_name.into())),
+        (Some(_), None, None, None) => Ok((CompactString::new("1"), None)),
+        (Some(_), Some(_), None, None) => Ok((CompactString::new("2"), None)),
+        (Some(ch), Some(_), Some(_), None) => Ok((
+            CompactString::new("3"),
+            Some(ch.to_lowercase().to_compact_string()),
+        )),
+        (Some(a), Some(b), Some(c), Some(d)) => Ok((
+            format_compact!("{}{}", a.to_lowercase(), b.to_lowercase()),
+            Some(format_compact!("{}{}", c.to_lowercase(), d.to_lowercase())),
+        )),
+        _ => unreachable!(),
+    }
+}
+
+pub(super) fn render_dl_template(
+    dl_template: &str,
+    crate_name: &str,
+    (c1, c2): &(CompactString, Option<CompactString>),
+    MatchedVersion { version, cksum }: &MatchedVersion,
+) -> Result<String, RegistryError> {
+    let template = Template::parse(dl_template)?;
+    if template.keys().next().is_some() {
+        let mut crate_prefix = c1.clone();
+        if let Some(c2) = c2 {
+            crate_prefix.push('/');
+            crate_prefix.push_str(c2);
+        }
+
+        struct Context<'a> {
+            crate_name: &'a str,
+            crate_prefix: CompactString,
+            crate_lowerprefix: String,
+            version: &'a str,
+            cksum: &'a str,
+        }
+        impl Values for Context<'_> {
+            fn get_value(&self, key: &str) -> Option<Cow<'_, str>> {
+                match key {
+                    "crate" => Some(Cow::Borrowed(self.crate_name)),
+                    "version" => Some(Cow::Borrowed(self.version)),
+                    "prefix" => Some(Cow::Borrowed(&self.crate_prefix)),
+                    "lowerprefix" => Some(Cow::Borrowed(&self.crate_lowerprefix)),
+                    "sha256-checksum" => Some(Cow::Borrowed(self.cksum)),
+                    _ => None,
+                }
+            }
+        }
+        Ok(template.render(&Context {
+            crate_name,
+            crate_lowerprefix: crate_prefix.to_lowercase(),
+            crate_prefix,
+            version,
+            cksum,
+        })?)
+    } else {
+        Ok(format!("{dl_template}/{crate_name}/{version}/download"))
+    }
+}
+
+#[derive(Deserialize)]
+pub(super) struct RegistryIndexEntry {
+    vers: CompactString,
+    yanked: bool,
+    cksum: String,
+}
+
+pub(super) struct MatchedVersion {
+    pub(super) version: CompactString,
+    /// sha256 checksum encoded in base16
+    pub(super) cksum: String,
+}
+
+impl MatchedVersion {
+    pub(super) fn find(
+        it: &mut dyn Iterator<Item = Result<RegistryIndexEntry, JsonError>>,
+        version_req: &VersionReq,
+    ) -> Result<Self, RegistryError> {
+        let mut ret = Option::<(Self, Version)>::None;
+
+        for res in it {
+            let entry = res.map_err(RegistryError::from)?;
+
+            if entry.yanked {
+                continue;
+            }
+
+            let num = entry.vers;
+
+            // Parse out version
+            let Ok(ver) = Version::parse(&num) else { continue };
+
+            // Filter by version match
+            if !version_req.matches(&ver) {
+                continue;
+            }
+
+            let matched = Self {
+                version: num,
+                cksum: entry.cksum,
+            };
+
+            if let Some((_, max_ver)) = &ret {
+                if ver > *max_ver {
+                    ret = Some((matched, ver));
+                }
+            } else {
+                ret = Some((matched, ver));
+            }
+        }
+
+        ret.map(|(num, _)| num)
+            .ok_or_else(|| RegistryError::VersionMismatch {
+                req: version_req.clone(),
+            })
+    }
+}

crates/binstalk-registry/src/crates_io_registry.rs

@@ -0,0 +1,196 @@
+use binstalk_downloader::remote::{Client, Error as RemoteError, Url};
+use binstalk_types::cargo_toml_binstall::Meta;
+use cargo_toml_workspace::cargo_toml::Manifest;
+use compact_str::{CompactString, ToCompactString};
+use semver::{Comparator, Op as ComparatorOp, Version as SemVersion, VersionReq};
+use serde::Deserialize;
+use tokio::{
+    sync::Mutex,
+    time::{interval, Duration, Interval, MissedTickBehavior},
+};
+use tracing::debug;
+
+use crate::{parse_manifest, MatchedVersion, RegistryError};
+
+#[derive(Debug)]
+pub struct CratesIoRateLimit(Mutex<Interval>);
+
+impl Default for CratesIoRateLimit {
+    fn default() -> Self {
+        let mut interval = interval(Duration::from_secs(1));
+        // If somehow one tick is delayed, then next tick should be at least
+        // 1s later than the current tick.
+        //
+        // Other MissedTickBehavior including Burst (default), which will
+        // tick as fast as possible to catch up, and Skip, which will
+        // skip the current tick for the next one.
+        //
+        // Both Burst and Skip is not the expected behavior for rate limit:
+        // ticking as fast as possible would violate crates.io crawler
+        // policy, and skipping the current one will slow down the resolution
+        // process.
+        interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
+        Self(Mutex::new(interval))
+    }
+}
+
+impl CratesIoRateLimit {
+    pub(super) async fn tick(&self) {
+        self.0.lock().await.tick().await;
+    }
+}
+
+/// Return `Some(checksum)` if the version is not yanked, otherwise `None`.
+async fn is_crate_yanked(client: &Client, url: Url) -> Result<Option<String>, RemoteError> {
+    #[derive(Deserialize)]
+    struct CrateInfo {
+        version: Inner,
+    }
+
+    #[derive(Deserialize)]
+    struct Inner {
+        yanked: bool,
+        checksum: String,
+    }
+
+    // Fetch / update index
+    debug!("Looking up crate information");
+
+    let info: CrateInfo = client.get(url).send(true).await?.json().await?;
+    let version = info.version;
+
+    Ok((!version.yanked).then_some(version.checksum))
+}
+
+async fn fetch_crate_cratesio_version_matched(
+    client: &Client,
+    url: Url,
+    version_req: &VersionReq,
+) -> Result<Option<(CompactString, String)>, RemoteError> {
+    #[derive(Deserialize)]
+    struct CrateInfo {
+        #[serde(rename = "crate")]
+        inner: CrateInfoInner,
+
+        versions: Vec<Version>,
+    }
+
+    #[derive(Deserialize)]
+    struct CrateInfoInner {
+        max_stable_version: CompactString,
+    }
+
+    #[derive(Deserialize)]
+    struct Version {
+        num: CompactString,
+        yanked: bool,
+        checksum: String,
+    }
+
+    // Fetch / update index
+    debug!("Looking up crate information");
+
+    let crate_info: CrateInfo = client.get(url).send(true).await?.json().await?;
+
+    let version_with_checksum = if version_req == &VersionReq::STAR {
+        let version = crate_info.inner.max_stable_version;
+        crate_info
+            .versions
+            .into_iter()
+            .find_map(|v| (v.num.as_str() == version.as_str()).then_some(v.checksum))
+            .map(|checksum| (version, checksum))
+    } else {
+        crate_info
+            .versions
+            .into_iter()
+            .filter_map(|item| {
+                if !item.yanked {
+                    // Remove leading `v` for git tags
+                    let num = if let Some(num) = item.num.strip_prefix('v') {
+                        num.into()
+                    } else {
+                        item.num
+                    };
+
+                    // Parse out version
+                    let ver = semver::Version::parse(&num).ok()?;
+
+                    // Filter by version match
+                    version_req
+                        .matches(&ver)
+                        .then_some((num, ver, item.checksum))
+                } else {
+                    None
+                }
+            })
+            // Return highest version
+            .max_by(
+                |(_ver_str_x, ver_x, _checksum_x), (_ver_str_y, ver_y, _checksum_y)| {
+                    ver_x.cmp(ver_y)
+                },
+            )
+            .map(|(ver_str, _, checksum)| (ver_str, checksum))
+    };
+
+    Ok(version_with_checksum)
+}
+
+/// Find the crate by name, get its latest stable version matches `version_req`,
+/// retrieve its Cargo.toml and infer all its bins.
+pub async fn fetch_crate_cratesio(
+    client: Client,
+    name: &str,
+    version_req: &VersionReq,
+    crates_io_rate_limit: &CratesIoRateLimit,
+) -> Result<Manifest<Meta>, RegistryError> {
+    // Wait until we can make another request to crates.io
+    crates_io_rate_limit.tick().await;
+
+    let url = Url::parse(&format!("https://crates.io/api/v1/crates/{name}"))?;
+
+    let (version, cksum) = match version_req.comparators.as_slice() {
+        [Comparator {
+            op: ComparatorOp::Exact,
+            major,
+            minor: Some(minor),
+            patch: Some(patch),
+            pre,
+        }] => {
+            let version = SemVersion {
+                major: *major,
+                minor: *minor,
+                patch: *patch,
+                pre: pre.clone(),
+                build: Default::default(),
+            }
+            .to_compact_string();
+
+            let mut url = url.clone();
+            url.path_segments_mut().unwrap().push(&version);
+
+            is_crate_yanked(&client, url)
+                .await
+                .map(|ret| ret.map(|checksum| (version, checksum)))
+        }
+        _ => fetch_crate_cratesio_version_matched(&client, url.clone(), version_req).await,
+    }
+    .map_err(|e| match e {
+        RemoteError::Http(e) if e.is_status() => RegistryError::NotFound(name.into()),
+        e => e.into(),
+    })?
+    .ok_or_else(|| RegistryError::VersionMismatch {
+        req: version_req.clone(),
+    })?;
+
+    debug!("Found information for crate version: '{version}'");
+
+    // Download crate to temporary dir (crates.io or git?)
+    let mut crate_url = url;
+    crate_url
+        .path_segments_mut()
+        .unwrap()
+        .push(&version)
+        .push("download");
+
+    parse_manifest(client, name, crate_url, MatchedVersion { version, cksum }).await
+}

crates/binstalk-registry/src/git_registry.rs

@@ -0,0 +1,145 @@
+use std::{io, path::PathBuf, sync::Arc};
+
+use binstalk_downloader::{
+    git::{GitCancellationToken, GitUrl, Repository},
+    remote::Client,
+};
+use binstalk_types::cargo_toml_binstall::Meta;
+use cargo_toml_workspace::cargo_toml::Manifest;
+use compact_str::{CompactString, ToCompactString};
+use once_cell::sync::OnceCell;
+use semver::VersionReq;
+use serde_json::{from_slice as json_from_slice, Deserializer as JsonDeserializer};
+use tempfile::TempDir;
+use tokio::task::spawn_blocking;
+use url::Url;
+
+use crate::{
+    crate_prefix_components, parse_manifest, render_dl_template, MatchedVersion, RegistryConfig,
+    RegistryError,
+};
+
+#[derive(Debug)]
+struct GitIndex {
+    _tempdir: TempDir,
+    repo: Repository,
+    dl_template: CompactString,
+}
+
+impl GitIndex {
+    fn new(url: GitUrl, cancellation_token: GitCancellationToken) -> Result<Self, RegistryError> {
+        let tempdir = TempDir::new()?;
+
+        let repo = Repository::shallow_clone_bare(
+            url.clone(),
+            tempdir.as_ref(),
+            Some(cancellation_token),
+        )?;
+
+        let config: RegistryConfig = {
+            let config = repo
+                .get_head_commit_entry_data_by_path("config.json")?
+                .ok_or_else(|| {
+                    io::Error::new(
+                        io::ErrorKind::NotFound,
+                        format!("config.json not found in repository `{url}`"),
+                    )
+                })?;
+
+            json_from_slice(&config).map_err(RegistryError::from)?
+        };
+
+        Ok(Self {
+            _tempdir: tempdir,
+            repo,
+            dl_template: config.dl,
+        })
+    }
+}
+
+#[derive(Debug)]
+struct GitRegistryInner {
+    url: GitUrl,
+    git_index: OnceCell<GitIndex>,
+}
+
+#[derive(Clone, Debug)]
+pub struct GitRegistry(Arc<GitRegistryInner>);
+
+impl GitRegistry {
+    pub fn new(url: GitUrl) -> Self {
+        Self(Arc::new(GitRegistryInner {
+            url,
+            git_index: Default::default(),
+        }))
+    }
+
+    /// WARNING: This is a blocking operation.
+    fn find_crate_matched_ver(
+        repo: &Repository,
+        crate_name: &str,
+        (c1, c2): &(CompactString, Option<CompactString>),
+        version_req: &VersionReq,
+    ) -> Result<MatchedVersion, RegistryError> {
+        let mut path = PathBuf::with_capacity(128);
+        path.push(&**c1);
+        if let Some(c2) = c2 {
+            path.push(&**c2);
+        }
+
+        path.push(&*crate_name.to_lowercase());
+        let crate_versions = repo
+            .get_head_commit_entry_data_by_path(path)?
+            .ok_or_else(|| RegistryError::NotFound(crate_name.into()))?;
+
+        MatchedVersion::find(
+            &mut JsonDeserializer::from_slice(&crate_versions).into_iter(),
+            version_req,
+        )
+    }
+
+    pub async fn fetch_crate_matched(
+        &self,
+        client: Client,
+        name: &str,
+        version_req: &VersionReq,
+    ) -> Result<Manifest<Meta>, RegistryError> {
+        let crate_prefix = crate_prefix_components(name)?;
+        let crate_name = name.to_compact_string();
+        let version_req = version_req.clone();
+        let this = self.clone();
+
+        let cancellation_token = GitCancellationToken::default();
+        // Cancel git operation if the future is cancelled (dropped).
+        let cancel_on_drop = cancellation_token.clone().cancel_on_drop();
+
+        let (matched_version, dl_url) = spawn_blocking(move || {
+            let GitIndex {
+                _tempdir: _,
+                repo,
+                dl_template,
+            } = this
+                .0
+                .git_index
+                .get_or_try_init(|| GitIndex::new(this.0.url.clone(), cancellation_token))?;
+
+            let matched_version =
+                Self::find_crate_matched_ver(repo, &crate_name, &crate_prefix, &version_req)?;
+
+            let url = Url::parse(&render_dl_template(
+                dl_template,
+                &crate_name,
+                &crate_prefix,
+                &matched_version,
+            )?)?;
+
+            Ok::<_, RegistryError>((matched_version, url))
+        })
+        .await??;
+
+        // Git operation done, disarm it
+        cancel_on_drop.disarm();
+
+        parse_manifest(client, name, dl_url, matched_version).await
+    }
+}

crates/binstalk-registry/src/lib.rs

@@ -0,0 +1,294 @@
+#![cfg_attr(docsrs, feature(doc_auto_cfg))]
+
+use std::{io, str::FromStr, sync::Arc};
+
+use base16::DecodeError as Base16DecodeError;
+use binstalk_downloader::{
+    download::DownloadError,
+    remote::{Client, Error as RemoteError},
+};
+use binstalk_types::cargo_toml_binstall::Meta;
+use cargo_toml_workspace::cargo_toml::{Error as CargoTomlError, Manifest};
+use compact_str::CompactString;
+use leon::{ParseError, RenderError};
+use miette::Diagnostic;
+use semver::VersionReq;
+use serde_json::Error as JsonError;
+use thiserror::Error as ThisError;
+use tokio::task;
+use url::{ParseError as UrlParseError, Url};
+
+#[cfg(feature = "git")]
+pub use binstalk_downloader::git::{GitError, GitUrl, GitUrlParseError};
+
+mod vfs;
+
+mod visitor;
+
+mod common;
+use common::*;
+
+#[cfg(feature = "git")]
+mod git_registry;
+#[cfg(feature = "git")]
+pub use git_registry::GitRegistry;
+
+mod crates_io_registry;
+pub use crates_io_registry::{fetch_crate_cratesio, CratesIoRateLimit};
+
+mod sparse_registry;
+pub use sparse_registry::SparseRegistry;
+
+#[derive(Debug, ThisError, Diagnostic)]
+#[diagnostic(severity(error), code(binstall::cargo_registry))]
+#[non_exhaustive]
+pub enum RegistryError {
+    #[error(transparent)]
+    Remote(#[from] RemoteError),
+
+    #[error("{0} is not found")]
+    #[diagnostic(
+        help("Check that the crate name you provided is correct.\nYou can also search for a matching crate at: https://lib.rs/search?q={0}")
+    )]
+    NotFound(CompactString),
+
+    #[error(transparent)]
+    Json(#[from] JsonError),
+
+    #[error("Failed to parse dl config: {0}")]
+    ParseDlConfig(#[from] ParseError),
+
+    #[error("Failed to render dl config: {0}")]
+    RenderDlConfig(#[from] RenderError),
+
+    #[error("Failed to parse checksum encoded in hex: {0}")]
+    InvalidHex(#[from] Base16DecodeError),
+
+    #[error("Expected checksum `{expected}`, actual checksum `{actual}`")]
+    UnmatchedChecksum {
+        expected: Box<str>,
+        actual: Box<str>,
+    },
+
+    #[error("no version matching requirement '{req}'")]
+    VersionMismatch { req: semver::VersionReq },
+
+    #[error("Failed to parse cargo manifest: {0}")]
+    #[diagnostic(help("If you used --manifest-path, check the Cargo.toml syntax."))]
+    CargoManifest(#[from] Box<CargoTomlError>),
+
+    #[error("Failed to parse url: {0}")]
+    UrlParse(#[from] url::ParseError),
+
+    #[error(transparent)]
+    Download(#[from] DownloadError),
+
+    #[error("I/O Error: {0}")]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    TaskJoinError(#[from] task::JoinError),
+
+    #[cfg(feature = "git")]
+    #[error("Failed to shallow clone git repository: {0}")]
+    GitError(#[from] GitError),
+}
+
+impl From<CargoTomlError> for RegistryError {
+    fn from(e: CargoTomlError) -> Self {
+        Self::from(Box::new(e))
+    }
+}
+
+#[derive(Clone, Debug)]
+#[non_exhaustive]
+pub enum Registry {
+    CratesIo(Arc<CratesIoRateLimit>),
+
+    Sparse(Arc<SparseRegistry>),
+
+    #[cfg(feature = "git")]
+    Git(GitRegistry),
+}
+
+impl Default for Registry {
+    fn default() -> Self {
+        Self::CratesIo(Default::default())
+    }
+}
+
+#[derive(Debug, ThisError)]
+#[error("Invalid registry `{src}`, {inner}")]
+pub struct InvalidRegistryError {
+    src: CompactString,
+    #[source]
+    inner: InvalidRegistryErrorInner,
+}
+
+#[derive(Debug, ThisError)]
+enum InvalidRegistryErrorInner {
+    #[cfg(feature = "git")]
+    #[error("failed to parse git url {0}")]
+    GitUrlParseErr(#[from] Box<GitUrlParseError>),
+
+    #[error("failed to parse sparse registry url: {0}")]
+    UrlParseErr(#[from] UrlParseError),
+
+    #[error("expected protocol http(s), actual url `{0}`")]
+    InvalidScheme(Box<Url>),
+
+    #[cfg(not(feature = "git"))]
+    #[error("git registry not supported")]
+    GitRegistryNotSupported,
+}
+
+impl Registry {
+    fn from_str_inner(s: &str) -> Result<Self, InvalidRegistryErrorInner> {
+        if let Some(s) = s.strip_prefix("sparse+") {
+            let url = Url::parse(s)?;
+
+            let scheme = url.scheme();
+            if scheme != "http" && scheme != "https" {
+                Err(InvalidRegistryErrorInner::InvalidScheme(Box::new(url)))
+            } else {
+                Ok(Self::Sparse(Arc::new(SparseRegistry::new(url))))
+            }
+        } else {
+            #[cfg(not(feature = "git"))]
+            {
+                Err(InvalidRegistryErrorInner::GitRegistryNotSupported)
+            }
+            #[cfg(feature = "git")]
+            {
+                let url = GitUrl::from_str(s).map_err(Box::new)?;
+                Ok(Self::Git(GitRegistry::new(url)))
+            }
+        }
+    }
+
+    /// Fetch the latest crate with `crate_name` and with version matching
+    /// `version_req`.
+    pub async fn fetch_crate_matched(
+        &self,
+        client: Client,
+        crate_name: &str,
+        version_req: &VersionReq,
+    ) -> Result<Manifest<Meta>, RegistryError> {
+        match self {
+            Self::CratesIo(rate_limit) => {
+                fetch_crate_cratesio(client, crate_name, version_req, rate_limit).await
+            }
+            Self::Sparse(sparse_registry) => {
+                sparse_registry
+                    .fetch_crate_matched(client, crate_name, version_req)
+                    .await
+            }
+            #[cfg(feature = "git")]
+            Self::Git(git_registry) => {
+                git_registry
+                    .fetch_crate_matched(client, crate_name, version_req)
+                    .await
+            }
+        }
+    }
+}
+
+impl FromStr for Registry {
+    type Err = InvalidRegistryError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::from_str_inner(s).map_err(|inner| InvalidRegistryError {
+            src: s.into(),
+            inner,
+        })
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::num::NonZeroU16;
+
+    use toml_edit::ser::to_string;
+
+    use super::*;
+
+    /// Mark this as an async fn so that you won't accidentally use it in
+    /// sync context.
+    async fn create_client() -> Client {
+        Client::new(
+            concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")),
+            None,
+            NonZeroU16::new(10).unwrap(),
+            1.try_into().unwrap(),
+            [],
+        )
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_crates_io_sparse_registry() {
+        let client = create_client().await;
+
+        let sparse_registry: Registry = "sparse+https://index.crates.io/".parse().unwrap();
+        assert!(
+            matches!(sparse_registry, Registry::Sparse(_)),
+            "{:?}",
+            sparse_registry
+        );
+
+        let crate_name = "cargo-binstall";
+        let version_req = &VersionReq::parse("=1.0.0").unwrap();
+        let manifest_from_sparse = sparse_registry
+            .fetch_crate_matched(client.clone(), crate_name, version_req)
+            .await
+            .unwrap();
+
+        let manifest_from_cratesio_api = Registry::default()
+            .fetch_crate_matched(client, crate_name, version_req)
+            .await
+            .unwrap();
+
+        let serialized_manifest_from_sparse = to_string(&manifest_from_sparse).unwrap();
+        let serialized_manifest_from_cratesio_api = to_string(&manifest_from_cratesio_api).unwrap();
+
+        assert_eq!(
+            serialized_manifest_from_sparse,
+            serialized_manifest_from_cratesio_api
+        );
+    }
+
+    #[cfg(feature = "git")]
+    #[tokio::test]
+    async fn test_crates_io_git_registry() {
+        let client = create_client().await;
+
+        let git_registry: Registry = "https://github.com/rust-lang/crates.io-index"
+            .parse()
+            .unwrap();
+        assert!(
+            matches!(git_registry, Registry::Git(_)),
+            "{:?}",
+            git_registry
+        );
+
+        let crate_name = "cargo-binstall";
+        let version_req = &VersionReq::parse("=1.0.0").unwrap();
+        let manifest_from_git = git_registry
+            .fetch_crate_matched(client.clone(), crate_name, version_req)
+            .await
+            .unwrap();
+
+        let manifest_from_cratesio_api = Registry::default()
+            .fetch_crate_matched(client, crate_name, version_req)
+            .await
+            .unwrap();
+
+        let serialized_manifest_from_git = to_string(&manifest_from_git).unwrap();
+        let serialized_manifest_from_cratesio_api = to_string(&manifest_from_cratesio_api).unwrap();
+
+        assert_eq!(
+            serialized_manifest_from_git,
+            serialized_manifest_from_cratesio_api
+        );
+    }
+}

crates/binstalk-registry/src/sparse_registry.rs

@@ -0,0 +1,105 @@
+use binstalk_downloader::remote::{Client, Error as RemoteError};
+use binstalk_types::cargo_toml_binstall::Meta;
+use cargo_toml_workspace::cargo_toml::Manifest;
+use compact_str::CompactString;
+use semver::VersionReq;
+use serde_json::Deserializer as JsonDeserializer;
+use tokio::sync::OnceCell;
+use url::Url;
+
+use crate::{
+    crate_prefix_components, parse_manifest, render_dl_template, MatchedVersion, RegistryConfig,
+    RegistryError,
+};
+
+#[derive(Debug)]
+pub struct SparseRegistry {
+    url: Url,
+    dl_template: OnceCell<CompactString>,
+}
+
+impl SparseRegistry {
+    /// * `url` - `url.cannot_be_a_base()` must be `false`
+    pub fn new(url: Url) -> Self {
+        Self {
+            url,
+            dl_template: Default::default(),
+        }
+    }
+
+    async fn get_dl_template(&self, client: &Client) -> Result<&str, RegistryError> {
+        self.dl_template
+            .get_or_try_init(|| {
+                Box::pin(async {
+                    let mut url = self.url.clone();
+                    url.path_segments_mut().unwrap().push("config.json");
+                    let config: RegistryConfig = client.get(url).send(true).await?.json().await?;
+                    Ok(config.dl)
+                })
+            })
+            .await
+            .map(AsRef::as_ref)
+    }
+
+    /// `url` must be a valid http(s) url.
+    async fn find_crate_matched_ver(
+        client: &Client,
+        mut url: Url,
+        crate_name: &str,
+        (c1, c2): &(CompactString, Option<CompactString>),
+        version_req: &VersionReq,
+    ) -> Result<MatchedVersion, RegistryError> {
+        {
+            let mut path = url.path_segments_mut().unwrap();
+
+            path.push(c1);
+            if let Some(c2) = c2 {
+                path.push(c2);
+            }
+
+            path.push(&crate_name.to_lowercase());
+        }
+
+        let body = client
+            .get(url)
+            .send(true)
+            .await
+            .map_err(|e| match e {
+                RemoteError::Http(e) if e.is_status() => RegistryError::NotFound(crate_name.into()),
+                e => e.into(),
+            })?
+            .bytes()
+            .await
+            .map_err(RegistryError::from)?;
+        MatchedVersion::find(
+            &mut JsonDeserializer::from_slice(&body).into_iter(),
+            version_req,
+        )
+    }
+
+    pub async fn fetch_crate_matched(
+        &self,
+        client: Client,
+        crate_name: &str,
+        version_req: &VersionReq,
+    ) -> Result<Manifest<Meta>, RegistryError> {
+        let crate_prefix = crate_prefix_components(crate_name)?;
+        let dl_template = self.get_dl_template(&client).await?;
+        let matched_version = Self::find_crate_matched_ver(
+            &client,
+            self.url.clone(),
+            crate_name,
+            &crate_prefix,
+            version_req,
+        )
+        .await?;
+        let dl_url = Url::parse(&render_dl_template(
+            dl_template,
+            crate_name,
+            &crate_prefix,
+            &matched_version,
+        )?)?;
+
+        parse_manifest(client, crate_name, dl_url, matched_version).await
+    }
+}

crates/binstalk-registry/src/vfs.rs

@@ -0,0 +1,43 @@
+use std::{
+    collections::{hash_set::HashSet, BTreeMap},
+    io,
+    path::Path,
+};
+
+use cargo_toml_workspace::cargo_toml::AbstractFilesystem;
+use normalize_path::NormalizePath;
+
+/// This type stores the filesystem structure for the crate tarball
+/// extracted in memory and can be passed to
+/// `cargo_toml::Manifest::complete_from_abstract_filesystem`.
+#[derive(Debug, Default)]
+pub(super) struct Vfs(BTreeMap<Box<Path>, HashSet<Box<str>>>);
+
+impl Vfs {
+    /// * `path` - must be canonical, must not be empty.
+    pub(super) fn add_path(&mut self, mut path: &Path) {
+        while let Some(parent) = path.parent() {
+            // Since path has parent, it must have a filename
+            let filename = path.file_name().unwrap();
+
+            // `cargo_toml`'s implementation does the same thing.
+            // https://docs.rs/cargo_toml/0.11.5/src/cargo_toml/afs.rs.html#24
+            let filename = filename.to_string_lossy();
+
+            self.0
+                .entry(parent.into())
+                .or_insert_with(|| HashSet::with_capacity(4))
+                .insert(filename.into());
+
+            path = parent;
+        }
+    }
+}
+
+impl AbstractFilesystem for Vfs {
+    fn file_names_in(&self, rel_path: &str) -> io::Result<HashSet<Box<str>>> {
+        let rel_path = Path::new(rel_path).normalize();
+
+        Ok(self.0.get(&*rel_path).map(Clone::clone).unwrap_or_default())
+    }
+}

crates/binstalk-registry/src/visitor.rs

@@ -0,0 +1,81 @@
+use std::path::{Path, PathBuf};
+
+use binstalk_downloader::download::{DownloadError, TarEntriesVisitor, TarEntry};
+use binstalk_types::cargo_toml_binstall::Meta;
+use cargo_toml_workspace::cargo_toml::{Manifest, Value};
+use normalize_path::NormalizePath;
+use tokio::io::AsyncReadExt;
+use tracing::debug;
+
+use crate::{vfs::Vfs, RegistryError};
+
+#[derive(Debug)]
+pub(super) struct ManifestVisitor {
+    cargo_toml_content: Vec<u8>,
+    /// manifest_dir_path is treated as the current dir.
+    manifest_dir_path: PathBuf,
+
+    vfs: Vfs,
+}
+
+impl ManifestVisitor {
+    pub(super) fn new(manifest_dir_path: PathBuf) -> Self {
+        Self {
+            // Cargo.toml is quite large usually.
+            cargo_toml_content: Vec::with_capacity(2000),
+            manifest_dir_path,
+            vfs: Vfs::default(),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl TarEntriesVisitor for ManifestVisitor {
+    async fn visit(&mut self, entry: &mut dyn TarEntry) -> Result<(), DownloadError> {
+        let path = entry.path()?;
+        let path = path.normalize();
+
+        let path = if let Ok(path) = path.strip_prefix(&self.manifest_dir_path) {
+            path
+        } else {
+            // The path is outside of the curr dir (manifest dir),
+            // ignore it.
+            return Ok(());
+        };
+
+        if path == Path::new("Cargo.toml")
+            || path == Path::new("src/main.rs")
+            || path.starts_with("src/bin")
+        {
+            self.vfs.add_path(path);
+        }
+
+        if path == Path::new("Cargo.toml") {
+            // Since it is possible for the same Cargo.toml to appear
+            // multiple times using `tar --keep-old-files`, here we
+            // clear the buffer first before reading into it.
+            self.cargo_toml_content.clear();
+            self.cargo_toml_content
+                .reserve_exact(entry.size()?.try_into().unwrap_or(usize::MAX));
+            entry.read_to_end(&mut self.cargo_toml_content).await?;
+        }
+
+        Ok(())
+    }
+}
+
+impl ManifestVisitor {
+    /// Load binstall metadata using the extracted information stored in memory.
+    pub(super) fn load_manifest(self) -> Result<Manifest<Meta>, RegistryError> {
+        debug!("Loading manifest directly from extracted file");
+
+        // Load and parse manifest
+        let mut manifest = Manifest::from_slice_with_metadata(&self.cargo_toml_content)?;
+
+        // Checks vfs for binary output names
+        manifest.complete_from_abstract_filesystem::<Value, _>(&self.vfs, None)?;
+
+        // Return metadata
+        Ok(manifest)
+    }
+}