Add phantom digest support to download (#315)

Cargo.lock

@@ -87,10 +87,12 @@ dependencies = [
  "compact_str",
  "crates_io_api",
  "detect-targets",
+ "digest",
  "env_logger",
  "flate2",
  "flock",
  "futures-util",
+ "generic-array",
  "home",
  "itertools",
  "jobserver",
@@ -125,6 +127,15 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "block-buffer"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.10.0"
@@ -351,6 +362,16 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "crypto-common"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.3.2"
@@ -373,6 +394,16 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "digest"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
 [[package]]
 name = "dirs"
 version = "4.0.0"
@@ -637,6 +668,16 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "generic-array"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.7"
@@ -2053,6 +2094,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
+[[package]]
+name = "typenum"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.8"

crates/lib/Cargo.toml

@@ -18,9 +18,11 @@ clap = { version = "3.2.17", features = ["derive"] }
 compact_str = { version = "0.6.0", features = ["serde"] }
 crates_io_api = { version = "0.8.0", default-features = false }
 detect-targets = { version = "0.1.0", path = "../detect-targets" }
+digest = "0.10.3"
 flate2 = { version = "1.0.24", default-features = false }
 flock = { version = "0.1.0", path = "../flock" }
 futures-util = { version = "0.3.23", default-features = false, features = ["std"] }
+generic-array = "0.14.6"
 home = "0.5.3"
 itertools = "0.10.3"
 jobserver = "0.1.24"

crates/lib/src/drivers/crates_io.rs

@@ -9,7 +9,7 @@ use url::Url;
 
 use crate::{
     errors::BinstallError,
-    helpers::download::download_tar_based_and_visit,
+    helpers::download::Download,
     manifests::cargo_toml_binstall::{Meta, TarBasedFmt},
 };
 
@@ -52,11 +52,7 @@ pub async fn fetch_crate_cratesio(
 
     let manifest_dir_path: PathBuf = format!("{name}-{version_name}").into();
 
-    download_tar_based_and_visit(
+    Download::new(client, Url::parse(&crate_url)?)
-        client,
+        .and_visit_tar(TarBasedFmt::Tgz, ManifestVisitor::new(manifest_dir_path))
-        Url::parse(&crate_url)?,
+        .await
-        TarBasedFmt::Tgz,
-        ManifestVisitor::new(manifest_dir_path),
-    )
-    .await
 }

crates/lib/src/fetchers/gh_crate_meta.rs

@@ -13,7 +13,7 @@ use url::Url;
 use crate::{
     errors::BinstallError,
     helpers::{
-        download::download_and_extract,
+        download::Download,
         remote::{get_redirected_final_url, remote_exists},
         tasks::AutoAbortJoinHandle,
     },
@@ -145,7 +145,9 @@ impl super::Fetcher for GhCrateMeta {
     async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> {
         let (url, pkg_fmt) = self.resolution.get().unwrap(); // find() is called first
         debug!("Downloading package from: '{url}'");
-        download_and_extract(&self.client, url, *pkg_fmt, dst).await
+        Download::new(&self.client, url.clone())
+            .and_extract(self.pkg_fmt(), dst)
+            .await
     }
 
     fn pkg_fmt(&self) -> PkgFmt {

crates/lib/src/fetchers/quickinstall.rs

@@ -9,7 +9,7 @@ use url::Url;
 
 use crate::{
     errors::BinstallError,
-    helpers::{download::download_and_extract, remote::remote_exists},
+    helpers::{download::Download, remote::remote_exists},
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
 };
 
@@ -49,7 +49,9 @@ impl super::Fetcher for QuickInstall {
     async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> {
         let url = self.package_url();
         debug!("Downloading package from: '{url}'");
-        download_and_extract(&self.client, &Url::parse(&url)?, self.pkg_fmt(), dst).await
+        Download::new(&self.client, Url::parse(&url)?)
+            .and_extract(self.pkg_fmt(), dst)
+            .await
     }
 
     fn pkg_fmt(&self) -> PkgFmt {

crates/lib/src/helpers/download.rs

@@ -1,5 +1,6 @@
-use std::{fmt::Debug, path::Path};
+use std::{fmt::Debug, marker::PhantomData, path::Path};
 
+use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update};
 use log::debug;
 use reqwest::{Client, Url};
 
@@ -15,47 +16,98 @@ use async_extracter::*;
 mod async_extracter;
 mod extracter;
 mod stream_readable;
-/// Download a file from the provided URL and extract it to the provided path.
-pub async fn download_and_extract<P: AsRef<Path>>(
-    client: &Client,
-    url: &Url,
-    fmt: PkgFmt,
-    path: P,
-) -> Result<(), BinstallError> {
-    let stream = create_request(client, url.clone()).await?;
 
-    let path = path.as_ref();
+#[derive(Debug)]
-    debug!("Downloading and extracting to: '{}'", path.display());
+pub struct Download<'client, D: Digest = NoDigest> {
+    client: &'client Client,
+    url: Url,
+    _digest: PhantomData<D>,
+    _checksum: Vec<u8>,
+}
 
-    match fmt.decompose() {
+impl<'client> Download<'client> {
-        PkgFmtDecomposed::Tar(fmt) => extract_tar_based_stream(stream, path, fmt).await?,
+    pub fn new(client: &'client Client, url: Url) -> Self {
-        PkgFmtDecomposed::Bin => extract_bin(stream, path).await?,
+        Self {
-        PkgFmtDecomposed::Zip => extract_zip(stream, path).await?,
+            client,
+            url,
+            _digest: PhantomData::default(),
+            _checksum: Vec::new(),
+        }
     }
 
-    debug!("Download OK, extracted to: '{}'", path.display());
+    /// Download a file from the provided URL and extract part of it to
+    /// the provided path.
+    ///
+    ///  * `filter` - If Some, then it will pass the path of the file to it
+    ///    and only extract ones which filter returns `true`.
+    ///
+    /// This does not support verifying a checksum due to the partial extraction
+    /// and will ignore one if specified.
+    pub async fn and_visit_tar<V: TarEntriesVisitor + Debug + Send + 'static>(
+        self,
+        fmt: TarBasedFmt,
+        visitor: V,
+    ) -> Result<V::Target, BinstallError> {
+        let stream = create_request(self.client, self.url).await?;
 
-    Ok(())
+        debug!("Downloading and extracting then in-memory processing");
+
+        let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?;
+
+        debug!("Download, extraction and in-memory procession OK");
+
+        Ok(ret)
+    }
+
+    /// Download a file from the provided URL and extract it to the provided path.
+    pub async fn and_extract(
+        self,
+        fmt: PkgFmt,
+        path: impl AsRef<Path>,
+    ) -> Result<(), BinstallError> {
+        let stream = create_request(self.client, self.url).await?;
+
+        let path = path.as_ref();
+        debug!("Downloading and extracting to: '{}'", path.display());
+
+        match fmt.decompose() {
+            PkgFmtDecomposed::Tar(fmt) => extract_tar_based_stream(stream, path, fmt).await?,
+            PkgFmtDecomposed::Bin => extract_bin(stream, path).await?,
+            PkgFmtDecomposed::Zip => extract_zip(stream, path).await?,
+        }
+
+        debug!("Download OK, extracted to: '{}'", path.display());
+
+        Ok(())
+    }
 }
 
-/// Download a file from the provided URL and extract part of it to
+impl<'client, D: Digest> Download<'client, D> {
-/// the provided path.
+    pub fn new_with_checksum(client: &'client Client, url: Url, checksum: Vec<u8>) -> Self {
-///
+        Self {
-///  * `filter` - If Some, then it will pass the path of the file to it
+            client,
-///    and only extract ones which filter returns `true`.
+            url,
-pub async fn download_tar_based_and_visit<V: TarEntriesVisitor + Debug + Send + 'static>(
+            _digest: PhantomData::default(),
-    client: &Client,
+            _checksum: checksum,
-    url: Url,
+        }
-    fmt: TarBasedFmt,
+    }
-    visitor: V,
-) -> Result<V::Target, BinstallError> {
-    let stream = create_request(client, url).await?;
 
-    debug!("Downloading and extracting then in-memory processing");
+    // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here
-
-    let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?;
-
-    debug!("Download, extraction and in-memory procession OK");
-
-    Ok(ret)
 }
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct NoDigest;
+
+impl FixedOutput for NoDigest {
+    fn finalize_into(self, _out: &mut Output<Self>) {}
+}
+
+impl OutputSizeUser for NoDigest {
+    type OutputSize = generic_array::typenum::U0;
+}
+
+impl Update for NoDigest {
+    fn update(&mut self, _data: &[u8]) {}
+}
+
+impl HashMarker for NoDigest {}