From 1cf6076d621ed796873422ba90318cc02f55f8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Saparelli?= Date: Sun, 4 Sep 2022 22:47:18 +1200 Subject: [PATCH] Add phantom digest support to download (#315) --- Cargo.lock | 47 +++++++++ crates/lib/Cargo.toml | 2 + crates/lib/src/drivers/crates_io.rs | 12 +-- crates/lib/src/fetchers/gh_crate_meta.rs | 6 +- crates/lib/src/fetchers/quickinstall.rs | 6 +- crates/lib/src/helpers/download.rs | 124 ++++++++++++++++------- 6 files changed, 149 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b766bdde..313d3807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,10 +87,12 @@ dependencies = [ "compact_str", "crates_io_api", "detect-targets", + "digest", "env_logger", "flate2", "flock", "futures-util", + "generic-array", "home", "itertools", "jobserver", @@ -125,6 +127,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.10.0" @@ -351,6 +362,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -373,6 +394,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "4.0.0" @@ -637,6 +668,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.7" @@ -2053,6 +2094,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + [[package]] name = "unicode-bidi" version = "0.3.8" diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index aaa2449a..bcbb2353 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -18,9 +18,11 @@ clap = { version = "3.2.17", features = ["derive"] } compact_str = { version = "0.6.0", features = ["serde"] } crates_io_api = { version = "0.8.0", default-features = false } detect-targets = { version = "0.1.0", path = "../detect-targets" } +digest = "0.10.3" flate2 = { version = "1.0.24", default-features = false } flock = { version = "0.1.0", path = "../flock" } futures-util = { version = "0.3.23", default-features = false, features = ["std"] } +generic-array = "0.14.6" home = "0.5.3" itertools = "0.10.3" jobserver = "0.1.24" diff --git a/crates/lib/src/drivers/crates_io.rs b/crates/lib/src/drivers/crates_io.rs index 0ac81d64..4858f246 100644 --- a/crates/lib/src/drivers/crates_io.rs +++ b/crates/lib/src/drivers/crates_io.rs @@ -9,7 +9,7 @@ use url::Url; use crate::{ errors::BinstallError, - helpers::download::download_tar_based_and_visit, + helpers::download::Download, manifests::cargo_toml_binstall::{Meta, TarBasedFmt}, }; @@ -52,11 +52,7 @@ pub async fn fetch_crate_cratesio( let manifest_dir_path: PathBuf = format!("{name}-{version_name}").into(); - download_tar_based_and_visit( - client, - Url::parse(&crate_url)?, - TarBasedFmt::Tgz, - ManifestVisitor::new(manifest_dir_path), - ) - .await + Download::new(client, Url::parse(&crate_url)?) + .and_visit_tar(TarBasedFmt::Tgz, ManifestVisitor::new(manifest_dir_path)) + .await } diff --git a/crates/lib/src/fetchers/gh_crate_meta.rs b/crates/lib/src/fetchers/gh_crate_meta.rs index d55d033e..56dac9fa 100644 --- a/crates/lib/src/fetchers/gh_crate_meta.rs +++ b/crates/lib/src/fetchers/gh_crate_meta.rs @@ -13,7 +13,7 @@ use url::Url; use crate::{ errors::BinstallError, helpers::{ - download::download_and_extract, + download::Download, remote::{get_redirected_final_url, remote_exists}, tasks::AutoAbortJoinHandle, }, @@ -145,7 +145,9 @@ impl super::Fetcher for GhCrateMeta { async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> { let (url, pkg_fmt) = self.resolution.get().unwrap(); // find() is called first debug!("Downloading package from: '{url}'"); - download_and_extract(&self.client, url, *pkg_fmt, dst).await + Download::new(&self.client, url.clone()) + .and_extract(self.pkg_fmt(), dst) + .await } fn pkg_fmt(&self) -> PkgFmt { diff --git a/crates/lib/src/fetchers/quickinstall.rs b/crates/lib/src/fetchers/quickinstall.rs index b0e8460c..a2bf4012 100644 --- a/crates/lib/src/fetchers/quickinstall.rs +++ b/crates/lib/src/fetchers/quickinstall.rs @@ -9,7 +9,7 @@ use url::Url; use crate::{ errors::BinstallError, - helpers::{download::download_and_extract, remote::remote_exists}, + helpers::{download::Download, remote::remote_exists}, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, }; @@ -49,7 +49,9 @@ impl super::Fetcher for QuickInstall { async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> { let url = self.package_url(); debug!("Downloading package from: '{url}'"); - download_and_extract(&self.client, &Url::parse(&url)?, self.pkg_fmt(), dst).await + Download::new(&self.client, Url::parse(&url)?) + .and_extract(self.pkg_fmt(), dst) + .await } fn pkg_fmt(&self) -> PkgFmt { diff --git a/crates/lib/src/helpers/download.rs b/crates/lib/src/helpers/download.rs index 987805a6..9fb6d8e5 100644 --- a/crates/lib/src/helpers/download.rs +++ b/crates/lib/src/helpers/download.rs @@ -1,5 +1,6 @@ -use std::{fmt::Debug, path::Path}; +use std::{fmt::Debug, marker::PhantomData, path::Path}; +use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update}; use log::debug; use reqwest::{Client, Url}; @@ -15,47 +16,98 @@ use async_extracter::*; mod async_extracter; mod extracter; mod stream_readable; -/// Download a file from the provided URL and extract it to the provided path. -pub async fn download_and_extract>( - client: &Client, - url: &Url, - fmt: PkgFmt, - path: P, -) -> Result<(), BinstallError> { - let stream = create_request(client, url.clone()).await?; - let path = path.as_ref(); - debug!("Downloading and extracting to: '{}'", path.display()); +#[derive(Debug)] +pub struct Download<'client, D: Digest = NoDigest> { + client: &'client Client, + url: Url, + _digest: PhantomData, + _checksum: Vec, +} - match fmt.decompose() { - PkgFmtDecomposed::Tar(fmt) => extract_tar_based_stream(stream, path, fmt).await?, - PkgFmtDecomposed::Bin => extract_bin(stream, path).await?, - PkgFmtDecomposed::Zip => extract_zip(stream, path).await?, +impl<'client> Download<'client> { + pub fn new(client: &'client Client, url: Url) -> Self { + Self { + client, + url, + _digest: PhantomData::default(), + _checksum: Vec::new(), + } } - debug!("Download OK, extracted to: '{}'", path.display()); + /// Download a file from the provided URL and extract part of it to + /// the provided path. + /// + /// * `filter` - If Some, then it will pass the path of the file to it + /// and only extract ones which filter returns `true`. + /// + /// This does not support verifying a checksum due to the partial extraction + /// and will ignore one if specified. + pub async fn and_visit_tar( + self, + fmt: TarBasedFmt, + visitor: V, + ) -> Result { + let stream = create_request(self.client, self.url).await?; - Ok(()) + debug!("Downloading and extracting then in-memory processing"); + + let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?; + + debug!("Download, extraction and in-memory procession OK"); + + Ok(ret) + } + + /// Download a file from the provided URL and extract it to the provided path. + pub async fn and_extract( + self, + fmt: PkgFmt, + path: impl AsRef, + ) -> Result<(), BinstallError> { + let stream = create_request(self.client, self.url).await?; + + let path = path.as_ref(); + debug!("Downloading and extracting to: '{}'", path.display()); + + match fmt.decompose() { + PkgFmtDecomposed::Tar(fmt) => extract_tar_based_stream(stream, path, fmt).await?, + PkgFmtDecomposed::Bin => extract_bin(stream, path).await?, + PkgFmtDecomposed::Zip => extract_zip(stream, path).await?, + } + + debug!("Download OK, extracted to: '{}'", path.display()); + + Ok(()) + } } -/// Download a file from the provided URL and extract part of it to -/// the provided path. -/// -/// * `filter` - If Some, then it will pass the path of the file to it -/// and only extract ones which filter returns `true`. -pub async fn download_tar_based_and_visit( - client: &Client, - url: Url, - fmt: TarBasedFmt, - visitor: V, -) -> Result { - let stream = create_request(client, url).await?; +impl<'client, D: Digest> Download<'client, D> { + pub fn new_with_checksum(client: &'client Client, url: Url, checksum: Vec) -> Self { + Self { + client, + url, + _digest: PhantomData::default(), + _checksum: checksum, + } + } - debug!("Downloading and extracting then in-memory processing"); - - let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?; - - debug!("Download, extraction and in-memory procession OK"); - - Ok(ret) + // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here } + +#[derive(Clone, Copy, Debug, Default)] +pub struct NoDigest; + +impl FixedOutput for NoDigest { + fn finalize_into(self, _out: &mut Output) {} +} + +impl OutputSizeUser for NoDigest { + type OutputSize = generic_array::typenum::U0; +} + +impl Update for NoDigest { + fn update(&mut self, _data: &[u8]) {} +} + +impl HashMarker for NoDigest {}