Refactor: Extract new crate binstalk-{signal, downloader} (#518)

* Refactor: Extract new crate binstalk-downloader
* Re-export `PkgFmt` from `binstalk_manifests`
* Update release-pr.yml
* Update dependabot

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

.github/dependabot.yml

@@ -24,6 +24,10 @@ updates:
     directory: "/crates/binstalk"
     schedule:
       interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/crates/binstalk-downloader"
+    schedule:
+      interval: "daily"
   - package-ecosystem: "cargo"
     directory: "/crates/detect-wasi"
     schedule:

.github/workflows/release-pr.yml

@@ -10,6 +10,7 @@ on:
           - bin
           - binstalk
           - binstalk-manifests
+          - binstalk-downloader
           - detect-targets
           - detect-wasi
           - fs-lock

Cargo.lock

@@ -109,29 +109,21 @@ name = "binstalk"
 version = "0.4.1"
 dependencies = [
  "async-trait",
+ "binstalk-downloader",
  "binstalk-manifests",
- "binstall-tar",
- "bytes",
- "bzip2",
  "cargo_toml",
  "compact_str",
  "crates_io_api",
  "detect-targets",
- "digest",
  "env_logger",
- "flate2",
  "futures-util",
- "generic-array",
  "home",
- "httpdate",
  "itertools",
  "jobslot",
  "log",
  "miette",
  "normalize-path",
  "once_cell",
- "reqwest",
- "scopeguard",
  "semver",
  "serde",
  "strum",
@@ -139,6 +131,29 @@ dependencies = [
  "thiserror",
  "tinytemplate",
  "tokio",
+ "url",
+ "xz2",
+]
+
+[[package]]
+name = "binstalk-downloader"
+version = "0.1.0"
+dependencies = [
+ "binstalk-manifests",
+ "binstall-tar",
+ "bytes",
+ "bzip2",
+ "digest",
+ "flate2",
+ "futures-util",
+ "generic-array",
+ "httpdate",
+ "log",
+ "reqwest",
+ "scopeguard",
+ "tempfile",
+ "thiserror",
+ "tokio",
  "tower",
  "trust-dns-resolver",
  "url",

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "crates/bin",
     "crates/binstalk",
     "crates/binstalk-manifests",
+    "crates/binstalk-downloader",
     "crates/detect-wasi",
     "crates/fs-lock",
     "crates/normalize-path",

crates/bin/src/entry.rs

@@ -35,7 +35,8 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien
         args.min_tls_version.map(|v| v.into()),
         Duration::from_millis(rate_limit.duration.get()),
         rate_limit.request_count,
-    )?;
+    )
+    .map_err(BinstallError::from)?;
 
     // Build crates.io api client
     let crates_io_api_client = crates_io_api::AsyncClient::with_http_client(

crates/binstalk-downloader/Cargo.toml

@@ -0,0 +1,70 @@
+[package]
+name = "binstalk-downloader"
+description = "The binstall toolkit for downloading and extracting file"
+repository = "https://github.com/cargo-bins/cargo-binstall"
+documentation = "https://docs.rs/binstalk-downloader"
+version = "0.1.0"
+rust-version = "1.61.0"
+authors = ["ryan <ryan@kurte.nz>"]
+edition = "2021"
+license = "GPL-3.0"
+
+[dependencies]
+binstalk-manifests = { version = "0.1.0", path = "../binstalk-manifests" }
+bytes = "1.2.1"
+bzip2 = "0.4.3"
+digest = "0.10.5"
+flate2 = { version = "1.0.24", default-features = false }
+futures-util = { version = "0.3.25", default-features = false, features = ["std"] }
+generic-array = "0.14.6"
+httpdate = "1.0.2"
+log = { version = "0.4.17", features = ["std"] }
+reqwest = { version = "0.11.12", features = ["stream", "gzip", "brotli", "deflate"], default-features = false }
+scopeguard = "1.1.0"
+# Use a fork here since we need PAX support, but the upstream
+# does not hav the PR merged yet.
+#
+#tar = "0.4.38"
+tar = { package = "binstall-tar", version = "0.4.39" }
+tempfile = "3.3.0"
+thiserror = "1.0.37"
+tokio = { version = "1.21.2", features = ["macros", "rt-multi-thread", "sync", "time"], default-features = false }
+tower = { version = "0.4.13", features = ["limit", "util"] }
+trust-dns-resolver = { version = "0.21.2", optional = true, default-features = false, features = ["dnssec-ring"] }
+url = "2.3.1"
+
+xz2 = "0.1.7"
+
+# Disable all features of zip except for features of compression algorithms:
+# Disabled features include:
+#  - aes-crypto: Enables decryption of files which were encrypted with AES, absolutely zero use for
+#    this crate.
+#  - time: Enables features using the [time](https://github.com/time-rs/time) crate,
+#    which is not used by this crate.
+zip = { version = "0.6.3", default-features = false, features = ["deflate", "bzip2", "zstd"] }
+
+# zstd is also depended by zip.
+# Since zip 0.6.3 depends on zstd 0.11, we also have to use 0.11 here,
+# otherwise there will be a link conflict.
+zstd = { version = "0.11.2", default-features = false }
+
+[features]
+default = ["static", "rustls"]
+
+static = ["bzip2/static", "xz2/static"]
+pkg-config = ["zstd/pkg-config"]
+
+zlib-ng = ["flate2/zlib-ng"]
+
+rustls = [
+    "reqwest/rustls-tls",
+
+    # Enable the following features only if trust-dns-resolver is enabled.
+    "trust-dns-resolver?/dns-over-rustls",
+    # trust-dns-resolver currently supports https with rustls
+    "trust-dns-resolver?/dns-over-https-rustls",
+]
+native-tls = ["reqwest/native-tls", "trust-dns-resolver?/dns-over-native-tls"]
+
+# Enable trust-dns-resolver so that features on it will also be enabled.
+trust-dns = ["trust-dns-resolver", "reqwest/trust-dns"]

crates/binstalk-downloader/src/download.rs

@@ -0,0 +1,170 @@
+use std::{fmt::Debug, future::Future, io, marker::PhantomData, path::Path, pin::Pin};
+
+use binstalk_manifests::cargo_toml_binstall::{PkgFmtDecomposed, TarBasedFmt};
+use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update};
+use log::debug;
+use thiserror::Error as ThisError;
+
+pub use binstalk_manifests::cargo_toml_binstall::PkgFmt;
+pub use tar::Entries;
+pub use zip::result::ZipError;
+
+use crate::remote::{Client, Error as RemoteError, Url};
+
+mod async_extracter;
+pub use async_extracter::TarEntriesVisitor;
+use async_extracter::*;
+
+mod extracter;
+mod stream_readable;
+
+pub type CancellationFuture = Option<Pin<Box<dyn Future<Output = Result<(), io::Error>> + Send>>>;
+
+#[derive(Debug, ThisError)]
+pub enum DownloadError {
+    #[error(transparent)]
+    Unzip(#[from] ZipError),
+
+    #[error(transparent)]
+    Remote(#[from] RemoteError),
+
+    /// A generic I/O error.
+    ///
+    /// - Code: `binstall::io`
+    /// - Exit: 74
+    #[error(transparent)]
+    Io(io::Error),
+
+    #[error("installation cancelled by user")]
+    UserAbort,
+}
+
+impl From<io::Error> for DownloadError {
+    fn from(err: io::Error) -> Self {
+        if err.get_ref().is_some() {
+            let kind = err.kind();
+
+            let inner = err
+                .into_inner()
+                .expect("err.get_ref() returns Some, so err.into_inner() should also return Some");
+
+            inner
+                .downcast()
+                .map(|b| *b)
+                .unwrap_or_else(|err| DownloadError::Io(io::Error::new(kind, err)))
+        } else {
+            DownloadError::Io(err)
+        }
+    }
+}
+
+impl From<DownloadError> for io::Error {
+    fn from(e: DownloadError) -> io::Error {
+        match e {
+            DownloadError::Io(io_error) => io_error,
+            e => io::Error::new(io::ErrorKind::Other, e),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Download<D: Digest = NoDigest> {
+    client: Client,
+    url: Url,
+    _digest: PhantomData<D>,
+    _checksum: Vec<u8>,
+}
+
+impl Download {
+    pub fn new(client: Client, url: Url) -> Self {
+        Self {
+            client,
+            url,
+            _digest: PhantomData::default(),
+            _checksum: Vec::new(),
+        }
+    }
+
+    /// Download a file from the provided URL and process them in memory.
+    ///
+    /// This does not support verifying a checksum due to the partial extraction
+    /// and will ignore one if specified.
+    ///
+    /// `cancellation_future` can be used to cancel the extraction and return
+    /// [`DownloadError::UserAbort`] error.
+    pub async fn and_visit_tar<V: TarEntriesVisitor + Debug + Send + 'static>(
+        self,
+        fmt: TarBasedFmt,
+        visitor: V,
+        cancellation_future: CancellationFuture,
+    ) -> Result<V::Target, DownloadError> {
+        let stream = self.client.get_stream(self.url).await?;
+
+        debug!("Downloading and extracting then in-memory processing");
+
+        let ret =
+            extract_tar_based_stream_and_visit(stream, fmt, visitor, cancellation_future).await?;
+
+        debug!("Download, extraction and in-memory procession OK");
+
+        Ok(ret)
+    }
+
+    /// Download a file from the provided URL and extract it to the provided path.
+    ///
+    /// `cancellation_future` can be used to cancel the extraction and return
+    /// [`DownloadError::UserAbort`] error.
+    pub async fn and_extract(
+        self,
+        fmt: PkgFmt,
+        path: impl AsRef<Path>,
+        cancellation_future: CancellationFuture,
+    ) -> Result<(), DownloadError> {
+        let stream = self.client.get_stream(self.url).await?;
+
+        let path = path.as_ref();
+        debug!("Downloading and extracting to: '{}'", path.display());
+
+        match fmt.decompose() {
+            PkgFmtDecomposed::Tar(fmt) => {
+                extract_tar_based_stream(stream, path, fmt, cancellation_future).await?
+            }
+            PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?,
+            PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?,
+        }
+
+        debug!("Download OK, extracted to: '{}'", path.display());
+
+        Ok(())
+    }
+}
+
+impl<D: Digest> Download<D> {
+    pub fn new_with_checksum(client: Client, url: Url, checksum: Vec<u8>) -> Self {
+        Self {
+            client,
+            url,
+            _digest: PhantomData::default(),
+            _checksum: checksum,
+        }
+    }
+
+    // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct NoDigest;
+
+impl FixedOutput for NoDigest {
+    fn finalize_into(self, _out: &mut Output<Self>) {}
+}
+
+impl OutputSizeUser for NoDigest {
+    type OutputSize = generic_array::typenum::U0;
+}
+
+impl Update for NoDigest {
+    fn update(&mut self, _data: &[u8]) {}
+}
+
+impl HashMarker for NoDigest {}

crates/binstalk-downloader/src/download/async_extracter.rs

@@ -13,15 +13,20 @@ use tar::Entries;
 use tempfile::tempfile;
 use tokio::task::block_in_place;
 
-use super::{extracter::*, stream_readable::StreamReadable};
-use crate::{errors::BinstallError, manifests::cargo_toml_binstall::TarBasedFmt};
+use super::{
+    extracter::*, stream_readable::StreamReadable, CancellationFuture, DownloadError, TarBasedFmt,
+};
 
-pub async fn extract_bin<S, E>(stream: S, path: &Path) -> Result<(), BinstallError>
+pub async fn extract_bin<S, E>(
+    stream: S,
+    path: &Path,
+    cancellation_future: CancellationFuture,
+) -> Result<(), DownloadError>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin + 'static,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
-    let mut reader = StreamReadable::new(stream).await;
+    let mut reader = StreamReadable::new(stream, cancellation_future).await;
     block_in_place(move || {
         fs::create_dir_all(path.parent().unwrap())?;
 
@@ -43,12 +48,16 @@ where
     })
 }
 
-pub async fn extract_zip<S, E>(stream: S, path: &Path) -> Result<(), BinstallError>
+pub async fn extract_zip<S, E>(
+    stream: S,
+    path: &Path,
+    cancellation_future: CancellationFuture,
+) -> Result<(), DownloadError>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin + 'static,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
-    let mut reader = StreamReadable::new(stream).await;
+    let mut reader = StreamReadable::new(stream, cancellation_future).await;
     block_in_place(move || {
         fs::create_dir_all(path.parent().unwrap())?;
 
@@ -67,12 +76,13 @@ pub async fn extract_tar_based_stream<S, E>(
     stream: S,
     path: &Path,
     fmt: TarBasedFmt,
-) -> Result<(), BinstallError>
+    cancellation_future: CancellationFuture,
+) -> Result<(), DownloadError>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin + 'static,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
-    let reader = StreamReadable::new(stream).await;
+    let reader = StreamReadable::new(stream, cancellation_future).await;
     block_in_place(move || {
         fs::create_dir_all(path.parent().unwrap())?;
 
@@ -89,21 +99,22 @@ where
 pub trait TarEntriesVisitor {
     type Target;
 
-    fn visit<R: Read>(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError>;
-    fn finish(self) -> Result<Self::Target, BinstallError>;
+    fn visit<R: Read>(&mut self, entries: Entries<'_, R>) -> Result<(), DownloadError>;
+    fn finish(self) -> Result<Self::Target, DownloadError>;
 }
 
 pub async fn extract_tar_based_stream_and_visit<S, V, E>(
     stream: S,
     fmt: TarBasedFmt,
     mut visitor: V,
-) -> Result<V::Target, BinstallError>
+    cancellation_future: CancellationFuture,
+) -> Result<V::Target, DownloadError>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin + 'static,
     V: TarEntriesVisitor + Debug + Send + 'static,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
-    let reader = StreamReadable::new(stream).await;
+    let reader = StreamReadable::new(stream, cancellation_future).await;
     block_in_place(move || {
         debug!("Extracting from {fmt} archive to process it in memory");
 

crates/binstalk-downloader/src/download/extracter.rs

@@ -12,7 +12,7 @@ use xz2::bufread::XzDecoder;
 use zip::read::ZipArchive;
 use zstd::stream::Decoder as ZstdDecoder;
 
-use crate::{errors::BinstallError, manifests::cargo_toml_binstall::TarBasedFmt};
+use super::{DownloadError, TarBasedFmt};
 
 pub fn create_tar_decoder(
     dat: impl BufRead + 'static,
@@ -36,7 +36,7 @@ pub fn create_tar_decoder(
     Ok(Archive::new(r))
 }
 
-pub fn unzip(dat: File, dst: &Path) -> Result<(), BinstallError> {
+pub fn unzip(dat: File, dst: &Path) -> Result<(), DownloadError> {
     debug!("Decompressing from zip archive to `{dst:?}`");
 
     let mut zip = ZipArchive::new(dat)?;

crates/binstalk-downloader/src/download/stream_readable.rs

@@ -1,15 +1,13 @@
 use std::{
     cmp::min,
-    future::Future,
     io::{self, BufRead, Read, Write},
-    pin::Pin,
 };
 
 use bytes::{Buf, Bytes};
 use futures_util::stream::{Stream, StreamExt};
 use tokio::runtime::Handle;
 
-use crate::{errors::BinstallError, helpers::signal::wait_on_cancellation_signal};
+use super::{CancellationFuture, DownloadError};
 
 /// This wraps an AsyncIterator as a `Read`able.
 /// It must be used in non-async context only,
@@ -20,16 +18,16 @@ pub struct StreamReadable<S> {
     stream: S,
     handle: Handle,
     bytes: Bytes,
-    cancellation_future: Pin<Box<dyn Future<Output = Result<(), io::Error>> + Send>>,
+    cancellation_future: CancellationFuture,
 }
 
 impl<S> StreamReadable<S> {
-    pub(super) async fn new(stream: S) -> Self {
+    pub(super) async fn new(stream: S, cancellation_future: CancellationFuture) -> Self {
         Self {
             stream,
             handle: Handle::current(),
             bytes: Bytes::new(),
-            cancellation_future: Box::pin(wait_on_cancellation_signal()),
+            cancellation_future,
         }
     }
 }
@@ -37,7 +35,7 @@ impl<S> StreamReadable<S> {
 impl<S, E> StreamReadable<S>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
     /// Copies from `self` to `writer`.
     ///
@@ -69,7 +67,7 @@ where
 impl<S, E> Read for StreamReadable<S>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         if buf.is_empty() {
@@ -96,14 +94,14 @@ where
 async fn next_stream<S, E>(stream: &mut S) -> io::Result<Option<Bytes>>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
     loop {
         let option = stream
             .next()
             .await
             .transpose()
-            .map_err(BinstallError::from)?;
+            .map_err(DownloadError::from)?;
 
         match option {
             Some(bytes) if bytes.is_empty() => continue,
@@ -115,19 +113,23 @@ where
 impl<S, E> BufRead for StreamReadable<S>
 where
     S: Stream<Item = Result<Bytes, E>> + Unpin,
-    BinstallError: From<E>,
+    DownloadError: From<E>,
 {
     fn fill_buf(&mut self) -> io::Result<&[u8]> {
         let bytes = &mut self.bytes;
 
         if !bytes.has_remaining() {
             let option = self.handle.block_on(async {
+                if let Some(cancellation_future) = self.cancellation_future.as_mut() {
                     tokio::select! {
                         res = next_stream(&mut self.stream) => res,
-                    res = self.cancellation_future.as_mut() => {
-                        Err(res.err().unwrap_or_else(|| io::Error::from(BinstallError::UserAbort)))
+                        res = cancellation_future => {
+                            Err(res.err().unwrap_or_else(|| io::Error::from(DownloadError::UserAbort)))
                         },
                     }
+                } else {
+                    next_stream(&mut self.stream).await
+                }
             })?;
 
             if let Some(new_bytes) = option {

crates/binstalk-downloader/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod download;
+pub mod remote;

crates/binstalk-downloader/src/remote.rs

@@ -6,24 +6,41 @@ use std::{
 };
 
 use bytes::Bytes;
-use futures_util::stream::Stream;
+use futures_util::stream::{Stream, StreamExt};
 use httpdate::parse_http_date;
 use log::{debug, info};
 use reqwest::{
     header::{HeaderMap, RETRY_AFTER},
     Request, Response, StatusCode,
 };
+use thiserror::Error as ThisError;
 use tokio::{sync::Mutex, time::sleep};
 use tower::{limit::rate::RateLimit, Service, ServiceBuilder, ServiceExt};
 
-use crate::errors::BinstallError;
-
-pub use reqwest::{tls, Method};
+pub use reqwest::{tls, Error as ReqwestError, Method};
 pub use url::Url;
 
 const MAX_RETRY_DURATION: Duration = Duration::from_secs(120);
 const MAX_RETRY_COUNT: u8 = 3;
 
+#[derive(Debug, ThisError)]
+pub enum Error {
+    #[error(transparent)]
+    Reqwest(#[from] reqwest::Error),
+
+    #[error(transparent)]
+    Http(HttpError),
+}
+
+#[derive(Debug, ThisError)]
+#[error("could not {method} {url}: {err}")]
+pub struct HttpError {
+    method: reqwest::Method,
+    url: url::Url,
+    #[source]
+    err: reqwest::Error,
+}
+
 #[derive(Clone, Debug)]
 pub struct Client {
     client: reqwest::Client,
@@ -32,11 +49,13 @@ pub struct Client {
 
 impl Client {
     /// * `per` - must not be 0.
+    /// * `num_request` - maximum number of requests to be processed for
+    ///   each `per` duration.
     pub fn new(
         min_tls: Option<tls::Version>,
         per: Duration,
         num_request: NonZeroU64,
-    ) -> Result<Self, BinstallError> {
+    ) -> Result<Self, Error> {
         const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"));
 
         let mut builder = reqwest::ClientBuilder::new()
@@ -61,6 +80,7 @@ impl Client {
         })
     }
 
+    /// Return inner reqwest client.
     pub fn get_inner(&self) -> &reqwest::Client {
         &self.client
     }
@@ -69,7 +89,7 @@ impl Client {
         &self,
         method: &Method,
         url: &Url,
-    ) -> Result<Response, reqwest::Error> {
+    ) -> Result<Response, ReqwestError> {
         let mut count = 0;
 
         loop {
@@ -108,7 +128,7 @@ impl Client {
         method: Method,
         url: Url,
         error_for_status: bool,
-    ) -> Result<Response, BinstallError> {
+    ) -> Result<Response, Error> {
         self.send_request_inner(&method, &url)
             .await
             .and_then(|response| {
@@ -118,10 +138,11 @@ impl Client {
                     Ok(response)
                 }
             })
-            .map_err(|err| BinstallError::Http { method, url, err })
+            .map_err(|err| Error::Http(HttpError { method, url, err }))
     }
 
-    pub async fn remote_exists(&self, url: Url, method: Method) -> Result<bool, BinstallError> {
+    /// Check if remote exists using `method`.
+    pub async fn remote_exists(&self, url: Url, method: Method) -> Result<bool, Error> {
         Ok(self
             .send_request(method, url, false)
             .await?
@@ -129,7 +150,8 @@ impl Client {
             .is_success())
     }
 
-    pub async fn get_redirected_final_url(&self, url: Url) -> Result<Url, BinstallError> {
+    /// Attempt to get final redirected url.
+    pub async fn get_redirected_final_url(&self, url: Url) -> Result<Url, Error> {
         Ok(self
             .send_request(Method::HEAD, url, true)
             .await?
@@ -137,15 +159,17 @@ impl Client {
             .clone())
     }
 
-    pub(crate) async fn create_request(
+    /// Create `GET` request to `url` and return a stream of the response data.
+    /// On status code other than 200, it will return an error.
+    pub async fn get_stream(
         &self,
         url: Url,
-    ) -> Result<impl Stream<Item = reqwest::Result<Bytes>>, BinstallError> {
+    ) -> Result<impl Stream<Item = Result<Bytes, Error>>, Error> {
         debug!("Downloading from: '{url}'");
 
         self.send_request(Method::GET, url, true)
             .await
-            .map(Response::bytes_stream)
+            .map(|response| response.bytes_stream().map(|res| res.map_err(Error::from)))
     }
 }
 

crates/binstalk/Cargo.toml

@@ -11,79 +11,43 @@ license = "GPL-3.0"
 
 [dependencies]
 async-trait = "0.1.58"
+binstalk-downloader = { version = "0.1.0", path = "../binstalk-downloader" }
 binstalk-manifests = { version = "0.1.0", path = "../binstalk-manifests" }
-bytes = "1.2.1"
-bzip2 = "0.4.3"
 cargo_toml = "0.13.0"
 compact_str = { version = "0.6.0", features = ["serde"] }
 crates_io_api = { version = "0.8.1", default-features = false }
 detect-targets = { version = "0.1.2", path = "../detect-targets" }
-digest = "0.10.5"
-flate2 = { version = "1.0.24", default-features = false }
 futures-util = { version = "0.3.25", default-features = false, features = ["std"] }
-generic-array = "0.14.6"
 home = "0.5.4"
-httpdate = "1.0.2"
 itertools = "0.10.5"
 jobslot = { version = "0.2.6", features = ["tokio"] }
 log = { version = "0.4.17", features = ["std"] }
 miette = "5.4.1"
 normalize-path = { version = "0.2.0", path = "../normalize-path" }
 once_cell = "1.16.0"
-reqwest = { version = "0.11.12", features = ["stream", "gzip", "brotli", "deflate"], default-features = false }
-scopeguard = "1.1.0"
 semver = { version = "1.0.14", features = ["serde"] }
 serde = { version = "1.0.147", features = ["derive"] }
 strum = "0.24.1"
-# Use a fork here since we need PAX support, but the upstream
-# does not hav the PR merged yet.
-#
-#tar = "0.4.38"
-tar = { package = "binstall-tar", version = "0.4.39" }
 tempfile = "3.3.0"
 thiserror = "1.0.37"
 tinytemplate = "1.2.1"
-# parking_lot - for OnceCell::const_new
-tokio = { version = "1.21.2", features = ["macros", "rt", "process", "sync", "signal", "time", "parking_lot"], default-features = false }
-tower = { version = "0.4.13", features = ["limit", "util"] }
-trust-dns-resolver = { version = "0.21.2", optional = true, default-features = false, features = ["dnssec-ring"] }
+# parking_lot for `tokio::sync::OnceCell::const_new`
+tokio = { version = "1.21.2", features = ["rt", "process", "sync", "signal", "parking_lot"], default-features = false }
 url = { version = "2.3.1", features = ["serde"] }
 xz2 = "0.1.7"
 
-# Disable all features of zip except for features of compression algorithms:
-# Disabled features include:
-#  - aes-crypto: Enables decryption of files which were encrypted with AES, absolutely zero use for
-#    this crate.
-#  - time: Enables features using the [time](https://github.com/time-rs/time) crate,
-#    which is not used by this crate.
-zip = { version = "0.6.3", default-features = false, features = ["deflate", "bzip2", "zstd"] }
-
-# zstd is also depended by zip.
-# Since zip 0.6.3 depends on zstd 0.11, we also have to use 0.11 here,
-# otherwise there will be a link conflict.
-zstd = { version = "0.11.2", default-features = false }
-
 [dev-dependencies]
 env_logger = "0.9.3"
 
 [features]
 default = ["static", "rustls"]
 
-static = ["bzip2/static", "xz2/static"]
-pkg-config = ["zstd/pkg-config"]
+static = ["binstalk-downloader/static"]
+pkg-config = ["binstalk-downloader/pkg-config"]
 
-zlib-ng = ["flate2/zlib-ng"]
+zlib-ng = ["binstalk-downloader/zlib-ng"]
 
-rustls = [
-    "crates_io_api/rustls",
-    "reqwest/rustls-tls",
+rustls = ["crates_io_api/rustls", "binstalk-downloader/rustls"]
+native-tls = ["binstalk-downloader/native-tls"]
 
-    # Enable the following features only if trust-dns-resolver is enabled.
-    "trust-dns-resolver?/dns-over-rustls",
-    # trust-dns-resolver currently supports https with rustls
-    "trust-dns-resolver?/dns-over-https-rustls",
-]
-native-tls = ["reqwest/native-tls", "trust-dns-resolver?/dns-over-native-tls"]
-
-# Enable trust-dns-resolver so that features on it will also be enabled.
-trust-dns = ["trust-dns-resolver", "reqwest/trust-dns"]
+trust-dns = ["binstalk-downloader/trust-dns"]

crates/binstalk/src/drivers/crates_io.rs

@@ -10,6 +10,7 @@ use crate::{
     helpers::{
         download::Download,
         remote::{Client, Url},
+        signal::wait_on_cancellation_signal,
     },
     manifests::cargo_toml_binstall::{Meta, TarBasedFmt},
 };
@@ -53,7 +54,11 @@ pub async fn fetch_crate_cratesio(
 
     let manifest_dir_path: PathBuf = format!("{name}-{version_name}").into();
 
-    Download::new(client, Url::parse(&crate_url)?)
-        .and_visit_tar(TarBasedFmt::Tgz, ManifestVisitor::new(manifest_dir_path))
-        .await
+    Ok(Download::new(client, Url::parse(&crate_url)?)
+        .and_visit_tar(
+            TarBasedFmt::Tgz,
+            ManifestVisitor::new(manifest_dir_path),
+            Some(Box::pin(wait_on_cancellation_signal())),
+        )
+        .await?)
 }

crates/binstalk/src/drivers/crates_io/visitor.rs

@@ -1,16 +1,16 @@
 use std::{
-    io::Read,
+    io::{self, Read},
     path::{Path, PathBuf},
 };
 
 use cargo_toml::Manifest;
 use log::debug;
 use normalize_path::NormalizePath;
-use tar::Entries;
 
 use super::vfs::Vfs;
 use crate::{
-    errors::BinstallError, helpers::download::TarEntriesVisitor,
+    errors::BinstallError,
+    helpers::download::{DownloadError, Entries, TarEntriesVisitor},
     manifests::cargo_toml_binstall::Meta,
 };
 
@@ -37,7 +37,7 @@ impl ManifestVisitor {
 impl TarEntriesVisitor for ManifestVisitor {
     type Target = Manifest<Meta>;
 
-    fn visit<R: Read>(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError> {
+    fn visit<R: Read>(&mut self, entries: Entries<'_, R>) -> Result<(), DownloadError> {
         for res in entries {
             let mut entry = res?;
             let path = entry.path()?;
@@ -71,16 +71,20 @@ impl TarEntriesVisitor for ManifestVisitor {
     }
 
     /// Load binstall metadata using the extracted information stored in memory.
-    fn finish(self) -> Result<Self::Target, BinstallError> {
+    fn finish(self) -> Result<Self::Target, DownloadError> {
+        Ok(load_manifest(&self.cargo_toml_content, &self.vfs).map_err(io::Error::from)?)
+    }
+}
+
+fn load_manifest(slice: &[u8], vfs: &Vfs) -> Result<Manifest<Meta>, BinstallError> {
     debug!("Loading manifest directly from extracted file");
 
     // Load and parse manifest
-        let mut manifest = Manifest::from_slice_with_metadata(&self.cargo_toml_content)?;
+    let mut manifest = Manifest::from_slice_with_metadata(slice)?;
 
     // Checks vfs for binary output names
-        manifest.complete_from_abstract_filesystem(&self.vfs)?;
+    manifest.complete_from_abstract_filesystem(vfs)?;
 
     // Return metadata
     Ok(manifest)
 }
-}

crates/binstalk/src/errors.rs

@@ -4,6 +4,10 @@ use std::{
     process::{ExitCode, ExitStatus, Termination},
 };
 
+use binstalk_downloader::{
+    download::{DownloadError, ZipError},
+    remote::{Error as RemoteError, HttpError, ReqwestError},
+};
 use compact_str::CompactString;
 use miette::{Diagnostic, Report};
 use thiserror::Error;
@@ -47,7 +51,7 @@ pub enum BinstallError {
     /// - Exit: 66
     #[error(transparent)]
     #[diagnostic(severity(error), code(binstall::unzip))]
-    Unzip(#[from] zip::result::ZipError),
+    Unzip(#[from] ZipError),
 
     /// A rendering error in a template.
     ///
@@ -65,7 +69,7 @@ pub enum BinstallError {
     /// - Exit: 68
     #[error(transparent)]
     #[diagnostic(severity(error), code(binstall::reqwest))]
-    Reqwest(#[from] reqwest::Error),
+    Reqwest(#[from] ReqwestError),
 
     /// An HTTP request failed.
     ///
@@ -74,14 +78,9 @@ pub enum BinstallError {
     ///
     /// - Code: `binstall::http`
     /// - Exit: 69
-    #[error("could not {method} {url}")]
+    #[error(transparent)]
     #[diagnostic(severity(error), code(binstall::http))]
-    Http {
-        method: reqwest::Method,
-        url: url::Url,
-        #[source]
-        err: reqwest::Error,
-    },
+    Http(#[from] HttpError),
 
     /// A subprocess failed.
     ///
@@ -418,3 +417,27 @@ impl From<BinstallError> for io::Error {
         }
     }
 }
+
+impl From<RemoteError> for BinstallError {
+    fn from(e: RemoteError) -> Self {
+        use RemoteError::*;
+
+        match e {
+            Reqwest(reqwest_error) => reqwest_error.into(),
+            Http(http_error) => http_error.into(),
+        }
+    }
+}
+
+impl From<DownloadError> for BinstallError {
+    fn from(e: DownloadError) -> Self {
+        use DownloadError::*;
+
+        match e {
+            Unzip(zip_error) => zip_error.into(),
+            Remote(remote_error) => remote_error.into(),
+            Io(io_error) => io_error.into(),
+            UserAbort => BinstallError::UserAbort,
+        }
+    }
+}

crates/binstalk/src/fetchers/gh_crate_meta.rs

@@ -14,6 +14,7 @@ use crate::{
     helpers::{
         download::Download,
         remote::{Client, Method},
+        signal::wait_on_cancellation_signal,
         tasks::AutoAbortJoinHandle,
     },
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
@@ -146,9 +147,9 @@ impl super::Fetcher for GhCrateMeta {
     async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> {
         let (url, pkg_fmt) = self.resolution.get().unwrap(); // find() is called first
         debug!("Downloading package from: '{url}' dst:{dst:?} fmt:{pkg_fmt:?}");
-        Download::new(self.client.clone(), url.clone())
-            .and_extract(*pkg_fmt, dst)
-            .await
+        Ok(Download::new(self.client.clone(), url.clone())
+            .and_extract(*pkg_fmt, dst, Some(Box::pin(wait_on_cancellation_signal())))
+            .await?)
     }
 
     fn pkg_fmt(&self) -> PkgFmt {

crates/binstalk/src/fetchers/quickinstall.rs

@@ -10,6 +10,7 @@ use crate::{
     helpers::{
         download::Download,
         remote::{Client, Method},
+        signal::wait_on_cancellation_signal,
     },
     manifests::cargo_toml_binstall::{PkgFmt, PkgMeta},
 };
@@ -44,17 +45,22 @@ impl super::Fetcher for QuickInstall {
         let url = self.package_url();
         self.report();
         debug!("Checking for package at: '{url}'");
-        self.client
+        Ok(self
+            .client
             .remote_exists(Url::parse(&url)?, Method::HEAD)
-            .await
+            .await?)
     }
 
     async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> {
         let url = self.package_url();
         debug!("Downloading package from: '{url}'");
-        Download::new(self.client.clone(), Url::parse(&url)?)
-            .and_extract(self.pkg_fmt(), dst)
-            .await
+        Ok(Download::new(self.client.clone(), Url::parse(&url)?)
+            .and_extract(
+                self.pkg_fmt(),
+                dst,
+                Some(Box::pin(wait_on_cancellation_signal())),
+            )
+            .await?)
     }
 
     fn pkg_fmt(&self) -> PkgFmt {

crates/binstalk/src/helpers.rs

@@ -1,5 +1,5 @@
-pub mod download;
 pub mod jobserver_client;
-pub mod remote;
 pub mod signal;
 pub mod tasks;
+
+pub use binstalk_downloader::{download, remote};

crates/binstalk/src/helpers/download.rs

@@ -1,112 +0,0 @@
-use std::{fmt::Debug, marker::PhantomData, path::Path};
-
-use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update};
-use log::debug;
-
-use crate::{
-    errors::BinstallError,
-    helpers::remote::{Client, Url},
-    manifests::cargo_toml_binstall::{PkgFmt, PkgFmtDecomposed, TarBasedFmt},
-};
-
-pub use async_extracter::TarEntriesVisitor;
-use async_extracter::*;
-
-mod async_extracter;
-mod extracter;
-mod stream_readable;
-
-#[derive(Debug)]
-pub struct Download<D: Digest = NoDigest> {
-    client: Client,
-    url: Url,
-    _digest: PhantomData<D>,
-    _checksum: Vec<u8>,
-}
-
-impl Download {
-    pub fn new(client: Client, url: Url) -> Self {
-        Self {
-            client,
-            url,
-            _digest: PhantomData::default(),
-            _checksum: Vec::new(),
-        }
-    }
-
-    /// Download a file from the provided URL and extract part of it to
-    /// the provided path.
-    ///
-    ///  * `filter` - If Some, then it will pass the path of the file to it
-    ///    and only extract ones which filter returns `true`.
-    ///
-    /// This does not support verifying a checksum due to the partial extraction
-    /// and will ignore one if specified.
-    pub async fn and_visit_tar<V: TarEntriesVisitor + Debug + Send + 'static>(
-        self,
-        fmt: TarBasedFmt,
-        visitor: V,
-    ) -> Result<V::Target, BinstallError> {
-        let stream = self.client.create_request(self.url).await?;
-
-        debug!("Downloading and extracting then in-memory processing");
-
-        let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?;
-
-        debug!("Download, extraction and in-memory procession OK");
-
-        Ok(ret)
-    }
-
-    /// Download a file from the provided URL and extract it to the provided path.
-    pub async fn and_extract(
-        self,
-        fmt: PkgFmt,
-        path: impl AsRef<Path>,
-    ) -> Result<(), BinstallError> {
-        let stream = self.client.create_request(self.url).await?;
-
-        let path = path.as_ref();
-        debug!("Downloading and extracting to: '{}'", path.display());
-
-        match fmt.decompose() {
-            PkgFmtDecomposed::Tar(fmt) => extract_tar_based_stream(stream, path, fmt).await?,
-            PkgFmtDecomposed::Bin => extract_bin(stream, path).await?,
-            PkgFmtDecomposed::Zip => extract_zip(stream, path).await?,
-        }
-
-        debug!("Download OK, extracted to: '{}'", path.display());
-
-        Ok(())
-    }
-}
-
-impl<D: Digest> Download<D> {
-    pub fn new_with_checksum(client: Client, url: Url, checksum: Vec<u8>) -> Self {
-        Self {
-            client,
-            url,
-            _digest: PhantomData::default(),
-            _checksum: checksum,
-        }
-    }
-
-    // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here
-}
-
-#[derive(Clone, Copy, Debug, Default)]
-pub struct NoDigest;
-
-impl FixedOutput for NoDigest {
-    fn finalize_into(self, _out: &mut Output<Self>) {}
-}
-
-impl OutputSizeUser for NoDigest {
-    type OutputSize = generic_array::typenum::U0;
-}
-
-impl Update for NoDigest {
-    fn update(&mut self, _data: &[u8]) {}
-}
-
-impl HashMarker for NoDigest {}

crates/binstalk/src/helpers/signal.rs

@@ -1,11 +1,10 @@
-use std::io;
-
-use futures_util::future::pending;
-use tokio::{signal, sync::OnceCell};
+use std::{future::pending, io};
 
 use super::tasks::AutoAbortJoinHandle;
 use crate::errors::BinstallError;
 
+use tokio::{signal, sync::OnceCell};
+
 /// This function will poll the handle while listening for ctrl_c,
 /// `SIGINT`, `SIGHUP`, `SIGTERM` and `SIGQUIT`.
 ///
@@ -18,19 +17,24 @@ use crate::errors::BinstallError;
 pub async fn cancel_on_user_sig_term<T>(
     handle: AutoAbortJoinHandle<T>,
 ) -> Result<T, BinstallError> {
-    #[cfg(unix)]
-    unix::ignore_signals_on_unix()?;
+    ignore_signals()?;
 
     tokio::select! {
         res = handle => res,
         res = wait_on_cancellation_signal() => {
-            res
-        .map_err(BinstallError::Io)
+            res.map_err(BinstallError::Io)
                 .and(Err(BinstallError::UserAbort))
         }
     }
 }
 
+pub fn ignore_signals() -> io::Result<()> {
+    #[cfg(unix)]
+    unix::ignore_signals_on_unix()?;
+
+    Ok(())
+}
+
 /// If call to it returns `Ok(())`, then all calls to this function after
 /// that also returns `Ok(())`.
 pub async fn wait_on_cancellation_signal() -> Result<(), io::Error> {
@@ -86,7 +90,7 @@ mod unix {
         }
     }
 
-    pub fn ignore_signals_on_unix() -> Result<(), BinstallError> {
+    pub fn ignore_signals_on_unix() -> Result<(), io::Error> {
         drop(signal(SignalKind::user_defined1())?);
         drop(signal(SignalKind::user_defined2())?);