diff --git a/.github/dependabot.yml b/.github/dependabot.yml index db3b2fd1..f293064b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -24,6 +24,10 @@ updates: directory: "/crates/binstalk" schedule: interval: "daily" + - package-ecosystem: "cargo" + directory: "/crates/binstalk-downloader" + schedule: + interval: "daily" - package-ecosystem: "cargo" directory: "/crates/detect-wasi" schedule: diff --git a/.github/workflows/release-pr.yml b/.github/workflows/release-pr.yml index c3dca191..3d126b3c 100644 --- a/.github/workflows/release-pr.yml +++ b/.github/workflows/release-pr.yml @@ -10,6 +10,7 @@ on: - bin - binstalk - binstalk-manifests + - binstalk-downloader - detect-targets - detect-wasi - fs-lock diff --git a/Cargo.lock b/Cargo.lock index 727df5f0..a6d0c038 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,29 +109,21 @@ name = "binstalk" version = "0.4.1" dependencies = [ "async-trait", + "binstalk-downloader", "binstalk-manifests", - "binstall-tar", - "bytes", - "bzip2", "cargo_toml", "compact_str", "crates_io_api", "detect-targets", - "digest", "env_logger", - "flate2", "futures-util", - "generic-array", "home", - "httpdate", "itertools", "jobslot", "log", "miette", "normalize-path", "once_cell", - "reqwest", - "scopeguard", "semver", "serde", "strum", @@ -139,6 +131,29 @@ dependencies = [ "thiserror", "tinytemplate", "tokio", + "url", + "xz2", +] + +[[package]] +name = "binstalk-downloader" +version = "0.1.0" +dependencies = [ + "binstalk-manifests", + "binstall-tar", + "bytes", + "bzip2", + "digest", + "flate2", + "futures-util", + "generic-array", + "httpdate", + "log", + "reqwest", + "scopeguard", + "tempfile", + "thiserror", + "tokio", "tower", "trust-dns-resolver", "url", diff --git a/Cargo.toml b/Cargo.toml index 38ccdc7d..bd0cb700 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/bin", "crates/binstalk", "crates/binstalk-manifests", + "crates/binstalk-downloader", "crates/detect-wasi", "crates/fs-lock", "crates/normalize-path", diff --git a/crates/bin/src/entry.rs b/crates/bin/src/entry.rs index b30fddee..4518694b 100644 --- a/crates/bin/src/entry.rs +++ b/crates/bin/src/entry.rs @@ -35,7 +35,8 @@ pub async fn install_crates(mut args: Args, jobserver_client: LazyJobserverClien args.min_tls_version.map(|v| v.into()), Duration::from_millis(rate_limit.duration.get()), rate_limit.request_count, - )?; + ) + .map_err(BinstallError::from)?; // Build crates.io api client let crates_io_api_client = crates_io_api::AsyncClient::with_http_client( diff --git a/crates/binstalk-downloader/Cargo.toml b/crates/binstalk-downloader/Cargo.toml new file mode 100644 index 00000000..bbc18ba7 --- /dev/null +++ b/crates/binstalk-downloader/Cargo.toml @@ -0,0 +1,70 @@ +[package] +name = "binstalk-downloader" +description = "The binstall toolkit for downloading and extracting file" +repository = "https://github.com/cargo-bins/cargo-binstall" +documentation = "https://docs.rs/binstalk-downloader" +version = "0.1.0" +rust-version = "1.61.0" +authors = ["ryan "] +edition = "2021" +license = "GPL-3.0" + +[dependencies] +binstalk-manifests = { version = "0.1.0", path = "../binstalk-manifests" } +bytes = "1.2.1" +bzip2 = "0.4.3" +digest = "0.10.5" +flate2 = { version = "1.0.24", default-features = false } +futures-util = { version = "0.3.25", default-features = false, features = ["std"] } +generic-array = "0.14.6" +httpdate = "1.0.2" +log = { version = "0.4.17", features = ["std"] } +reqwest = { version = "0.11.12", features = ["stream", "gzip", "brotli", "deflate"], default-features = false } +scopeguard = "1.1.0" +# Use a fork here since we need PAX support, but the upstream +# does not hav the PR merged yet. +# +#tar = "0.4.38" +tar = { package = "binstall-tar", version = "0.4.39" } +tempfile = "3.3.0" +thiserror = "1.0.37" +tokio = { version = "1.21.2", features = ["macros", "rt-multi-thread", "sync", "time"], default-features = false } +tower = { version = "0.4.13", features = ["limit", "util"] } +trust-dns-resolver = { version = "0.21.2", optional = true, default-features = false, features = ["dnssec-ring"] } +url = "2.3.1" + +xz2 = "0.1.7" + +# Disable all features of zip except for features of compression algorithms: +# Disabled features include: +# - aes-crypto: Enables decryption of files which were encrypted with AES, absolutely zero use for +# this crate. +# - time: Enables features using the [time](https://github.com/time-rs/time) crate, +# which is not used by this crate. +zip = { version = "0.6.3", default-features = false, features = ["deflate", "bzip2", "zstd"] } + +# zstd is also depended by zip. +# Since zip 0.6.3 depends on zstd 0.11, we also have to use 0.11 here, +# otherwise there will be a link conflict. +zstd = { version = "0.11.2", default-features = false } + +[features] +default = ["static", "rustls"] + +static = ["bzip2/static", "xz2/static"] +pkg-config = ["zstd/pkg-config"] + +zlib-ng = ["flate2/zlib-ng"] + +rustls = [ + "reqwest/rustls-tls", + + # Enable the following features only if trust-dns-resolver is enabled. + "trust-dns-resolver?/dns-over-rustls", + # trust-dns-resolver currently supports https with rustls + "trust-dns-resolver?/dns-over-https-rustls", +] +native-tls = ["reqwest/native-tls", "trust-dns-resolver?/dns-over-native-tls"] + +# Enable trust-dns-resolver so that features on it will also be enabled. +trust-dns = ["trust-dns-resolver", "reqwest/trust-dns"] diff --git a/crates/binstalk-downloader/src/download.rs b/crates/binstalk-downloader/src/download.rs new file mode 100644 index 00000000..6b3985e1 --- /dev/null +++ b/crates/binstalk-downloader/src/download.rs @@ -0,0 +1,170 @@ +use std::{fmt::Debug, future::Future, io, marker::PhantomData, path::Path, pin::Pin}; + +use binstalk_manifests::cargo_toml_binstall::{PkgFmtDecomposed, TarBasedFmt}; +use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update}; +use log::debug; +use thiserror::Error as ThisError; + +pub use binstalk_manifests::cargo_toml_binstall::PkgFmt; +pub use tar::Entries; +pub use zip::result::ZipError; + +use crate::remote::{Client, Error as RemoteError, Url}; + +mod async_extracter; +pub use async_extracter::TarEntriesVisitor; +use async_extracter::*; + +mod extracter; +mod stream_readable; + +pub type CancellationFuture = Option> + Send>>>; + +#[derive(Debug, ThisError)] +pub enum DownloadError { + #[error(transparent)] + Unzip(#[from] ZipError), + + #[error(transparent)] + Remote(#[from] RemoteError), + + /// A generic I/O error. + /// + /// - Code: `binstall::io` + /// - Exit: 74 + #[error(transparent)] + Io(io::Error), + + #[error("installation cancelled by user")] + UserAbort, +} + +impl From for DownloadError { + fn from(err: io::Error) -> Self { + if err.get_ref().is_some() { + let kind = err.kind(); + + let inner = err + .into_inner() + .expect("err.get_ref() returns Some, so err.into_inner() should also return Some"); + + inner + .downcast() + .map(|b| *b) + .unwrap_or_else(|err| DownloadError::Io(io::Error::new(kind, err))) + } else { + DownloadError::Io(err) + } + } +} + +impl From for io::Error { + fn from(e: DownloadError) -> io::Error { + match e { + DownloadError::Io(io_error) => io_error, + e => io::Error::new(io::ErrorKind::Other, e), + } + } +} + +#[derive(Debug)] +pub struct Download { + client: Client, + url: Url, + _digest: PhantomData, + _checksum: Vec, +} + +impl Download { + pub fn new(client: Client, url: Url) -> Self { + Self { + client, + url, + _digest: PhantomData::default(), + _checksum: Vec::new(), + } + } + + /// Download a file from the provided URL and process them in memory. + /// + /// This does not support verifying a checksum due to the partial extraction + /// and will ignore one if specified. + /// + /// `cancellation_future` can be used to cancel the extraction and return + /// [`DownloadError::UserAbort`] error. + pub async fn and_visit_tar( + self, + fmt: TarBasedFmt, + visitor: V, + cancellation_future: CancellationFuture, + ) -> Result { + let stream = self.client.get_stream(self.url).await?; + + debug!("Downloading and extracting then in-memory processing"); + + let ret = + extract_tar_based_stream_and_visit(stream, fmt, visitor, cancellation_future).await?; + + debug!("Download, extraction and in-memory procession OK"); + + Ok(ret) + } + + /// Download a file from the provided URL and extract it to the provided path. + /// + /// `cancellation_future` can be used to cancel the extraction and return + /// [`DownloadError::UserAbort`] error. + pub async fn and_extract( + self, + fmt: PkgFmt, + path: impl AsRef, + cancellation_future: CancellationFuture, + ) -> Result<(), DownloadError> { + let stream = self.client.get_stream(self.url).await?; + + let path = path.as_ref(); + debug!("Downloading and extracting to: '{}'", path.display()); + + match fmt.decompose() { + PkgFmtDecomposed::Tar(fmt) => { + extract_tar_based_stream(stream, path, fmt, cancellation_future).await? + } + PkgFmtDecomposed::Bin => extract_bin(stream, path, cancellation_future).await?, + PkgFmtDecomposed::Zip => extract_zip(stream, path, cancellation_future).await?, + } + + debug!("Download OK, extracted to: '{}'", path.display()); + + Ok(()) + } +} + +impl Download { + pub fn new_with_checksum(client: Client, url: Url, checksum: Vec) -> Self { + Self { + client, + url, + _digest: PhantomData::default(), + _checksum: checksum, + } + } + + // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct NoDigest; + +impl FixedOutput for NoDigest { + fn finalize_into(self, _out: &mut Output) {} +} + +impl OutputSizeUser for NoDigest { + type OutputSize = generic_array::typenum::U0; +} + +impl Update for NoDigest { + fn update(&mut self, _data: &[u8]) {} +} + +impl HashMarker for NoDigest {} diff --git a/crates/binstalk/src/helpers/download/async_extracter.rs b/crates/binstalk-downloader/src/download/async_extracter.rs similarity index 68% rename from crates/binstalk/src/helpers/download/async_extracter.rs rename to crates/binstalk-downloader/src/download/async_extracter.rs index 21311b3d..bcaaa925 100644 --- a/crates/binstalk/src/helpers/download/async_extracter.rs +++ b/crates/binstalk-downloader/src/download/async_extracter.rs @@ -13,15 +13,20 @@ use tar::Entries; use tempfile::tempfile; use tokio::task::block_in_place; -use super::{extracter::*, stream_readable::StreamReadable}; -use crate::{errors::BinstallError, manifests::cargo_toml_binstall::TarBasedFmt}; +use super::{ + extracter::*, stream_readable::StreamReadable, CancellationFuture, DownloadError, TarBasedFmt, +}; -pub async fn extract_bin(stream: S, path: &Path) -> Result<(), BinstallError> +pub async fn extract_bin( + stream: S, + path: &Path, + cancellation_future: CancellationFuture, +) -> Result<(), DownloadError> where S: Stream> + Unpin + 'static, - BinstallError: From, + DownloadError: From, { - let mut reader = StreamReadable::new(stream).await; + let mut reader = StreamReadable::new(stream, cancellation_future).await; block_in_place(move || { fs::create_dir_all(path.parent().unwrap())?; @@ -43,12 +48,16 @@ where }) } -pub async fn extract_zip(stream: S, path: &Path) -> Result<(), BinstallError> +pub async fn extract_zip( + stream: S, + path: &Path, + cancellation_future: CancellationFuture, +) -> Result<(), DownloadError> where S: Stream> + Unpin + 'static, - BinstallError: From, + DownloadError: From, { - let mut reader = StreamReadable::new(stream).await; + let mut reader = StreamReadable::new(stream, cancellation_future).await; block_in_place(move || { fs::create_dir_all(path.parent().unwrap())?; @@ -67,12 +76,13 @@ pub async fn extract_tar_based_stream( stream: S, path: &Path, fmt: TarBasedFmt, -) -> Result<(), BinstallError> + cancellation_future: CancellationFuture, +) -> Result<(), DownloadError> where S: Stream> + Unpin + 'static, - BinstallError: From, + DownloadError: From, { - let reader = StreamReadable::new(stream).await; + let reader = StreamReadable::new(stream, cancellation_future).await; block_in_place(move || { fs::create_dir_all(path.parent().unwrap())?; @@ -89,21 +99,22 @@ where pub trait TarEntriesVisitor { type Target; - fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError>; - fn finish(self) -> Result; + fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), DownloadError>; + fn finish(self) -> Result; } pub async fn extract_tar_based_stream_and_visit( stream: S, fmt: TarBasedFmt, mut visitor: V, -) -> Result + cancellation_future: CancellationFuture, +) -> Result where S: Stream> + Unpin + 'static, V: TarEntriesVisitor + Debug + Send + 'static, - BinstallError: From, + DownloadError: From, { - let reader = StreamReadable::new(stream).await; + let reader = StreamReadable::new(stream, cancellation_future).await; block_in_place(move || { debug!("Extracting from {fmt} archive to process it in memory"); diff --git a/crates/binstalk/src/helpers/download/extracter.rs b/crates/binstalk-downloader/src/download/extracter.rs similarity index 88% rename from crates/binstalk/src/helpers/download/extracter.rs rename to crates/binstalk-downloader/src/download/extracter.rs index 86ebe5f5..096eca79 100644 --- a/crates/binstalk/src/helpers/download/extracter.rs +++ b/crates/binstalk-downloader/src/download/extracter.rs @@ -12,7 +12,7 @@ use xz2::bufread::XzDecoder; use zip::read::ZipArchive; use zstd::stream::Decoder as ZstdDecoder; -use crate::{errors::BinstallError, manifests::cargo_toml_binstall::TarBasedFmt}; +use super::{DownloadError, TarBasedFmt}; pub fn create_tar_decoder( dat: impl BufRead + 'static, @@ -36,7 +36,7 @@ pub fn create_tar_decoder( Ok(Archive::new(r)) } -pub fn unzip(dat: File, dst: &Path) -> Result<(), BinstallError> { +pub fn unzip(dat: File, dst: &Path) -> Result<(), DownloadError> { debug!("Decompressing from zip archive to `{dst:?}`"); let mut zip = ZipArchive::new(dat)?; diff --git a/crates/binstalk/src/helpers/download/stream_readable.rs b/crates/binstalk-downloader/src/download/stream_readable.rs similarity index 77% rename from crates/binstalk/src/helpers/download/stream_readable.rs rename to crates/binstalk-downloader/src/download/stream_readable.rs index 6685c6bf..af6e9c67 100644 --- a/crates/binstalk/src/helpers/download/stream_readable.rs +++ b/crates/binstalk-downloader/src/download/stream_readable.rs @@ -1,15 +1,13 @@ use std::{ cmp::min, - future::Future, io::{self, BufRead, Read, Write}, - pin::Pin, }; use bytes::{Buf, Bytes}; use futures_util::stream::{Stream, StreamExt}; use tokio::runtime::Handle; -use crate::{errors::BinstallError, helpers::signal::wait_on_cancellation_signal}; +use super::{CancellationFuture, DownloadError}; /// This wraps an AsyncIterator as a `Read`able. /// It must be used in non-async context only, @@ -20,16 +18,16 @@ pub struct StreamReadable { stream: S, handle: Handle, bytes: Bytes, - cancellation_future: Pin> + Send>>, + cancellation_future: CancellationFuture, } impl StreamReadable { - pub(super) async fn new(stream: S) -> Self { + pub(super) async fn new(stream: S, cancellation_future: CancellationFuture) -> Self { Self { stream, handle: Handle::current(), bytes: Bytes::new(), - cancellation_future: Box::pin(wait_on_cancellation_signal()), + cancellation_future, } } } @@ -37,7 +35,7 @@ impl StreamReadable { impl StreamReadable where S: Stream> + Unpin, - BinstallError: From, + DownloadError: From, { /// Copies from `self` to `writer`. /// @@ -69,7 +67,7 @@ where impl Read for StreamReadable where S: Stream> + Unpin, - BinstallError: From, + DownloadError: From, { fn read(&mut self, buf: &mut [u8]) -> io::Result { if buf.is_empty() { @@ -96,14 +94,14 @@ where async fn next_stream(stream: &mut S) -> io::Result> where S: Stream> + Unpin, - BinstallError: From, + DownloadError: From, { loop { let option = stream .next() .await .transpose() - .map_err(BinstallError::from)?; + .map_err(DownloadError::from)?; match option { Some(bytes) if bytes.is_empty() => continue, @@ -115,18 +113,22 @@ where impl BufRead for StreamReadable where S: Stream> + Unpin, - BinstallError: From, + DownloadError: From, { fn fill_buf(&mut self) -> io::Result<&[u8]> { let bytes = &mut self.bytes; if !bytes.has_remaining() { let option = self.handle.block_on(async { - tokio::select! { - res = next_stream(&mut self.stream) => res, - res = self.cancellation_future.as_mut() => { - Err(res.err().unwrap_or_else(|| io::Error::from(BinstallError::UserAbort))) - }, + if let Some(cancellation_future) = self.cancellation_future.as_mut() { + tokio::select! { + res = next_stream(&mut self.stream) => res, + res = cancellation_future => { + Err(res.err().unwrap_or_else(|| io::Error::from(DownloadError::UserAbort))) + }, + } + } else { + next_stream(&mut self.stream).await } })?; diff --git a/crates/binstalk-downloader/src/lib.rs b/crates/binstalk-downloader/src/lib.rs new file mode 100644 index 00000000..abb27a74 --- /dev/null +++ b/crates/binstalk-downloader/src/lib.rs @@ -0,0 +1,2 @@ +pub mod download; +pub mod remote; diff --git a/crates/binstalk/src/helpers/remote.rs b/crates/binstalk-downloader/src/remote.rs similarity index 78% rename from crates/binstalk/src/helpers/remote.rs rename to crates/binstalk-downloader/src/remote.rs index f1ce9f6a..bfe68409 100644 --- a/crates/binstalk/src/helpers/remote.rs +++ b/crates/binstalk-downloader/src/remote.rs @@ -6,24 +6,41 @@ use std::{ }; use bytes::Bytes; -use futures_util::stream::Stream; +use futures_util::stream::{Stream, StreamExt}; use httpdate::parse_http_date; use log::{debug, info}; use reqwest::{ header::{HeaderMap, RETRY_AFTER}, Request, Response, StatusCode, }; +use thiserror::Error as ThisError; use tokio::{sync::Mutex, time::sleep}; use tower::{limit::rate::RateLimit, Service, ServiceBuilder, ServiceExt}; -use crate::errors::BinstallError; - -pub use reqwest::{tls, Method}; +pub use reqwest::{tls, Error as ReqwestError, Method}; pub use url::Url; const MAX_RETRY_DURATION: Duration = Duration::from_secs(120); const MAX_RETRY_COUNT: u8 = 3; +#[derive(Debug, ThisError)] +pub enum Error { + #[error(transparent)] + Reqwest(#[from] reqwest::Error), + + #[error(transparent)] + Http(HttpError), +} + +#[derive(Debug, ThisError)] +#[error("could not {method} {url}: {err}")] +pub struct HttpError { + method: reqwest::Method, + url: url::Url, + #[source] + err: reqwest::Error, +} + #[derive(Clone, Debug)] pub struct Client { client: reqwest::Client, @@ -32,11 +49,13 @@ pub struct Client { impl Client { /// * `per` - must not be 0. + /// * `num_request` - maximum number of requests to be processed for + /// each `per` duration. pub fn new( min_tls: Option, per: Duration, num_request: NonZeroU64, - ) -> Result { + ) -> Result { const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); let mut builder = reqwest::ClientBuilder::new() @@ -61,6 +80,7 @@ impl Client { }) } + /// Return inner reqwest client. pub fn get_inner(&self) -> &reqwest::Client { &self.client } @@ -69,7 +89,7 @@ impl Client { &self, method: &Method, url: &Url, - ) -> Result { + ) -> Result { let mut count = 0; loop { @@ -108,7 +128,7 @@ impl Client { method: Method, url: Url, error_for_status: bool, - ) -> Result { + ) -> Result { self.send_request_inner(&method, &url) .await .and_then(|response| { @@ -118,10 +138,11 @@ impl Client { Ok(response) } }) - .map_err(|err| BinstallError::Http { method, url, err }) + .map_err(|err| Error::Http(HttpError { method, url, err })) } - pub async fn remote_exists(&self, url: Url, method: Method) -> Result { + /// Check if remote exists using `method`. + pub async fn remote_exists(&self, url: Url, method: Method) -> Result { Ok(self .send_request(method, url, false) .await? @@ -129,7 +150,8 @@ impl Client { .is_success()) } - pub async fn get_redirected_final_url(&self, url: Url) -> Result { + /// Attempt to get final redirected url. + pub async fn get_redirected_final_url(&self, url: Url) -> Result { Ok(self .send_request(Method::HEAD, url, true) .await? @@ -137,15 +159,17 @@ impl Client { .clone()) } - pub(crate) async fn create_request( + /// Create `GET` request to `url` and return a stream of the response data. + /// On status code other than 200, it will return an error. + pub async fn get_stream( &self, url: Url, - ) -> Result>, BinstallError> { + ) -> Result>, Error> { debug!("Downloading from: '{url}'"); self.send_request(Method::GET, url, true) .await - .map(Response::bytes_stream) + .map(|response| response.bytes_stream().map(|res| res.map_err(Error::from))) } } diff --git a/crates/binstalk/Cargo.toml b/crates/binstalk/Cargo.toml index 832785c9..759cb110 100644 --- a/crates/binstalk/Cargo.toml +++ b/crates/binstalk/Cargo.toml @@ -11,79 +11,43 @@ license = "GPL-3.0" [dependencies] async-trait = "0.1.58" +binstalk-downloader = { version = "0.1.0", path = "../binstalk-downloader" } binstalk-manifests = { version = "0.1.0", path = "../binstalk-manifests" } -bytes = "1.2.1" -bzip2 = "0.4.3" cargo_toml = "0.13.0" compact_str = { version = "0.6.0", features = ["serde"] } crates_io_api = { version = "0.8.1", default-features = false } detect-targets = { version = "0.1.2", path = "../detect-targets" } -digest = "0.10.5" -flate2 = { version = "1.0.24", default-features = false } futures-util = { version = "0.3.25", default-features = false, features = ["std"] } -generic-array = "0.14.6" home = "0.5.4" -httpdate = "1.0.2" itertools = "0.10.5" jobslot = { version = "0.2.6", features = ["tokio"] } log = { version = "0.4.17", features = ["std"] } miette = "5.4.1" normalize-path = { version = "0.2.0", path = "../normalize-path" } once_cell = "1.16.0" -reqwest = { version = "0.11.12", features = ["stream", "gzip", "brotli", "deflate"], default-features = false } -scopeguard = "1.1.0" semver = { version = "1.0.14", features = ["serde"] } serde = { version = "1.0.147", features = ["derive"] } strum = "0.24.1" -# Use a fork here since we need PAX support, but the upstream -# does not hav the PR merged yet. -# -#tar = "0.4.38" -tar = { package = "binstall-tar", version = "0.4.39" } tempfile = "3.3.0" thiserror = "1.0.37" tinytemplate = "1.2.1" -# parking_lot - for OnceCell::const_new -tokio = { version = "1.21.2", features = ["macros", "rt", "process", "sync", "signal", "time", "parking_lot"], default-features = false } -tower = { version = "0.4.13", features = ["limit", "util"] } -trust-dns-resolver = { version = "0.21.2", optional = true, default-features = false, features = ["dnssec-ring"] } +# parking_lot for `tokio::sync::OnceCell::const_new` +tokio = { version = "1.21.2", features = ["rt", "process", "sync", "signal", "parking_lot"], default-features = false } url = { version = "2.3.1", features = ["serde"] } xz2 = "0.1.7" -# Disable all features of zip except for features of compression algorithms: -# Disabled features include: -# - aes-crypto: Enables decryption of files which were encrypted with AES, absolutely zero use for -# this crate. -# - time: Enables features using the [time](https://github.com/time-rs/time) crate, -# which is not used by this crate. -zip = { version = "0.6.3", default-features = false, features = ["deflate", "bzip2", "zstd"] } - -# zstd is also depended by zip. -# Since zip 0.6.3 depends on zstd 0.11, we also have to use 0.11 here, -# otherwise there will be a link conflict. -zstd = { version = "0.11.2", default-features = false } - [dev-dependencies] env_logger = "0.9.3" [features] default = ["static", "rustls"] -static = ["bzip2/static", "xz2/static"] -pkg-config = ["zstd/pkg-config"] +static = ["binstalk-downloader/static"] +pkg-config = ["binstalk-downloader/pkg-config"] -zlib-ng = ["flate2/zlib-ng"] +zlib-ng = ["binstalk-downloader/zlib-ng"] -rustls = [ - "crates_io_api/rustls", - "reqwest/rustls-tls", +rustls = ["crates_io_api/rustls", "binstalk-downloader/rustls"] +native-tls = ["binstalk-downloader/native-tls"] - # Enable the following features only if trust-dns-resolver is enabled. - "trust-dns-resolver?/dns-over-rustls", - # trust-dns-resolver currently supports https with rustls - "trust-dns-resolver?/dns-over-https-rustls", -] -native-tls = ["reqwest/native-tls", "trust-dns-resolver?/dns-over-native-tls"] - -# Enable trust-dns-resolver so that features on it will also be enabled. -trust-dns = ["trust-dns-resolver", "reqwest/trust-dns"] +trust-dns = ["binstalk-downloader/trust-dns"] diff --git a/crates/binstalk/src/drivers/crates_io.rs b/crates/binstalk/src/drivers/crates_io.rs index aa7f0abe..6fbcbec8 100644 --- a/crates/binstalk/src/drivers/crates_io.rs +++ b/crates/binstalk/src/drivers/crates_io.rs @@ -10,6 +10,7 @@ use crate::{ helpers::{ download::Download, remote::{Client, Url}, + signal::wait_on_cancellation_signal, }, manifests::cargo_toml_binstall::{Meta, TarBasedFmt}, }; @@ -53,7 +54,11 @@ pub async fn fetch_crate_cratesio( let manifest_dir_path: PathBuf = format!("{name}-{version_name}").into(); - Download::new(client, Url::parse(&crate_url)?) - .and_visit_tar(TarBasedFmt::Tgz, ManifestVisitor::new(manifest_dir_path)) - .await + Ok(Download::new(client, Url::parse(&crate_url)?) + .and_visit_tar( + TarBasedFmt::Tgz, + ManifestVisitor::new(manifest_dir_path), + Some(Box::pin(wait_on_cancellation_signal())), + ) + .await?) } diff --git a/crates/binstalk/src/drivers/crates_io/visitor.rs b/crates/binstalk/src/drivers/crates_io/visitor.rs index f36cae9d..b6d082c3 100644 --- a/crates/binstalk/src/drivers/crates_io/visitor.rs +++ b/crates/binstalk/src/drivers/crates_io/visitor.rs @@ -1,16 +1,16 @@ use std::{ - io::Read, + io::{self, Read}, path::{Path, PathBuf}, }; use cargo_toml::Manifest; use log::debug; use normalize_path::NormalizePath; -use tar::Entries; use super::vfs::Vfs; use crate::{ - errors::BinstallError, helpers::download::TarEntriesVisitor, + errors::BinstallError, + helpers::download::{DownloadError, Entries, TarEntriesVisitor}, manifests::cargo_toml_binstall::Meta, }; @@ -37,7 +37,7 @@ impl ManifestVisitor { impl TarEntriesVisitor for ManifestVisitor { type Target = Manifest; - fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError> { + fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), DownloadError> { for res in entries { let mut entry = res?; let path = entry.path()?; @@ -71,16 +71,20 @@ impl TarEntriesVisitor for ManifestVisitor { } /// Load binstall metadata using the extracted information stored in memory. - fn finish(self) -> Result { - debug!("Loading manifest directly from extracted file"); - - // Load and parse manifest - let mut manifest = Manifest::from_slice_with_metadata(&self.cargo_toml_content)?; - - // Checks vfs for binary output names - manifest.complete_from_abstract_filesystem(&self.vfs)?; - - // Return metadata - Ok(manifest) + fn finish(self) -> Result { + Ok(load_manifest(&self.cargo_toml_content, &self.vfs).map_err(io::Error::from)?) } } + +fn load_manifest(slice: &[u8], vfs: &Vfs) -> Result, BinstallError> { + debug!("Loading manifest directly from extracted file"); + + // Load and parse manifest + let mut manifest = Manifest::from_slice_with_metadata(slice)?; + + // Checks vfs for binary output names + manifest.complete_from_abstract_filesystem(vfs)?; + + // Return metadata + Ok(manifest) +} diff --git a/crates/binstalk/src/errors.rs b/crates/binstalk/src/errors.rs index b618eba4..351ba02b 100644 --- a/crates/binstalk/src/errors.rs +++ b/crates/binstalk/src/errors.rs @@ -4,6 +4,10 @@ use std::{ process::{ExitCode, ExitStatus, Termination}, }; +use binstalk_downloader::{ + download::{DownloadError, ZipError}, + remote::{Error as RemoteError, HttpError, ReqwestError}, +}; use compact_str::CompactString; use miette::{Diagnostic, Report}; use thiserror::Error; @@ -47,7 +51,7 @@ pub enum BinstallError { /// - Exit: 66 #[error(transparent)] #[diagnostic(severity(error), code(binstall::unzip))] - Unzip(#[from] zip::result::ZipError), + Unzip(#[from] ZipError), /// A rendering error in a template. /// @@ -65,7 +69,7 @@ pub enum BinstallError { /// - Exit: 68 #[error(transparent)] #[diagnostic(severity(error), code(binstall::reqwest))] - Reqwest(#[from] reqwest::Error), + Reqwest(#[from] ReqwestError), /// An HTTP request failed. /// @@ -74,14 +78,9 @@ pub enum BinstallError { /// /// - Code: `binstall::http` /// - Exit: 69 - #[error("could not {method} {url}")] + #[error(transparent)] #[diagnostic(severity(error), code(binstall::http))] - Http { - method: reqwest::Method, - url: url::Url, - #[source] - err: reqwest::Error, - }, + Http(#[from] HttpError), /// A subprocess failed. /// @@ -418,3 +417,27 @@ impl From for io::Error { } } } + +impl From for BinstallError { + fn from(e: RemoteError) -> Self { + use RemoteError::*; + + match e { + Reqwest(reqwest_error) => reqwest_error.into(), + Http(http_error) => http_error.into(), + } + } +} + +impl From for BinstallError { + fn from(e: DownloadError) -> Self { + use DownloadError::*; + + match e { + Unzip(zip_error) => zip_error.into(), + Remote(remote_error) => remote_error.into(), + Io(io_error) => io_error.into(), + UserAbort => BinstallError::UserAbort, + } + } +} diff --git a/crates/binstalk/src/fetchers/gh_crate_meta.rs b/crates/binstalk/src/fetchers/gh_crate_meta.rs index 3d33db51..63ced201 100644 --- a/crates/binstalk/src/fetchers/gh_crate_meta.rs +++ b/crates/binstalk/src/fetchers/gh_crate_meta.rs @@ -14,6 +14,7 @@ use crate::{ helpers::{ download::Download, remote::{Client, Method}, + signal::wait_on_cancellation_signal, tasks::AutoAbortJoinHandle, }, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, @@ -146,9 +147,9 @@ impl super::Fetcher for GhCrateMeta { async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> { let (url, pkg_fmt) = self.resolution.get().unwrap(); // find() is called first debug!("Downloading package from: '{url}' dst:{dst:?} fmt:{pkg_fmt:?}"); - Download::new(self.client.clone(), url.clone()) - .and_extract(*pkg_fmt, dst) - .await + Ok(Download::new(self.client.clone(), url.clone()) + .and_extract(*pkg_fmt, dst, Some(Box::pin(wait_on_cancellation_signal()))) + .await?) } fn pkg_fmt(&self) -> PkgFmt { diff --git a/crates/binstalk/src/fetchers/quickinstall.rs b/crates/binstalk/src/fetchers/quickinstall.rs index 0886b200..e374b143 100644 --- a/crates/binstalk/src/fetchers/quickinstall.rs +++ b/crates/binstalk/src/fetchers/quickinstall.rs @@ -10,6 +10,7 @@ use crate::{ helpers::{ download::Download, remote::{Client, Method}, + signal::wait_on_cancellation_signal, }, manifests::cargo_toml_binstall::{PkgFmt, PkgMeta}, }; @@ -44,17 +45,22 @@ impl super::Fetcher for QuickInstall { let url = self.package_url(); self.report(); debug!("Checking for package at: '{url}'"); - self.client + Ok(self + .client .remote_exists(Url::parse(&url)?, Method::HEAD) - .await + .await?) } async fn fetch_and_extract(&self, dst: &Path) -> Result<(), BinstallError> { let url = self.package_url(); debug!("Downloading package from: '{url}'"); - Download::new(self.client.clone(), Url::parse(&url)?) - .and_extract(self.pkg_fmt(), dst) - .await + Ok(Download::new(self.client.clone(), Url::parse(&url)?) + .and_extract( + self.pkg_fmt(), + dst, + Some(Box::pin(wait_on_cancellation_signal())), + ) + .await?) } fn pkg_fmt(&self) -> PkgFmt { diff --git a/crates/binstalk/src/helpers.rs b/crates/binstalk/src/helpers.rs index b0b8c703..83730ec3 100644 --- a/crates/binstalk/src/helpers.rs +++ b/crates/binstalk/src/helpers.rs @@ -1,5 +1,5 @@ -pub mod download; pub mod jobserver_client; -pub mod remote; pub mod signal; pub mod tasks; + +pub use binstalk_downloader::{download, remote}; diff --git a/crates/binstalk/src/helpers/download.rs b/crates/binstalk/src/helpers/download.rs deleted file mode 100644 index cb606000..00000000 --- a/crates/binstalk/src/helpers/download.rs +++ /dev/null @@ -1,112 +0,0 @@ -use std::{fmt::Debug, marker::PhantomData, path::Path}; - -use digest::{Digest, FixedOutput, HashMarker, Output, OutputSizeUser, Update}; -use log::debug; - -use crate::{ - errors::BinstallError, - helpers::remote::{Client, Url}, - manifests::cargo_toml_binstall::{PkgFmt, PkgFmtDecomposed, TarBasedFmt}, -}; - -pub use async_extracter::TarEntriesVisitor; -use async_extracter::*; - -mod async_extracter; -mod extracter; -mod stream_readable; - -#[derive(Debug)] -pub struct Download { - client: Client, - url: Url, - _digest: PhantomData, - _checksum: Vec, -} - -impl Download { - pub fn new(client: Client, url: Url) -> Self { - Self { - client, - url, - _digest: PhantomData::default(), - _checksum: Vec::new(), - } - } - - /// Download a file from the provided URL and extract part of it to - /// the provided path. - /// - /// * `filter` - If Some, then it will pass the path of the file to it - /// and only extract ones which filter returns `true`. - /// - /// This does not support verifying a checksum due to the partial extraction - /// and will ignore one if specified. - pub async fn and_visit_tar( - self, - fmt: TarBasedFmt, - visitor: V, - ) -> Result { - let stream = self.client.create_request(self.url).await?; - - debug!("Downloading and extracting then in-memory processing"); - - let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?; - - debug!("Download, extraction and in-memory procession OK"); - - Ok(ret) - } - - /// Download a file from the provided URL and extract it to the provided path. - pub async fn and_extract( - self, - fmt: PkgFmt, - path: impl AsRef, - ) -> Result<(), BinstallError> { - let stream = self.client.create_request(self.url).await?; - - let path = path.as_ref(); - debug!("Downloading and extracting to: '{}'", path.display()); - - match fmt.decompose() { - PkgFmtDecomposed::Tar(fmt) => extract_tar_based_stream(stream, path, fmt).await?, - PkgFmtDecomposed::Bin => extract_bin(stream, path).await?, - PkgFmtDecomposed::Zip => extract_zip(stream, path).await?, - } - - debug!("Download OK, extracted to: '{}'", path.display()); - - Ok(()) - } -} - -impl Download { - pub fn new_with_checksum(client: Client, url: Url, checksum: Vec) -> Self { - Self { - client, - url, - _digest: PhantomData::default(), - _checksum: checksum, - } - } - - // TODO: implement checking the sum, may involve bringing (parts of) and_extract() back in here -} - -#[derive(Clone, Copy, Debug, Default)] -pub struct NoDigest; - -impl FixedOutput for NoDigest { - fn finalize_into(self, _out: &mut Output) {} -} - -impl OutputSizeUser for NoDigest { - type OutputSize = generic_array::typenum::U0; -} - -impl Update for NoDigest { - fn update(&mut self, _data: &[u8]) {} -} - -impl HashMarker for NoDigest {} diff --git a/crates/binstalk/src/helpers/signal.rs b/crates/binstalk/src/helpers/signal.rs index d01041df..30eb57bf 100644 --- a/crates/binstalk/src/helpers/signal.rs +++ b/crates/binstalk/src/helpers/signal.rs @@ -1,11 +1,10 @@ -use std::io; - -use futures_util::future::pending; -use tokio::{signal, sync::OnceCell}; +use std::{future::pending, io}; use super::tasks::AutoAbortJoinHandle; use crate::errors::BinstallError; +use tokio::{signal, sync::OnceCell}; + /// This function will poll the handle while listening for ctrl_c, /// `SIGINT`, `SIGHUP`, `SIGTERM` and `SIGQUIT`. /// @@ -18,19 +17,24 @@ use crate::errors::BinstallError; pub async fn cancel_on_user_sig_term( handle: AutoAbortJoinHandle, ) -> Result { - #[cfg(unix)] - unix::ignore_signals_on_unix()?; + ignore_signals()?; tokio::select! { res = handle => res, res = wait_on_cancellation_signal() => { - res - .map_err(BinstallError::Io) + res.map_err(BinstallError::Io) .and(Err(BinstallError::UserAbort)) } } } +pub fn ignore_signals() -> io::Result<()> { + #[cfg(unix)] + unix::ignore_signals_on_unix()?; + + Ok(()) +} + /// If call to it returns `Ok(())`, then all calls to this function after /// that also returns `Ok(())`. pub async fn wait_on_cancellation_signal() -> Result<(), io::Error> { @@ -86,7 +90,7 @@ mod unix { } } - pub fn ignore_signals_on_unix() -> Result<(), BinstallError> { + pub fn ignore_signals_on_unix() -> Result<(), io::Error> { drop(signal(SignalKind::user_defined1())?); drop(signal(SignalKind::user_defined2())?);