Use untar in extract_compressed_from_readable

So that we can specify the files we want to extract to avoid io and save
disk usage.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2022-06-09 13:26:47 +10:00
parent be5e8616a2
commit 72983e4113
No known key found for this signature in database
GPG key ID: 591C0B03040416D6
3 changed files with 41 additions and 19 deletions

View file

@ -1,4 +1,5 @@
use std::{ use std::{
borrow::Cow,
fs, fs,
io::{stderr, stdin, Write}, io::{stderr, stdin, Write},
path::{Path, PathBuf}, path::{Path, PathBuf},
@ -53,14 +54,19 @@ pub async fn remote_exists(url: Url, method: Method) -> Result<bool, BinstallErr
/// Download a file from the provided URL to the provided path /// Download a file from the provided URL to the provided path
pub async fn download<P: AsRef<Path>>(url: &str, path: P) -> Result<(), BinstallError> { pub async fn download<P: AsRef<Path>>(url: &str, path: P) -> Result<(), BinstallError> {
let url = Url::parse(url)?; let url = Url::parse(url)?;
download_and_extract(url, PkgFmt::Bin, path.as_ref()).await download_and_extract::<_, 0>(url, PkgFmt::Bin, path.as_ref(), None).await
} }
/// Download a file from the provided URL and extract it to the provided path /// Download a file from the provided URL and extract it to the provided path
pub async fn download_and_extract<P: AsRef<Path>>( ///
/// * `desired_outputs - If Some(_) and `fmt` is not `PkgFmt::Bin` or
/// `PkgFmt::Zip`, then it will filter the tar and only extract files
/// specified in it.
pub async fn download_and_extract<P: AsRef<Path>, const N: usize>(
url: Url, url: Url,
fmt: PkgFmt, fmt: PkgFmt,
path: P, path: P,
desired_outputs: Option<[Cow<'static, Path>; N]>,
) -> Result<(), BinstallError> { ) -> Result<(), BinstallError> {
debug!("Downloading from: '{url}'"); debug!("Downloading from: '{url}'");
@ -77,7 +83,7 @@ pub async fn download_and_extract<P: AsRef<Path>>(
debug!("Downloading to file: '{}'", path.display()); debug!("Downloading to file: '{}'", path.display());
let mut bytes_stream = resp.bytes_stream(); let mut bytes_stream = resp.bytes_stream();
let mut writer = AsyncFileWriter::new(path, fmt); let mut writer = AsyncFileWriter::new(path, fmt, desired_outputs);
while let Some(res) = bytes_stream.next().await { while let Some(res) = bytes_stream.next().await {
writer.write(res?).await?; writer.write(res?).await?;

View file

@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::fs; use std::fs;
use std::io::{self, Seek, Write}; use std::io::{self, Seek, Write};
use std::path::Path; use std::path::Path;
@ -27,7 +28,13 @@ struct AsyncFileWriterInner {
} }
impl AsyncFileWriterInner { impl AsyncFileWriterInner {
fn new(path: &Path, fmt: PkgFmt) -> Self { /// * `desired_outputs - If Some(_), then it will filter the tar
/// and only extract files specified in it.
fn new<const N: usize>(
path: &Path,
fmt: PkgFmt,
desired_outputs: Option<[Cow<'static, Path>; N]>,
) -> Self {
let path = path.to_owned(); let path = path.to_owned();
let (tx, rx) = mpsc::channel::<Content>(100); let (tx, rx) = mpsc::channel::<Content>(100);
@ -65,7 +72,12 @@ impl AsyncFileWriterInner {
unzip(file, &path)?; unzip(file, &path)?;
} }
_ => extract_compressed_from_readable(ReadableRx::new(&mut rx), fmt, &path)?, _ => extract_compressed_from_readable(
ReadableRx::new(&mut rx),
fmt,
&path,
desired_outputs.as_ref().map(|arr| &arr[..]),
)?,
} }
Ok(()) Ok(())
@ -141,8 +153,15 @@ impl AsyncFileWriterInner {
pub struct AsyncFileWriter(ScopeGuard<AsyncFileWriterInner, fn(AsyncFileWriterInner), Always>); pub struct AsyncFileWriter(ScopeGuard<AsyncFileWriterInner, fn(AsyncFileWriterInner), Always>);
impl AsyncFileWriter { impl AsyncFileWriter {
pub fn new(path: &Path, fmt: PkgFmt) -> Self { /// * `desired_outputs - If Some(_) and `fmt` is not `PkgFmt::Bin` or
let inner = AsyncFileWriterInner::new(path, fmt); /// `PkgFmt::Zip`, then it will filter the tar and only extract files
/// specified in it.
pub fn new<const N: usize>(
path: &Path,
fmt: PkgFmt,
desired_outputs: Option<[Cow<'static, Path>; N]>,
) -> Self {
let inner = AsyncFileWriterInner::new(path, fmt, desired_outputs);
Self(guard(inner, AsyncFileWriterInner::abort)) Self(guard(inner, AsyncFileWriterInner::abort))
} }

View file

@ -12,6 +12,8 @@ use zstd::stream::Decoder as ZstdDecoder;
use crate::{BinstallError, PkgFmt}; use crate::{BinstallError, PkgFmt};
/// * `desired_outputs - If Some(_), then it will filter the tar
/// and only extract files specified in it.
fn untar( fn untar(
dat: impl Read, dat: impl Read,
path: &Path, path: &Path,
@ -40,37 +42,34 @@ fn untar(
/// Extract files from the specified source onto the specified path. /// Extract files from the specified source onto the specified path.
/// ///
/// * `fmt` - must not be `PkgFmt::Bin` or `PkgFmt::Zip`. /// * `fmt` - must not be `PkgFmt::Bin` or `PkgFmt::Zip`.
/// * `desired_outputs - If Some(_), then it will filter the tar
/// and only extract files specified in it.
pub(crate) fn extract_compressed_from_readable( pub(crate) fn extract_compressed_from_readable(
dat: impl Read, dat: impl Read,
fmt: PkgFmt, fmt: PkgFmt,
path: &Path, path: &Path,
desired_outputs: Option<&[Cow<'_, Path>]>,
) -> Result<(), BinstallError> { ) -> Result<(), BinstallError> {
match fmt { match fmt {
PkgFmt::Tar => { PkgFmt::Tar => {
// Extract to install dir // Extract to install dir
debug!("Extracting from tar archive to `{path:?}`"); debug!("Extracting from tar archive to `{path:?}`");
let mut tar = Archive::new(dat); untar(dat, path, desired_outputs)?
tar.unpack(path)?;
} }
PkgFmt::Tgz => { PkgFmt::Tgz => {
// Extract to install dir // Extract to install dir
debug!("Decompressing from tgz archive to `{path:?}`"); debug!("Decompressing from tgz archive to `{path:?}`");
let tar = GzDecoder::new(dat); let tar = GzDecoder::new(dat);
let mut tgz = Archive::new(tar); untar(tar, path, desired_outputs)?;
tgz.unpack(path)?;
} }
PkgFmt::Txz => { PkgFmt::Txz => {
// Extract to install dir // Extract to install dir
debug!("Decompressing from txz archive to `{path:?}`"); debug!("Decompressing from txz archive to `{path:?}`");
let tar = XzDecoder::new(dat); let tar = XzDecoder::new(dat);
let mut txz = Archive::new(tar); untar(tar, path, desired_outputs)?;
txz.unpack(path)?;
} }
PkgFmt::Tzstd => { PkgFmt::Tzstd => {
// Extract to install dir // Extract to install dir
@ -81,9 +80,7 @@ pub(crate) fn extract_compressed_from_readable(
// as &[] by ZstdDecoder::new, thus ZstdDecoder::new // as &[] by ZstdDecoder::new, thus ZstdDecoder::new
// should not return any error. // should not return any error.
let tar = ZstdDecoder::new(dat)?; let tar = ZstdDecoder::new(dat)?;
let mut txz = Archive::new(tar); untar(tar, path, desired_outputs)?;
txz.unpack(path)?;
} }
PkgFmt::Zip => panic!("Unexpected PkgFmt::Zip!"), PkgFmt::Zip => panic!("Unexpected PkgFmt::Zip!"),
PkgFmt::Bin => panic!("Unexpected PkgFmt::Bin!"), PkgFmt::Bin => panic!("Unexpected PkgFmt::Bin!"),