From a35db557eaf2e80014265483b3f6fe701a6e2632 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 23 Jun 2022 19:48:03 +1000 Subject: [PATCH 1/3] Run "Install binaries" step in `block_in_place` mode Since they execute a lot of blocking fs operations. Signed-off-by: Jiahao XU --- src/main.rs | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index 39e832e0..169e74d7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,11 @@ use log::{debug, error, info, warn, LevelFilter}; use miette::{miette, IntoDiagnostic, Result, WrapErr}; use simplelog::{ColorChoice, ConfigBuilder, TermLogger, TerminalMode}; use tempfile::TempDir; -use tokio::{process::Command, runtime::Runtime, task::JoinError}; +use tokio::{ + process::Command, + runtime::Runtime, + task::{block_in_place, JoinError}, +}; use cargo_binstall::{ bins, @@ -441,26 +445,28 @@ async fn install_from_package( uithread.confirm().await?; info!("Installing binaries..."); - for file in &bin_files { - file.install_bin()?; - } - - // Generate symlinks - if !opts.no_symlinks { + block_in_place(|| { for file in &bin_files { - file.install_link()?; + file.install_bin()?; } - } - if opts.no_cleanup { - let _ = temp_dir.into_path(); - } else { - temp_dir.close().unwrap_or_else(|err| { - warn!("Failed to clean up some resources: {err}"); - }); - } + // Generate symlinks + if !opts.no_symlinks { + for file in &bin_files { + file.install_link()?; + } + } - Ok(()) + if opts.no_cleanup { + let _ = temp_dir.into_path(); + } else { + temp_dir.close().unwrap_or_else(|err| { + warn!("Failed to clean up some resources: {err}"); + }); + } + + Ok(()) + }) } async fn install_from_source( From 60caa9ee177574236b975767e767b7889e7da7d0 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 23 Jun 2022 19:50:25 +1000 Subject: [PATCH 2/3] Run `load_manifest_path` in `block_in_place` mode Signed-off-by: Jiahao XU --- src/helpers.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/helpers.rs b/src/helpers.rs index 03bf97c9..55edc19a 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -9,6 +9,7 @@ use once_cell::sync::OnceCell; use reqwest::{Client, ClientBuilder, Method, Response}; use serde::Serialize; use tinytemplate::TinyTemplate; +use tokio::task::block_in_place; use url::Url; use crate::{BinstallError, Meta, PkgFmt, PkgFmtDecomposed, TarBasedFmt}; @@ -38,13 +39,15 @@ pub static REQWESTGLOBALCONFIG: OnceCell<(bool, Option)> = OnceCell: pub fn load_manifest_path>( manifest_path: P, ) -> Result, BinstallError> { - debug!("Reading manifest: {}", manifest_path.as_ref().display()); + block_in_place(|| { + debug!("Reading manifest: {}", manifest_path.as_ref().display()); - // Load and parse manifest (this checks file system for binary output names) - let manifest = Manifest::::from_path_with_metadata(manifest_path)?; + // Load and parse manifest (this checks file system for binary output names) + let manifest = Manifest::::from_path_with_metadata(manifest_path)?; - // Return metadata - Ok(manifest) + // Return metadata + Ok(manifest) + }) } pub fn new_reqwest_client_builder() -> ClientBuilder { From e18ac6e117e95524046bc46b709775cdc1280d38 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 23 Jun 2022 19:55:09 +1000 Subject: [PATCH 3/3] Run `Manifest` parsing in `block_in_place` mode Signed-off-by: Jiahao XU --- src/drivers/crates_io.rs | 3 +-- src/drivers/crates_io/visitor.rs | 30 ++++++++++++++++-------------- src/helpers.rs | 6 +++--- src/helpers/async_extracter.rs | 13 +++++-------- 4 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/drivers/crates_io.rs b/src/drivers/crates_io.rs index 68f04e82..fa1a8063 100644 --- a/src/drivers/crates_io.rs +++ b/src/drivers/crates_io.rs @@ -71,6 +71,5 @@ pub async fn fetch_crate_cratesio( TarBasedFmt::Tgz, ManifestVisitor::new(manifest_dir_path), ) - .await? - .load_manifest() + .await } diff --git a/src/drivers/crates_io/visitor.rs b/src/drivers/crates_io/visitor.rs index 3f354771..0eb7bd6c 100644 --- a/src/drivers/crates_io/visitor.rs +++ b/src/drivers/crates_io/visitor.rs @@ -29,23 +29,11 @@ impl ManifestVisitor { vfs: Vfs::new(), } } - - /// Load binstall metadata using the extracted information stored in memory. - pub(super) fn load_manifest(&self) -> Result, BinstallError> { - debug!("Loading manifest directly from extracted file"); - - // Load and parse manifest - let mut manifest = Manifest::::from_slice_with_metadata(&self.cargo_toml_content)?; - - // Checks vfs for binary output names - manifest.complete_from_abstract_filesystem(&self.vfs)?; - - // Return metadata - Ok(manifest) - } } impl TarEntriesVisitor for ManifestVisitor { + type Target = Manifest; + fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError> { for res in entries { let mut entry = res?; @@ -78,4 +66,18 @@ impl TarEntriesVisitor for ManifestVisitor { Ok(()) } + + /// Load binstall metadata using the extracted information stored in memory. + fn finish(self) -> Result { + debug!("Loading manifest directly from extracted file"); + + // Load and parse manifest + let mut manifest = Manifest::from_slice_with_metadata(&self.cargo_toml_content)?; + + // Checks vfs for binary output names + manifest.complete_from_abstract_filesystem(&self.vfs)?; + + // Return metadata + Ok(manifest) + } } diff --git a/src/helpers.rs b/src/helpers.rs index 55edc19a..18794acb 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -130,16 +130,16 @@ pub async fn download_tar_based_and_visit Result { +) -> Result { let stream = create_request(url).await?; debug!("Downloading and extracting then in-memory processing"); - let visitor = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?; + let ret = extract_tar_based_stream_and_visit(stream, fmt, visitor).await?; debug!("Download, extraction and in-memory procession OK"); - Ok(visitor) + Ok(ret) } /// Fetch install path from environment diff --git a/src/helpers/async_extracter.rs b/src/helpers/async_extracter.rs index 5a7ef9c1..9986c48a 100644 --- a/src/helpers/async_extracter.rs +++ b/src/helpers/async_extracter.rs @@ -99,20 +99,17 @@ where /// Visitor must iterate over all entries. /// Entires can be in arbitary order. pub trait TarEntriesVisitor { - fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError>; -} + type Target; -impl TarEntriesVisitor for &mut V { - fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError> { - (*self).visit(entries) - } + fn visit(&mut self, entries: Entries<'_, R>) -> Result<(), BinstallError>; + fn finish(self) -> Result; } pub async fn extract_tar_based_stream_and_visit( stream: S, fmt: TarBasedFmt, mut visitor: V, -) -> Result +) -> Result where S: Stream> + Unpin + 'static, V: TarEntriesVisitor + Debug + Send + 'static, @@ -124,6 +121,6 @@ where let mut tar = create_tar_decoder(reader, fmt)?; visitor.visit(tar.entries()?)?; - Ok(visitor) + visitor.finish() }) }