actions/cargo-binstall
1
0
Fork 0
mirror of https://github.com/cargo-bins/cargo-binstall.git synced 2025-05-03 18:50:02 +00:00
Code Issues Projects Releases Packages Wiki Activity

Impl GhApiClient and use it in cargo-binstall to speedup resolution process (#832)

Browse source 
Fixed #776

 - Add new feature gh-api-client to binstalk-downloader
 - Impl new type `binstalk_downloader::remote::{RequestBuilder, Response}`
 - Impl `binstalk_downloader::gh_api_client::GhApiClient`, exposed if `cfg(feature = "gh-api-client")` and add e2e and unit tests for it
 - Use `binstalk_downloader::gh_api_client::GhApiClient` to speedup `cargo-binstall`
 - Add new option `--github-token` to supply the token for GitHub restful API, or read from env variable `GITHUB_TOKEN` if not present.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2023-03-02 12:04:22 +11:00 • committed by GitHub
parent 263c836757
commit 599bcaf333
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 960 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

66
crates/binstalk-downloader/src/download/async_extracter.rs
View file

@ -7,18 +7,15 @@ use std::{
use async_zip::read::stream::ZipFileReader;
use bytes::{Bytes, BytesMut};
use futures_lite::{
future::try_zip as try_join,
stream::{Stream, StreamExt},
};
use futures_lite::stream::Stream;
use tokio::sync::mpsc;
use tokio_util::io::StreamReader;
use tracing::debug;
use super::{
extracter::*, stream_readable::StreamReadable, utils::asyncify,
zip_extraction::extract_zip_entry, DownloadError, TarBasedFmt, ZipError,
extracter::*, zip_extraction::extract_zip_entry, DownloadError, TarBasedFmt, ZipError,
};
use crate::utils::{extract_with_blocking_task, StreamReadable};
pub async fn extract_bin<S>(stream: S, path: &Path) -> Result<(), DownloadError>
where
@ -77,71 +74,22 @@ where
.await
}
async fn extract_with_blocking_decoder<S, F>(
fn extract_with_blocking_decoder<S, F>(
stream: S,
path: &Path,
f: F,
) -> Result<(), DownloadError>
) -> impl Future<Output = Result<(), DownloadError>>
where
S: Stream<Item = Result<Bytes, DownloadError>> + Send + Sync + Unpin + 'static,
F: FnOnce(mpsc::Receiver<Bytes>, &Path) -> io::Result<()> + Send + Sync + 'static,
{
async fn inner<S, Fut>(
mut stream: S,
task: Fut,
tx: mpsc::Sender<Bytes>,
) -> Result<(), DownloadError>
where
// We do not use trait object for S since there will only be one
// S used with this function.
S: Stream<Item = Result<Bytes, DownloadError>> + Send + Sync + Unpin + 'static,
// asyncify would always return the same future, so no need to
// use trait object here.
Fut: Future<Output = io::Result<()>> + Send + Sync,
{
try_join(
async move {
while let Some(bytes) = stream.next().await.transpose()? {
if bytes.is_empty() {
continue;
}
if tx.send(bytes).await.is_err() {
// The extract tar returns, which could be that:
// - Extraction fails with an error
// - Extraction success without the rest of the data
//
//
// It's hard to tell the difference here, so we assume
// the first scienario occurs.
//
// Even if the second scienario occurs, it won't affect the
// extraction process anyway, so we can jsut ignore it.
return Ok(());
}
}
Ok(())
},
task,
)
.await?;
Ok(())
}
// Use channel size = 5 to minimize the waiting time in the extraction task
let (tx, rx) = mpsc::channel(5);
let path = path.to_owned();
let task = asyncify(move || {
extract_with_blocking_task(stream, move |rx| {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
f(rx, &path)
});
inner(stream, task, tx).await
})
}