Impl GhApiClient and use it in cargo-binstall to speedup resolution process (#832)

Fixed #776

 - Add new feature gh-api-client to binstalk-downloader
 - Impl new type `binstalk_downloader::remote::{RequestBuilder, Response}`
 - Impl `binstalk_downloader::gh_api_client::GhApiClient`, exposed if `cfg(feature = "gh-api-client")` and add e2e and unit tests for it
 - Use `binstalk_downloader::gh_api_client::GhApiClient` to speedup `cargo-binstall`
 - Add new option `--github-token` to supply the token for GitHub restful API, or read from env variable `GITHUB_TOKEN` if not present.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

crates/binstalk-downloader/src/download/async_extracter.rs

@@ -7,18 +7,15 @@ use std::{
 
 use async_zip::read::stream::ZipFileReader;
 use bytes::{Bytes, BytesMut};
-use futures_lite::{
-    future::try_zip as try_join,
-    stream::{Stream, StreamExt},
-};
+use futures_lite::stream::Stream;
 use tokio::sync::mpsc;
 use tokio_util::io::StreamReader;
 use tracing::debug;
 
 use super::{
-    extracter::*, stream_readable::StreamReadable, utils::asyncify,
-    zip_extraction::extract_zip_entry, DownloadError, TarBasedFmt, ZipError,
+    extracter::*, zip_extraction::extract_zip_entry, DownloadError, TarBasedFmt, ZipError,
 };
+use crate::utils::{extract_with_blocking_task, StreamReadable};
 
 pub async fn extract_bin<S>(stream: S, path: &Path) -> Result<(), DownloadError>
 where
@@ -77,71 +74,22 @@ where
     .await
 }
 
-async fn extract_with_blocking_decoder<S, F>(
+fn extract_with_blocking_decoder<S, F>(
     stream: S,
     path: &Path,
     f: F,
-) -> Result<(), DownloadError>
+) -> impl Future<Output = Result<(), DownloadError>>
 where
     S: Stream<Item = Result<Bytes, DownloadError>> + Send + Sync + Unpin + 'static,
     F: FnOnce(mpsc::Receiver<Bytes>, &Path) -> io::Result<()> + Send + Sync + 'static,
 {
-    async fn inner<S, Fut>(
-        mut stream: S,
-        task: Fut,
-        tx: mpsc::Sender<Bytes>,
-    ) -> Result<(), DownloadError>
-    where
-        // We do not use trait object for S since there will only be one
-        // S used with this function.
-        S: Stream<Item = Result<Bytes, DownloadError>> + Send + Sync + Unpin + 'static,
-        // asyncify would always return the same future, so no need to
-        // use trait object here.
-        Fut: Future<Output = io::Result<()>> + Send + Sync,
-    {
-        try_join(
-            async move {
-                while let Some(bytes) = stream.next().await.transpose()? {
-                    if bytes.is_empty() {
-                        continue;
-                    }
-
-                    if tx.send(bytes).await.is_err() {
-                        // The extract tar returns, which could be that:
-                        //  - Extraction fails with an error
-                        //  - Extraction success without the rest of the data
-                        //
-                        //
-                        // It's hard to tell the difference here, so we assume
-                        // the first scienario occurs.
-                        //
-                        // Even if the second scienario occurs, it won't affect the
-                        // extraction process anyway, so we can jsut ignore it.
-                        return Ok(());
-                    }
-                }
-
-                Ok(())
-            },
-            task,
-        )
-        .await?;
-
-        Ok(())
-    }
-
-    // Use channel size = 5 to minimize the waiting time in the extraction task
-    let (tx, rx) = mpsc::channel(5);
-
     let path = path.to_owned();
 
-    let task = asyncify(move || {
+    extract_with_blocking_task(stream, move |rx| {
         if let Some(parent) = path.parent() {
             fs::create_dir_all(parent)?;
         }
 
         f(rx, &path)
-    });
-
-    inner(stream, task, tx).await
+    })
 }

crates/binstalk-downloader/src/download/stream_readable.rs

@@ -1,65 +0,0 @@
-use std::io::{self, BufRead, Read};
-
-use bytes::{Buf, Bytes};
-use tokio::sync::mpsc;
-
-/// This wraps an AsyncIterator as a `Read`able.
-/// It must be used in non-async context only,
-/// meaning you have to use it with
-/// `tokio::task::{block_in_place, spawn_blocking}` or
-/// `std::thread::spawn`.
-pub struct StreamReadable {
-    rx: mpsc::Receiver<Bytes>,
-    bytes: Bytes,
-}
-
-impl StreamReadable {
-    pub(super) fn new(rx: mpsc::Receiver<Bytes>) -> Self {
-        Self {
-            rx,
-            bytes: Bytes::new(),
-        }
-    }
-}
-
-impl Read for StreamReadable {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        if buf.is_empty() {
-            return Ok(0);
-        }
-
-        if self.fill_buf()?.is_empty() {
-            return Ok(0);
-        }
-
-        let bytes = &mut self.bytes;
-
-        // copy_to_slice requires the bytes to have enough remaining bytes
-        // to fill buf.
-        let n = buf.len().min(bytes.remaining());
-
-        // <Bytes as Buf>::copy_to_slice copies and consumes the bytes
-        bytes.copy_to_slice(&mut buf[..n]);
-
-        Ok(n)
-    }
-}
-
-impl BufRead for StreamReadable {
-    fn fill_buf(&mut self) -> io::Result<&[u8]> {
-        let bytes = &mut self.bytes;
-
-        if !bytes.has_remaining() {
-            if let Some(new_bytes) = self.rx.blocking_recv() {
-                // new_bytes are guaranteed to be non-empty.
-                *bytes = new_bytes;
-            }
-        }
-
-        Ok(&*bytes)
-    }
-
-    fn consume(&mut self, amt: usize) {
-        self.bytes.advance(amt);
-    }
-}

crates/binstalk-downloader/src/download/utils.rs

@@ -1,22 +0,0 @@
-use std::{future::Future, io};
-
-use tokio::task;
-
-/// Copied from tokio https://docs.rs/tokio/latest/src/tokio/fs/mod.rs.html#132
-pub(super) fn asyncify<F, T>(f: F) -> impl Future<Output = io::Result<T>> + Send + Sync + 'static
-where
-    F: FnOnce() -> io::Result<T> + Send + 'static,
-    T: Send + 'static,
-{
-    async fn inner<T: Send + 'static>(handle: task::JoinHandle<io::Result<T>>) -> io::Result<T> {
-        match handle.await {
-            Ok(res) => res,
-            Err(err) => Err(io::Error::new(
-                io::ErrorKind::Other,
-                format!("background task failed: {err}"),
-            )),
-        }
-    }
-
-    inner(task::spawn_blocking(f))
-}

crates/binstalk-downloader/src/download/zip_extraction.rs

@@ -12,7 +12,8 @@ use tokio::{
     sync::mpsc,
 };
 
-use super::{utils::asyncify, DownloadError};
+use super::DownloadError;
+use crate::utils::asyncify;
 
 #[derive(Debug, ThisError)]
 enum ZipErrorInner {