From 16b16c482ccb3be9985eee596875e41d4b99b619 Mon Sep 17 00:00:00 2001
From: Jiahao XU <Jiahao_XU@outlook.com>
Date: Sat, 27 Aug 2022 19:56:43 +1000
Subject: [PATCH] Optimize `GhCrateMeta::find`: use `FuturesUnordered` (#321)

* Optimize `GhCrateMeta::find` using `FuturesUnordered`
* Optimize `MultiFetcher`: Impl `with_capacity` use it in `resolve_inner` to avoid over-reservation
* Enable feature "std" of dep futures-util

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
---
 crates/lib/Cargo.toml                    | 2 +-
 crates/lib/src/fetchers.rs               | 5 ++++-
 crates/lib/src/fetchers/gh_crate_meta.rs | 7 ++++---
 crates/lib/src/ops/resolve.rs            | 4 ++--
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml
index d1067729..aaa2449a 100644
--- a/crates/lib/Cargo.toml
+++ b/crates/lib/Cargo.toml
@@ -20,7 +20,7 @@ crates_io_api = { version = "0.8.0", default-features = false }
 detect-targets = { version = "0.1.0", path = "../detect-targets" }
 flate2 = { version = "1.0.24", default-features = false }
 flock = { version = "0.1.0", path = "../flock" }
-futures-util = { version = "0.3.23", default-features = false }
+futures-util = { version = "0.3.23", default-features = false, features = ["std"] }
 home = "0.5.3"
 itertools = "0.10.3"
 jobserver = "0.1.24"
diff --git a/crates/lib/src/fetchers.rs b/crates/lib/src/fetchers.rs
index a6565bc3..bc7ab9fd 100644
--- a/crates/lib/src/fetchers.rs
+++ b/crates/lib/src/fetchers.rs
@@ -63,10 +63,13 @@ pub struct Data {
 
 type FetcherJoinHandle = AutoAbortJoinHandle<Result<bool, BinstallError>>;
 
-#[derive(Default)]
 pub struct MultiFetcher(Vec<(Arc<dyn Fetcher>, FetcherJoinHandle)>);
 
 impl MultiFetcher {
+    pub fn with_capacity(n: usize) -> Self {
+        Self(Vec::with_capacity(n))
+    }
+
     pub fn add(&mut self, fetcher: Arc<dyn Fetcher>) {
         self.0.push((
             fetcher.clone(),
diff --git a/crates/lib/src/fetchers/gh_crate_meta.rs b/crates/lib/src/fetchers/gh_crate_meta.rs
index 267af11b..29c22f7b 100644
--- a/crates/lib/src/fetchers/gh_crate_meta.rs
+++ b/crates/lib/src/fetchers/gh_crate_meta.rs
@@ -1,6 +1,7 @@
 use std::{borrow::Cow, path::Path, sync::Arc};
 
 use compact_str::{CompactString, ToCompactString};
+use futures_util::stream::{FuturesUnordered, StreamExt};
 use log::{debug, warn};
 use once_cell::sync::OnceCell;
 use reqwest::{Client, Method};
@@ -120,7 +121,7 @@ impl super::Fetcher for GhCrateMeta {
                 .flat_map(move |pkg_url| self.launch_baseline_find_tasks(pkg_fmt, pkg_url, repo))
         };
 
-        let handles: Vec<_> = if let Some(pkg_fmt) = self.data.meta.pkg_fmt {
+        let mut handles: FuturesUnordered<_> = if let Some(pkg_fmt) = self.data.meta.pkg_fmt {
             launch_baseline_find_tasks(pkg_fmt).collect()
         } else {
             PkgFmt::iter()
@@ -128,8 +129,8 @@ impl super::Fetcher for GhCrateMeta {
                 .collect()
         };
 
-        for handle in handles {
-            if let Some((url, pkg_fmt)) = handle.await?? {
+        while let Some(res) = handles.next().await {
+            if let Some((url, pkg_fmt)) = res?? {
                 debug!("Winning URL is {url}, with pkg_fmt {pkg_fmt}");
                 self.resolution.set((url, pkg_fmt)).unwrap(); // find() is called first
                 return Ok(true);
diff --git a/crates/lib/src/ops/resolve.rs b/crates/lib/src/ops/resolve.rs
index 381a4aaf..bcbaae6b 100644
--- a/crates/lib/src/ops/resolve.rs
+++ b/crates/lib/src/ops/resolve.rs
@@ -169,10 +169,10 @@ async fn resolve_inner(
         manifest.bin,
     );
 
-    let mut fetchers = MultiFetcher::default();
-
     let desired_targets = opts.desired_targets.get().await;
 
+    let mut fetchers = MultiFetcher::with_capacity(desired_targets.len() * 2);
+
     for target in desired_targets {
         debug!("Building metadata for target: {target}");
         let mut target_meta = meta.clone_without_overrides();