feature: faster self-install without rate limit issue (#1963)

* Add new hidden option `--self-install`

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix typo

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Optimize: Only call `LazyJobserverClient::new` when necessary

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* `--self-install` should include a path to a binary

On windows, we cannot just copy an executable with process running from it, so better to copy it to a temporary location.

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Impl manifest update and basic API of `self_install`

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Add dep atomic-file-install to cargo-binstall

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Impl `self_install`

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Make `--self-install` a boolean flag

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Accept no duration in `MainExit::new`

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Impl self-install mode in main_impl.rs

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Use `--self-install` mode in install-from-binstall-release.sh

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Use `--self-install` in install-from-binstall-release.ps1

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix import in mod entry

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix call of `self_install` in `main_impl`

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix `--self-install` clap doc

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix `entry::self_install`

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Apply suggestions from code review

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix parsing semver number in entry.rs

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix compilation in entry.rs

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* fix entry.rs

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix fmt in bin_util.rs

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix fmt in entry.rs

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Use --self-install if supported in unix install script

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Use --self-install if available in powershell install script

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix install-from-binstall-release.ps1

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fi install-from-binstall-release.ps1

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix install-from-binstall-release.ps1

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Create self-install.sh

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Add e2e-test-self-install to justfile

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix args::parse() for self-install mode

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

* Fix args parsing: Do no require positional arg if --self-install is present

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

---------

Signed-off-by: Jiahao XU <30436523+NobodyXu@users.noreply.github.com>

crates/bin/Cargo.toml

@@ -22,6 +22,7 @@ pkg-fmt = "zip"
 pkg-fmt = "zip"
 
 [dependencies]
+atomic-file-install = { version = "1.0.5", path = "../atomic-file-install" }
 binstalk = { path = "../binstalk", version = "0.28.11", default-features = false }
 binstalk-manifests = { path = "../binstalk-manifests", version = "0.15.8" }
 clap = { version = "4.5.3", features = ["derive", "env"] }

crates/bin/src/args.rs

@@ -52,7 +52,7 @@ pub struct Args {
     #[clap(
         help_heading = "Package selection",
         value_name = "crate[@version]",
-        required_unless_present_any = ["version", "help"],
+        required_unless_present_any = ["version", "self_install", "help"],
     )]
     pub(crate) crate_names: Vec<CrateName>,
 
@@ -404,6 +404,9 @@ pub struct Args {
     /// This would override the `log_level`.
     #[clap(help_heading = "Meta", short, long, conflicts_with("verbose"))]
     pub(crate) quiet: bool,
+
+    #[clap(long, hide(true))]
+    pub(crate) self_install: bool,
 }
 
 #[derive(Debug, Clone)]
@@ -515,6 +518,10 @@ pub fn parse() -> (Args, PkgOverride) {
     // Load options
     let mut opts = Args::parse_from(args);
 
+    if opts.self_install {
+        return (opts, Default::default());
+    }
+
     if opts.log_level.is_none() {
         if let Some(log) = env::var("BINSTALL_LOG_LEVEL")
             .ok()

crates/bin/src/bin_util.rs

@@ -36,9 +36,8 @@ impl Termination for MainExit {
 }
 
 impl MainExit {
-    pub fn new(res: Result<()>, done: Duration) -> Self {
+    pub fn new(res: Result<()>, done: Option<Duration>) -> Self {
-        res.map(|()| MainExit::Success(Some(done)))
+        res.map(|()| MainExit::Success(done)).unwrap_or_else(|err| {
-            .unwrap_or_else(|err| {
             err.downcast::<BinstallError>()
                 .map(MainExit::Error)
                 .unwrap_or_else(MainExit::Report)

crates/bin/src/entry.rs

@@ -5,6 +5,7 @@ use std::{
     time::Duration,
 };
 
+use atomic_file_install::atomic_install;
 use binstalk::{
     errors::{BinstallError, CrateContextError},
     fetchers::{Fetcher, GhCrateMeta, QuickInstall, SignaturePolicy},
@@ -20,16 +21,20 @@ use binstalk::{
         resolve::{CrateName, Resolution, ResolutionFetch, VersionReqExt},
         CargoTomlFetchOverride, Options, Resolver,
     },
+    TARGET,
 };
 use binstalk_manifests::{
     cargo_config::Config,
     cargo_toml_binstall::{PkgOverride, Strategy},
+    crate_info::{CrateInfo, CrateSource},
     crates_manifests::Manifests,
 };
+use compact_str::CompactString;
 use file_format::FileFormat;
 use home::cargo_home;
 use log::LevelFilter;
 use miette::{miette, Report, Result, WrapErr};
+use semver::Version;
 use tokio::task::block_in_place;
 use tracing::{debug, error, info, warn};
 
@@ -582,3 +587,44 @@ fn do_install_fetches_continue_on_failure(
         Ok(())
     })
 }
+
+pub fn self_install(args: Args) -> Result<()> {
+    // Load .cargo/config.toml
+    let cargo_home = cargo_home().map_err(BinstallError::from)?;
+    let mut config = Config::load_from_path(cargo_home.join("config.toml"))?;
+
+    // Compute paths
+    let cargo_root = args.root;
+    let (install_path, manifests, _) = compute_paths_and_load_manifests(
+        cargo_root.clone(),
+        args.install_path,
+        args.no_track,
+        cargo_home,
+        &mut config,
+    )?;
+
+    let mut dest = install_path.join("cargo-binstall");
+    if cfg!(windows) {
+        assert!(dest.set_extension("exe"));
+    }
+
+    atomic_install(&env::current_exe().map_err(BinstallError::from)?, &dest)
+        .map_err(BinstallError::from)?;
+
+    if let Some(manifests) = manifests {
+        manifests.update(vec![CrateInfo {
+            name: CompactString::const_new("cargo-binstall"),
+            version_req: CompactString::const_new("*"),
+            current_version: Version::new(
+                env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(),
+                env!("CARGO_PKG_VERSION_MINOR").parse().unwrap(),
+                env!("CARGO_PKG_VERSION_PATCH").parse().unwrap(),
+            ),
+            source: CrateSource::cratesio_registry(),
+            target: CompactString::const_new(TARGET),
+            bins: vec![CompactString::const_new("cargo-binstall")],
+        }])?;
+    }
+
+    Ok(())
+}

crates/bin/src/main_impl.rs

@@ -12,9 +12,6 @@ use crate::{
 };
 
 pub fn do_main() -> impl Termination {
-    // This must be the very first thing to happen
-    let jobserver_client = LazyJobserverClient::new();
-
     let (args, cli_overrides) = args::parse();
 
     if args.version {
@@ -46,6 +43,8 @@ rustc-llvm-version: {rustc_llvm_version}"#
             println!("{cargo_binstall_version}");
         }
         MainExit::Success(None)
+    } else if args.self_install {
+        MainExit::new(entry::self_install(args), None)
     } else {
         logging(
             args.log_level.unwrap_or(LevelFilter::Info),
@@ -54,12 +53,14 @@ rustc-llvm-version: {rustc_llvm_version}"#
 
         let start = Instant::now();
 
+        let jobserver_client = LazyJobserverClient::new();
+
         let result =
             run_tokio_main(|| entry::install_crates(args, cli_overrides, jobserver_client));
 
         let done = start.elapsed();
         debug!("run time: {done:?}");
 
-        MainExit::new(result, done)
+        MainExit::new(result, Some(done))
     }
 }

e2e-tests/self-install.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+unset CARGO_INSTALL_ROOT
+
+CARGO_HOME=$(mktemp -d 2>/dev/null || mktemp -d -t 'cargo-home')
+export CARGO_HOME
+export PATH="$CARGO_HOME/bin:$PATH"
+
+"./$1" --self-install
+
+cargo binstall --help
+cargo install --list

install-from-binstall-release.ps1

@@ -19,7 +19,12 @@ $url = "$base_url$arch-pc-windows-msvc.zip"
 Invoke-WebRequest $url -OutFile $tmpdir\cargo-binstall.zip
 Expand-Archive -Force $tmpdir\cargo-binstall.zip $tmpdir\cargo-binstall
 Write-Host ""
-Invoke-Expression "$tmpdir\cargo-binstall\cargo-binstall.exe -y --force cargo-binstall"
+
+$ps = Start-Process -PassThru -Wait "$tmpdir\cargo-binstall\cargo-binstall.exe" "--self-install"
+if ($ps.ExitCode -ne 0) {
+    Invoke-Expression "$tmpdir\cargo-binstall\cargo-binstall.exe -y --force cargo-binstall"
+}
+
 Remove-Item -Force $tmpdir\cargo-binstall.zip
 Remove-Item -Recurse -Force $tmpdir\cargo-binstall
 $cargo_home = if ($Env:CARGO_HOME -ne $null) { $Env:CARGO_HOME } else { "$HOME\.cargo" }

install-from-binstall-release.sh

@@ -36,7 +36,7 @@ else
     exit 1
 fi
 
-./cargo-binstall -y --force cargo-binstall
+./cargo-binstall --self-install || ./cargo-binstall -y --force cargo-binstall
 
 CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
 

justfile

@@ -244,6 +244,7 @@ e2e-test-registries: (e2e-test "registries")
 e2e-test-signing: (e2e-test "signing")
 e2e-test-continue-on-failure: (e2e-test "continue-on-failure")
 e2e-test-private-github-repo: (e2e-test "private-github-repo")
+e2e-test-self-install: (e2e-test "self-install")
 
 # WinTLS (Windows in CI) does not have TLS 1.3 support
 [windows]
@@ -252,7 +253,7 @@ e2e-test-tls: (e2e-test "tls" "1.2")
 [macos]
 e2e-test-tls: (e2e-test "tls" "1.2") (e2e-test "tls" "1.3")
 
-e2e-tests: e2e-test-live e2e-test-manifest-path e2e-test-git e2e-test-other-repos e2e-test-strategies e2e-test-version-syntax e2e-test-upgrade e2e-test-tls e2e-test-self-upgrade-no-symlink e2e-test-uninstall e2e-test-subcrate e2e-test-no-track e2e-test-registries e2e-test-signing e2e-test-continue-on-failure e2e-test-private-github-repo
+e2e-tests: e2e-test-live e2e-test-manifest-path e2e-test-git e2e-test-other-repos e2e-test-strategies e2e-test-version-syntax e2e-test-upgrade e2e-test-tls e2e-test-self-upgrade-no-symlink e2e-test-uninstall e2e-test-subcrate e2e-test-no-track e2e-test-registries e2e-test-signing e2e-test-continue-on-failure e2e-test-private-github-repo e2e-test-self-install
 
 unit-tests: print-env
     cargo test --no-run --target {{target}}