mirror of
https://github.com/cargo-bins/cargo-binstall.git
synced 2025-04-24 22:30:03 +00:00
Support private github repository (#1690)
* Refactor: Create new crate binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix CI lint warnings Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `just check`: Rm deleted features from `cargo-hack` check Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new mod error Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Ret artifact url in `has_release_artifact` So that we can use it to download from private repositories. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Move `test_graph_ql_error_type` to mod `error` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix running `cargo test` in `binstalk-git-repo-api`` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unnecessary import in mod `error::test` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename mod `request`` to `release_artifacts` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl draft version of fetching repo info Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Move `HasReleaseArtifacts` failure variants into `GhApiError` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Use `GhRepo` in `GhRelease` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Return `'static` future Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Make sure `'static` Future is returned To make it easier to create generic function Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add logging to unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new fn `GhApiClient::do_fetch` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rm unused `percent_encode_http_url_path` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `cargo test` run on CI `cargo test` run all tests in one process. As such, `set_global_default` would fail on the second call. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Optimize `GhApiClient::do_fetch`: Avoid unnecessary restful API call Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Rm param `auth_token` for restful API fn which is always set to `None` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit test for `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor testing: Parameter-ize testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelise `test_get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Create parameter-ised `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelize `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: `gh_api_client::test::create_client` shall not be `async` as there is no `.await` in it. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Return `Url` in `GhApiClient::has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::download_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unused deps added to binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new API `GhApiClient::remote_client` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhApiClient::has_gh_token` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhRepo::try_extract_from_url` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename `ReleaseArtifactUrl` to `GhReleaseArtifactUrl` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new fn `Download::with_data_verifier` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * feature: Support private repository Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add e2e-test/private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `launch_baseline_find_tasks`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix test failure: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Temporarily enable debug output for e2e-test-private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `get_repo_info`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Improve `debug!` logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debugging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Apply suggestions from code review * Fix compilation Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix cargo fmt Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add crate binstalk-git-repo-api to release-pr.yml * Update crates/binstalk-git-repo-api/Cargo.toml * Apply suggestions from code review * Update crates/binstalk/Cargo.toml --------- Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU
GitHub
parent
48ee0b0e3e
commit
1dbd2460a3
30 changed files with 1838 additions and 1127 deletions
|
|
@ -11,9 +11,8 @@ license = "GPL-3.0-only"
|
|||
|
||||
[dependencies]
|
||||
binstalk-bins = { version = "0.4.0", path = "../binstalk-bins" }
|
||||
binstalk-downloader = { version = "0.10.3", path = "../binstalk-downloader", default-features = false, features = [
|
||||
"gh-api-client",
|
||||
] }
|
||||
binstalk-downloader = { version = "0.10.3", path = "../binstalk-downloader", default-features = false }
|
||||
binstalk-git-repo-api = { version = "0.0.0", path = "../binstalk-git-repo-api" }
|
||||
binstalk-fetchers = { version = "0.4.1", path = "../binstalk-fetchers", features = [
|
||||
"quickinstall",
|
||||
] }
|
||||
|
|
|
|||
|
|
@ -4,9 +4,7 @@ use std::{
|
|||
process::{ExitCode, ExitStatus, Termination},
|
||||
};
|
||||
|
||||
use binstalk_downloader::{
|
||||
download::DownloadError, gh_api_client::GhApiError, remote::Error as RemoteError,
|
||||
};
|
||||
use binstalk_downloader::{download::DownloadError, remote::Error as RemoteError};
|
||||
use binstalk_fetchers::FetchError;
|
||||
use compact_str::CompactString;
|
||||
use itertools::Itertools;
|
||||
|
|
@ -19,7 +17,8 @@ use tracing::{error, warn};
|
|||
use crate::{
|
||||
bins,
|
||||
helpers::{
|
||||
cargo_toml::Error as CargoTomlError, cargo_toml_workspace::Error as LoadManifestFromWSError,
|
||||
cargo_toml::Error as CargoTomlError,
|
||||
cargo_toml_workspace::Error as LoadManifestFromWSError, gh_api_client::GhApiError,
|
||||
},
|
||||
registry::{InvalidRegistryError, RegistryError},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
pub mod jobserver_client;
|
||||
pub mod remote;
|
||||
pub mod remote {
|
||||
pub use binstalk_downloader::remote::*;
|
||||
pub use url::ParseError as UrlParseError;
|
||||
}
|
||||
pub(crate) mod target_triple;
|
||||
pub mod tasks;
|
||||
|
||||
pub(crate) use binstalk_downloader::download;
|
||||
pub use binstalk_downloader::gh_api_client;
|
||||
pub use binstalk_git_repo_api::gh_api_client;
|
||||
|
||||
pub(crate) use cargo_toml_workspace::{self, cargo_toml};
|
||||
#[cfg(feature = "git")]
|
||||
|
|
|
|||
|
|
@ -1,54 +0,0 @@
|
|||
pub use binstalk_downloader::remote::*;
|
||||
pub use url::ParseError as UrlParseError;
|
||||
|
||||
use binstalk_downloader::gh_api_client::{GhApiClient, GhReleaseArtifact, HasReleaseArtifact};
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering::Relaxed},
|
||||
Once,
|
||||
};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::errors::BinstallError;
|
||||
|
||||
/// This function returns a future where its size should be at most size of
|
||||
/// 2 pointers.
|
||||
pub async fn does_url_exist(
|
||||
client: Client,
|
||||
gh_api_client: GhApiClient,
|
||||
url: &Url,
|
||||
) -> Result<bool, BinstallError> {
|
||||
static GH_API_CLIENT_FAILED: AtomicBool = AtomicBool::new(false);
|
||||
static WARN_RATE_LIMIT_ONCE: Once = Once::new();
|
||||
static WARN_UNAUTHORIZED_ONCE: Once = Once::new();
|
||||
|
||||
debug!("Checking for package at: '{url}'");
|
||||
|
||||
if !GH_API_CLIENT_FAILED.load(Relaxed) {
|
||||
if let Some(artifact) = GhReleaseArtifact::try_extract_from_url(url) {
|
||||
debug!("Using GitHub API to check for existence of artifact, which will also cache the API response");
|
||||
|
||||
// The future returned has the same size as a pointer
|
||||
match gh_api_client.has_release_artifact(artifact).await? {
|
||||
HasReleaseArtifact::Yes => return Ok(true),
|
||||
HasReleaseArtifact::No | HasReleaseArtifact::NoSuchRelease => return Ok(false),
|
||||
|
||||
HasReleaseArtifact::RateLimit { retry_after } => {
|
||||
WARN_RATE_LIMIT_ONCE.call_once(|| {
|
||||
warn!("Your GitHub API token (if any) has reached its rate limit and cannot be used again until {retry_after:?}, so we will fallback to HEAD/GET on the url.");
|
||||
warn!("If you did not supply a github token, consider doing so: GitHub limits unauthorized users to 60 requests per hour per origin IP address.");
|
||||
});
|
||||
}
|
||||
HasReleaseArtifact::Unauthorized => {
|
||||
WARN_UNAUTHORIZED_ONCE.call_once(|| {
|
||||
warn!("GitHub API somehow requires a token for the API access, so we will fallback to HEAD/GET on the url.");
|
||||
warn!("Please consider supplying a token to cargo-binstall to speedup resolution.");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
GH_API_CLIENT_FAILED.store(true, Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Box::pin(client.remote_gettable(url.clone())).await?)
|
||||
}
|
||||
|
|
@ -6,9 +6,7 @@ use semver::VersionReq;
|
|||
|
||||
use crate::{
|
||||
fetchers::{Data, Fetcher, SignaturePolicy, TargetDataErased},
|
||||
helpers::{
|
||||
self, gh_api_client::GhApiClient, jobserver_client::LazyJobserverClient, remote::Client,
|
||||
},
|
||||
helpers::{gh_api_client::GhApiClient, jobserver_client::LazyJobserverClient, remote::Client},
|
||||
manifests::cargo_toml_binstall::PkgOverride,
|
||||
registry::Registry,
|
||||
DesiredTargets,
|
||||
|
|
@ -23,7 +21,7 @@ pub type Resolver =
|
|||
#[non_exhaustive]
|
||||
pub enum CargoTomlFetchOverride {
|
||||
#[cfg(feature = "git")]
|
||||
Git(helpers::git::GitUrl),
|
||||
Git(crate::helpers::git::GitUrl),
|
||||
Path(PathBuf),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ use itertools::Itertools;
|
|||
use leon::Template;
|
||||
use maybe_owned::MaybeOwned;
|
||||
use semver::{Version, VersionReq};
|
||||
use tempfile::TempDir;
|
||||
use tokio::task::spawn_blocking;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
|
|
@ -21,7 +20,7 @@ use crate::{
|
|||
errors::{BinstallError, VersionParseError},
|
||||
fetchers::{Data, Fetcher, TargetData},
|
||||
helpers::{
|
||||
self, cargo_toml::Manifest, cargo_toml_workspace::load_manifest_from_workspace,
|
||||
cargo_toml::Manifest, cargo_toml_workspace::load_manifest_from_workspace,
|
||||
download::ExtractedFiles, remote::Client, target_triple::TargetTriple,
|
||||
tasks::AutoAbortJoinHandle,
|
||||
},
|
||||
|
|
@ -361,7 +360,7 @@ impl PackageInfo {
|
|||
}
|
||||
#[cfg(feature = "git")]
|
||||
Some(Git(git_url)) => {
|
||||
use helpers::git::{GitCancellationToken, Repository as GitRepository};
|
||||
use crate::helpers::git::{GitCancellationToken, Repository as GitRepository};
|
||||
|
||||
let git_url = git_url.clone();
|
||||
let name = name.clone();
|
||||
|
|
@ -370,7 +369,7 @@ impl PackageInfo {
|
|||
let cancel_on_drop = cancellation_token.clone().cancel_on_drop();
|
||||
|
||||
let ret = spawn_blocking(move || {
|
||||
let dir = TempDir::new()?;
|
||||
let dir = tempfile::TempDir::new()?;
|
||||
GitRepository::shallow_clone(git_url, dir.as_ref(), Some(cancellation_token))?;
|
||||
|
||||
load_manifest_from_workspace(dir.as_ref(), &name).map_err(BinstallError::from)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue