mirror of
https://github.com/cargo-bins/cargo-binstall.git
synced 2025-06-03 09:32:55 +00:00
Support private github repository (#1690)
* Refactor: Create new crate binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix CI lint warnings Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `just check`: Rm deleted features from `cargo-hack` check Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new mod error Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Ret artifact url in `has_release_artifact` So that we can use it to download from private repositories. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Move `test_graph_ql_error_type` to mod `error` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix running `cargo test` in `binstalk-git-repo-api`` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unnecessary import in mod `error::test` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename mod `request`` to `release_artifacts` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl draft version of fetching repo info Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Move `HasReleaseArtifacts` failure variants into `GhApiError` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Use `GhRepo` in `GhRelease` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Return `'static` future Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Make sure `'static` Future is returned To make it easier to create generic function Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add logging to unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new fn `GhApiClient::do_fetch` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rm unused `percent_encode_http_url_path` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `cargo test` run on CI `cargo test` run all tests in one process. As such, `set_global_default` would fail on the second call. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Optimize `GhApiClient::do_fetch`: Avoid unnecessary restful API call Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Rm param `auth_token` for restful API fn which is always set to `None` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit test for `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor testing: Parameter-ize testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelise `test_get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Create parameter-ised `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelize `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: `gh_api_client::test::create_client` shall not be `async` as there is no `.await` in it. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Return `Url` in `GhApiClient::has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::download_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unused deps added to binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new API `GhApiClient::remote_client` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhApiClient::has_gh_token` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhRepo::try_extract_from_url` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename `ReleaseArtifactUrl` to `GhReleaseArtifactUrl` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new fn `Download::with_data_verifier` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * feature: Support private repository Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add e2e-test/private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `launch_baseline_find_tasks`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix test failure: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Temporarily enable debug output for e2e-test-private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `get_repo_info`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Improve `debug!` logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debugging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Apply suggestions from code review * Fix compilation Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix cargo fmt Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add crate binstalk-git-repo-api to release-pr.yml * Update crates/binstalk-git-repo-api/Cargo.toml * Apply suggestions from code review * Update crates/binstalk/Cargo.toml --------- Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU
GitHub
parent
48ee0b0e3e
commit
1dbd2460a3
30 changed files with 1838 additions and 1127 deletions
|
|
@ -3,12 +3,12 @@ use std::sync::{
|
|||
Once,
|
||||
};
|
||||
|
||||
use binstalk_downloader::gh_api_client::{GhReleaseArtifact, HasReleaseArtifact};
|
||||
pub(super) use binstalk_downloader::{
|
||||
download::{Download, ExtractedFiles},
|
||||
gh_api_client::GhApiClient,
|
||||
remote::{Client, Url},
|
||||
};
|
||||
pub(super) use binstalk_git_repo_api::gh_api_client::GhApiClient;
|
||||
use binstalk_git_repo_api::gh_api_client::{GhApiError, GhReleaseArtifact, GhReleaseArtifactUrl};
|
||||
pub(super) use binstalk_types::cargo_toml_binstall::{PkgFmt, PkgMeta};
|
||||
pub(super) use compact_str::CompactString;
|
||||
pub(super) use tokio::task::JoinHandle;
|
||||
|
|
@ -16,6 +16,39 @@ pub(super) use tracing::{debug, instrument, warn};
|
|||
|
||||
use crate::FetchError;
|
||||
|
||||
static WARN_RATE_LIMIT_ONCE: Once = Once::new();
|
||||
static WARN_UNAUTHORIZED_ONCE: Once = Once::new();
|
||||
|
||||
pub(super) async fn get_gh_release_artifact_url(
|
||||
gh_api_client: GhApiClient,
|
||||
artifact: GhReleaseArtifact,
|
||||
) -> Result<Option<GhReleaseArtifactUrl>, GhApiError> {
|
||||
debug!("Using GitHub API to check for existence of artifact, which will also cache the API response");
|
||||
|
||||
// The future returned has the same size as a pointer
|
||||
match gh_api_client.has_release_artifact(artifact).await {
|
||||
Ok(ret) => Ok(ret),
|
||||
Err(GhApiError::NotFound) => Ok(None),
|
||||
|
||||
Err(GhApiError::RateLimit { retry_after }) => {
|
||||
WARN_RATE_LIMIT_ONCE.call_once(|| {
|
||||
warn!("Your GitHub API token (if any) has reached its rate limit and cannot be used again until {retry_after:?}, so we will fallback to HEAD/GET on the url.");
|
||||
warn!("If you did not supply a github token, consider doing so: GitHub limits unauthorized users to 60 requests per hour per origin IP address.");
|
||||
});
|
||||
Err(GhApiError::RateLimit { retry_after })
|
||||
}
|
||||
Err(GhApiError::Unauthorized) => {
|
||||
WARN_UNAUTHORIZED_ONCE.call_once(|| {
|
||||
warn!("GitHub API somehow requires a token for the API access, so we will fallback to HEAD/GET on the url.");
|
||||
warn!("Please consider supplying a token to cargo-binstall to speedup resolution.");
|
||||
});
|
||||
Err(GhApiError::Unauthorized)
|
||||
}
|
||||
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// This function returns a future where its size should be at most size of
|
||||
/// 2-4 pointers.
|
||||
pub(super) async fn does_url_exist(
|
||||
|
|
@ -24,32 +57,17 @@ pub(super) async fn does_url_exist(
|
|||
url: &Url,
|
||||
) -> Result<bool, FetchError> {
|
||||
static GH_API_CLIENT_FAILED: AtomicBool = AtomicBool::new(false);
|
||||
static WARN_RATE_LIMIT_ONCE: Once = Once::new();
|
||||
static WARN_UNAUTHORIZED_ONCE: Once = Once::new();
|
||||
|
||||
debug!("Checking for package at: '{url}'");
|
||||
|
||||
if !GH_API_CLIENT_FAILED.load(Relaxed) {
|
||||
if let Some(artifact) = GhReleaseArtifact::try_extract_from_url(url) {
|
||||
debug!("Using GitHub API to check for existence of artifact, which will also cache the API response");
|
||||
match get_gh_release_artifact_url(gh_api_client, artifact).await {
|
||||
Ok(ret) => return Ok(ret.is_some()),
|
||||
|
||||
// The future returned has the same size as a pointer
|
||||
match gh_api_client.has_release_artifact(artifact).await? {
|
||||
HasReleaseArtifact::Yes => return Ok(true),
|
||||
HasReleaseArtifact::No | HasReleaseArtifact::NoSuchRelease => return Ok(false),
|
||||
Err(GhApiError::RateLimit { .. }) | Err(GhApiError::Unauthorized) => {}
|
||||
|
||||
HasReleaseArtifact::RateLimit { retry_after } => {
|
||||
WARN_RATE_LIMIT_ONCE.call_once(|| {
|
||||
warn!("Your GitHub API token (if any) has reached its rate limit and cannot be used again until {retry_after:?}, so we will fallback to HEAD/GET on the url.");
|
||||
warn!("If you did not supply a github token, consider doing so: GitHub limits unauthorized users to 60 requests per hour per origin IP address.");
|
||||
});
|
||||
}
|
||||
HasReleaseArtifact::Unauthorized => {
|
||||
WARN_UNAUTHORIZED_ONCE.call_once(|| {
|
||||
warn!("GitHub API somehow requires a token for the API access, so we will fallback to HEAD/GET on the url.");
|
||||
warn!("Please consider supplying a token to cargo-binstall to speedup resolution.");
|
||||
});
|
||||
}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
|
||||
GH_API_CLIENT_FAILED.store(true, Relaxed);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue