mirror of
https://github.com/cargo-bins/cargo-binstall.git
synced 2025-05-15 08:20:03 +00:00
Support private github repository (#1690)
* Refactor: Create new crate binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix CI lint warnings Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `just check`: Rm deleted features from `cargo-hack` check Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new mod error Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Ret artifact url in `has_release_artifact` So that we can use it to download from private repositories. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Move `test_graph_ql_error_type` to mod `error` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix running `cargo test` in `binstalk-git-repo-api`` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unnecessary import in mod `error::test` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename mod `request`` to `release_artifacts` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl draft version of fetching repo info Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Move `HasReleaseArtifacts` failure variants into `GhApiError` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Use `GhRepo` in `GhRelease` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Return `'static` future Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Make sure `'static` Future is returned To make it easier to create generic function Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add logging to unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new fn `GhApiClient::do_fetch` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rm unused `percent_encode_http_url_path` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `cargo test` run on CI `cargo test` run all tests in one process. As such, `set_global_default` would fail on the second call. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Optimize `GhApiClient::do_fetch`: Avoid unnecessary restful API call Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Rm param `auth_token` for restful API fn which is always set to `None` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit test for `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor testing: Parameter-ize testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelise `test_get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Create parameter-ised `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelize `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: `gh_api_client::test::create_client` shall not be `async` as there is no `.await` in it. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Return `Url` in `GhApiClient::has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::download_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unused deps added to binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new API `GhApiClient::remote_client` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhApiClient::has_gh_token` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhRepo::try_extract_from_url` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename `ReleaseArtifactUrl` to `GhReleaseArtifactUrl` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new fn `Download::with_data_verifier` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * feature: Support private repository Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add e2e-test/private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `launch_baseline_find_tasks`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix test failure: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Temporarily enable debug output for e2e-test-private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `get_repo_info`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Improve `debug!` logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debugging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Apply suggestions from code review * Fix compilation Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix cargo fmt Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add crate binstalk-git-repo-api to release-pr.yml * Update crates/binstalk-git-repo-api/Cargo.toml * Apply suggestions from code review * Update crates/binstalk/Cargo.toml --------- Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU
GitHub
parent
48ee0b0e3e
commit
1dbd2460a3
30 changed files with 1838 additions and 1127 deletions
|
|
@ -1,16 +1,18 @@
|
|||
use std::{borrow::Cow, fmt, iter, path::Path, sync::Arc};
|
||||
|
||||
use binstalk_git_repo_api::gh_api_client::{GhApiError, GhReleaseArtifact, GhReleaseArtifactUrl};
|
||||
use compact_str::{CompactString, ToCompactString};
|
||||
use either::Either;
|
||||
use leon::Template;
|
||||
use once_cell::sync::OnceCell;
|
||||
use strum::IntoEnumIterator;
|
||||
use tokio::time::sleep;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use url::Url;
|
||||
|
||||
use crate::{
|
||||
common::*, futures_resolver::FuturesResolver, Data, FetchError, InvalidPkgFmtError, RepoInfo,
|
||||
SignaturePolicy, SignatureVerifier, TargetDataErased,
|
||||
SignaturePolicy, SignatureVerifier, TargetDataErased, DEFAULT_GH_API_RETRY_DURATION,
|
||||
};
|
||||
|
||||
pub(crate) mod hosting;
|
||||
|
|
@ -31,6 +33,8 @@ struct Resolved {
|
|||
archive_suffix: Option<String>,
|
||||
repo: Option<String>,
|
||||
subcrate: Option<String>,
|
||||
gh_release_artifact_url: Option<GhReleaseArtifactUrl>,
|
||||
is_repo_private: bool,
|
||||
}
|
||||
|
||||
impl GhCrateMeta {
|
||||
|
|
@ -41,6 +45,7 @@ impl GhCrateMeta {
|
|||
pkg_url: &Template<'_>,
|
||||
repo: Option<&str>,
|
||||
subcrate: Option<&str>,
|
||||
is_repo_private: bool,
|
||||
) {
|
||||
let render_url = |ext| {
|
||||
let ctx = Context::from_data_with_repo(
|
||||
|
|
@ -82,16 +87,45 @@ impl GhCrateMeta {
|
|||
let repo = repo.map(ToString::to_string);
|
||||
let subcrate = subcrate.map(ToString::to_string);
|
||||
let archive_suffix = ext.map(ToString::to_string);
|
||||
let gh_release_artifact = GhReleaseArtifact::try_extract_from_url(&url);
|
||||
|
||||
async move {
|
||||
Ok(does_url_exist(client, gh_api_client, &url)
|
||||
debug!("Checking for package at: '{url}'");
|
||||
|
||||
let mut resolved = Resolved {
|
||||
url: url.clone(),
|
||||
pkg_fmt,
|
||||
repo,
|
||||
subcrate,
|
||||
archive_suffix,
|
||||
is_repo_private,
|
||||
gh_release_artifact_url: None,
|
||||
};
|
||||
|
||||
if let Some(artifact) = gh_release_artifact {
|
||||
loop {
|
||||
match get_gh_release_artifact_url(gh_api_client.clone(), artifact.clone())
|
||||
.await
|
||||
{
|
||||
Ok(Some(artifact_url)) => {
|
||||
resolved.gh_release_artifact_url = Some(artifact_url);
|
||||
return Ok(Some(resolved));
|
||||
}
|
||||
Ok(None) => return Ok(None),
|
||||
|
||||
Err(GhApiError::RateLimit { retry_after }) => {
|
||||
sleep(retry_after.unwrap_or(DEFAULT_GH_API_RETRY_DURATION)).await;
|
||||
}
|
||||
Err(GhApiError::Unauthorized) if !is_repo_private => break,
|
||||
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Box::pin(client.remote_gettable(url))
|
||||
.await?
|
||||
.then_some(Resolved {
|
||||
url,
|
||||
pkg_fmt,
|
||||
repo,
|
||||
subcrate,
|
||||
archive_suffix,
|
||||
}))
|
||||
.then_some(resolved))
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
|
@ -118,10 +152,11 @@ impl super::Fetcher for GhCrateMeta {
|
|||
|
||||
fn find(self: Arc<Self>) -> JoinHandle<Result<bool, FetchError>> {
|
||||
tokio::spawn(async move {
|
||||
let info = self.data.get_repo_info(&self.client).await?.as_ref();
|
||||
let info = self.data.get_repo_info(&self.gh_api_client).await?;
|
||||
|
||||
let repo = info.map(|info| &info.repo);
|
||||
let subcrate = info.and_then(|info| info.subcrate.as_deref());
|
||||
let is_repo_private = info.map(|info| info.is_private).unwrap_or_default();
|
||||
|
||||
let mut pkg_fmt = self.target_data.meta.pkg_fmt;
|
||||
|
||||
|
|
@ -230,13 +265,22 @@ impl super::Fetcher for GhCrateMeta {
|
|||
// basically cartesian product.
|
||||
// |
|
||||
for pkg_fmt in pkg_fmts.clone() {
|
||||
this.launch_baseline_find_tasks(&resolver, pkg_fmt, &pkg_url, repo, subcrate);
|
||||
this.launch_baseline_find_tasks(
|
||||
&resolver,
|
||||
pkg_fmt,
|
||||
&pkg_url,
|
||||
repo,
|
||||
subcrate,
|
||||
is_repo_private,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(resolved) = resolver.resolve().await? {
|
||||
debug!(?resolved, "Winning URL found!");
|
||||
self.resolution.set(resolved).unwrap(); // find() is called first
|
||||
self.resolution
|
||||
.set(resolved)
|
||||
.expect("find() should be only called once");
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
|
|
@ -245,7 +289,10 @@ impl super::Fetcher for GhCrateMeta {
|
|||
}
|
||||
|
||||
async fn fetch_and_extract(&self, dst: &Path) -> Result<ExtractedFiles, FetchError> {
|
||||
let resolved = self.resolution.get().unwrap(); // find() is called first
|
||||
let resolved = self
|
||||
.resolution
|
||||
.get()
|
||||
.expect("find() should be called once before fetch_and_extract()");
|
||||
trace!(?resolved, "preparing to fetch");
|
||||
|
||||
let verifier = match (self.signature_policy, &self.target_data.meta.signing) {
|
||||
|
|
@ -290,11 +337,18 @@ impl super::Fetcher for GhCrateMeta {
|
|||
"Downloading package",
|
||||
);
|
||||
let mut data_verifier = verifier.data_verifier()?;
|
||||
let files = Download::new_with_data_verifier(
|
||||
self.client.clone(),
|
||||
resolved.url.clone(),
|
||||
data_verifier.as_mut(),
|
||||
)
|
||||
let files = match resolved.gh_release_artifact_url.as_ref() {
|
||||
Some(artifact_url) if resolved.is_repo_private => self
|
||||
.gh_api_client
|
||||
.download_artifact(artifact_url.clone())
|
||||
.await?
|
||||
.with_data_verifier(data_verifier.as_mut()),
|
||||
_ => Download::new_with_data_verifier(
|
||||
self.client.clone(),
|
||||
resolved.url.clone(),
|
||||
data_verifier.as_mut(),
|
||||
),
|
||||
}
|
||||
.and_extract(resolved.pkg_fmt, dst)
|
||||
.await?;
|
||||
trace!("validating signature (if any)");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue