mirror of
https://github.com/cargo-bins/cargo-binstall.git
synced 2025-04-24 14:28:42 +00:00
Support private github repository (#1690)
* Refactor: Create new crate binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix CI lint warnings Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `just check`: Rm deleted features from `cargo-hack` check Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new mod error Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Ret artifact url in `has_release_artifact` So that we can use it to download from private repositories. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Move `test_graph_ql_error_type` to mod `error` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix running `cargo test` in `binstalk-git-repo-api`` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unnecessary import in mod `error::test` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename mod `request`` to `release_artifacts` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl draft version of fetching repo info Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Move `HasReleaseArtifacts` failure variants into `GhApiError` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Use `GhRepo` in `GhRelease` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Return `'static` future Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Make sure `'static` Future is returned To make it easier to create generic function Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add logging to unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Extract new fn `GhApiClient::do_fetch` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rm unused `percent_encode_http_url_path` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `cargo test` run on CI `cargo test` run all tests in one process. As such, `set_global_default` would fail on the second call. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Optimize `GhApiClient::do_fetch`: Avoid unnecessary restful API call Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Rm param `auth_token` for restful API fn which is always set to `None` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix unit test for `GhApiClient::get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor testing: Parameter-ize testing Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelise `test_get_repo_info` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: Create parameter-ised `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Parallelize `test_has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Refactor: `gh_api_client::test::create_client` shall not be `async` as there is no `.await` in it. Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Return `Url` in `GhApiClient::has_release_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Impl new API `GhApiClient::download_artifact` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Remove unused deps added to binstalk-git-repo-api Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new API `GhApiClient::remote_client` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhApiClient::has_gh_token` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add `GhRepo::try_extract_from_url` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Rename `ReleaseArtifactUrl` to `GhReleaseArtifactUrl` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add new fn `Download::with_data_verifier` Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * feature: Support private repository Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add e2e-test/private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix clippy lints Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `launch_baseline_find_tasks`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix test failure: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Temporarily enable debug output for e2e-test-private-github-repo Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix `get_repo_info`: Retry on rate limit Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Improve `debug!` logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debugging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add more debug logging Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Apply suggestions from code review * Fix compilation Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Fix cargo fmt Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com> * Add crate binstalk-git-repo-api to release-pr.yml * Update crates/binstalk-git-repo-api/Cargo.toml * Apply suggestions from code review * Update crates/binstalk/Cargo.toml --------- Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU
GitHub
parent
48ee0b0e3e
commit
1dbd2460a3
30 changed files with 1838 additions and 1127 deletions
28
crates/binstalk-git-repo-api/Cargo.toml
Normal file
28
crates/binstalk-git-repo-api/Cargo.toml
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
[package]
|
||||
name = "binstalk-git-repo-api"
|
||||
description = "The binstall toolkit for accessing API for git repository"
|
||||
repository = "https://github.com/cargo-bins/cargo-binstall"
|
||||
documentation = "https://docs.rs/binstalk-git-repo-api"
|
||||
version = "0.0.0"
|
||||
rust-version = "1.70.0"
|
||||
authors = ["Jiahao XU <Jiahao_XU@outlook.com>"]
|
||||
edition = "2021"
|
||||
license = "Apache-2.0 OR MIT"
|
||||
|
||||
[dependencies]
|
||||
binstalk-downloader = { version = "0.10.3", path = "../binstalk-downloader", default-features = false, features = [
|
||||
"json",
|
||||
] }
|
||||
compact_str = "0.7.0"
|
||||
percent-encoding = "2.2.0"
|
||||
serde = { version = "1.0.163", features = ["derive"] }
|
||||
serde-tuple-vec-map = "1.0.1"
|
||||
serde_json = { version = "1.0.107" }
|
||||
thiserror = "1.0.52"
|
||||
tokio = { version = "1.35.0", features = ["sync"], default-features = false }
|
||||
tracing = "0.1.39"
|
||||
url = "2.3.1"
|
||||
|
||||
[dev-dependencies]
|
||||
binstalk-downloader = { version = "0.10.3", path = "../binstalk-downloader" }
|
||||
tracing-subscriber = "0.3"
|
||||
176
crates/binstalk-git-repo-api/LICENSE-APACHE
Normal file
176
crates/binstalk-git-repo-api/LICENSE-APACHE
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
23
crates/binstalk-git-repo-api/LICENSE-MIT
Normal file
23
crates/binstalk-git-repo-api/LICENSE-MIT
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
Permission is hereby granted, free of charge, to any
|
||||
person obtaining a copy of this software and associated
|
||||
documentation files (the "Software"), to deal in the
|
||||
Software without restriction, including without
|
||||
limitation the rights to use, copy, modify, merge,
|
||||
publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software
|
||||
is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
The above copyright notice and this permission notice
|
||||
shall be included in all copies or substantial portions
|
||||
of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
|
||||
ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
|
||||
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
||||
SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
|
||||
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
703
crates/binstalk-git-repo-api/src/gh_api_client.rs
Normal file
703
crates/binstalk-git-repo-api/src/gh_api_client.rs
Normal file
|
|
@ -0,0 +1,703 @@
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
future::Future,
|
||||
ops::Deref,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering::Relaxed},
|
||||
Arc, Mutex, RwLock,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use binstalk_downloader::{download::Download, remote};
|
||||
use compact_str::{format_compact, CompactString, ToCompactString};
|
||||
use tokio::sync::OnceCell;
|
||||
use tracing::instrument;
|
||||
use url::Url;
|
||||
|
||||
mod common;
|
||||
mod error;
|
||||
mod release_artifacts;
|
||||
mod repo_info;
|
||||
|
||||
use common::{check_http_status_and_header, percent_decode_http_url_path};
|
||||
pub use error::{GhApiContextError, GhApiError, GhGraphQLErrors};
|
||||
pub use repo_info::RepoInfo;
|
||||
|
||||
/// default retry duration if x-ratelimit-reset is not found in response header
|
||||
const DEFAULT_RETRY_DURATION: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
#[derive(Clone, Eq, PartialEq, Hash, Debug)]
|
||||
pub struct GhRepo {
|
||||
pub owner: CompactString,
|
||||
pub repo: CompactString,
|
||||
}
|
||||
impl GhRepo {
|
||||
pub fn repo_url(&self) -> Result<Url, url::ParseError> {
|
||||
Url::parse(&format_compact!(
|
||||
"https://github.com/{}/{}",
|
||||
self.owner,
|
||||
self.repo
|
||||
))
|
||||
}
|
||||
|
||||
pub fn try_extract_from_url(url: &Url) -> Option<Self> {
|
||||
if url.domain() != Some("github.com") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut path_segments = url.path_segments()?;
|
||||
|
||||
Some(Self {
|
||||
owner: path_segments.next()?.to_compact_string(),
|
||||
repo: path_segments.next()?.to_compact_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The keys required to identify a github release.
|
||||
#[derive(Clone, Eq, PartialEq, Hash, Debug)]
|
||||
pub struct GhRelease {
|
||||
pub repo: GhRepo,
|
||||
pub tag: CompactString,
|
||||
}
|
||||
|
||||
/// The Github Release and one of its artifact.
|
||||
#[derive(Clone, Eq, PartialEq, Hash, Debug)]
|
||||
pub struct GhReleaseArtifact {
|
||||
pub release: GhRelease,
|
||||
pub artifact_name: CompactString,
|
||||
}
|
||||
|
||||
impl GhReleaseArtifact {
|
||||
/// Create [`GhReleaseArtifact`] from url.
|
||||
pub fn try_extract_from_url(url: &remote::Url) -> Option<Self> {
|
||||
if url.domain() != Some("github.com") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut path_segments = url.path_segments()?;
|
||||
|
||||
let owner = path_segments.next()?;
|
||||
let repo = path_segments.next()?;
|
||||
|
||||
if (path_segments.next()?, path_segments.next()?) != ("releases", "download") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let tag = path_segments.next()?;
|
||||
let artifact_name = path_segments.next()?;
|
||||
|
||||
(path_segments.next().is_none() && url.fragment().is_none() && url.query().is_none()).then(
|
||||
|| Self {
|
||||
release: GhRelease {
|
||||
repo: GhRepo {
|
||||
owner: percent_decode_http_url_path(owner),
|
||||
repo: percent_decode_http_url_path(repo),
|
||||
},
|
||||
tag: percent_decode_http_url_path(tag),
|
||||
},
|
||||
artifact_name: percent_decode_http_url_path(artifact_name),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Map<K, V>(RwLock<HashMap<K, Arc<V>>>);
|
||||
|
||||
impl<K, V> Default for Map<K, V> {
|
||||
fn default() -> Self {
|
||||
Self(Default::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl<K, V> Map<K, V>
|
||||
where
|
||||
K: Eq + std::hash::Hash,
|
||||
V: Default,
|
||||
{
|
||||
fn get(&self, k: K) -> Arc<V> {
|
||||
let optional_value = self.0.read().unwrap().deref().get(&k).cloned();
|
||||
optional_value.unwrap_or_else(|| Arc::clone(self.0.write().unwrap().entry(k).or_default()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Inner {
|
||||
client: remote::Client,
|
||||
release_artifacts: Map<GhRelease, OnceCell<Option<release_artifacts::Artifacts>>>,
|
||||
retry_after: Mutex<Option<Instant>>,
|
||||
|
||||
auth_token: Option<CompactString>,
|
||||
is_auth_token_valid: AtomicBool,
|
||||
}
|
||||
|
||||
/// Github API client for querying whether a release artifact exitsts.
|
||||
/// Can only handle github.com for now.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GhApiClient(Arc<Inner>);
|
||||
|
||||
impl GhApiClient {
|
||||
pub fn new(client: remote::Client, auth_token: Option<CompactString>) -> Self {
|
||||
Self(Arc::new(Inner {
|
||||
client,
|
||||
release_artifacts: Default::default(),
|
||||
retry_after: Default::default(),
|
||||
|
||||
auth_token,
|
||||
is_auth_token_valid: AtomicBool::new(true),
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn remote_client(&self) -> &remote::Client {
|
||||
&self.0.client
|
||||
}
|
||||
}
|
||||
|
||||
impl GhApiClient {
|
||||
fn check_retry_after(&self) -> Result<(), GhApiError> {
|
||||
let mut guard = self.0.retry_after.lock().unwrap();
|
||||
|
||||
if let Some(retry_after) = *guard {
|
||||
if retry_after.elapsed().is_zero() {
|
||||
return Err(GhApiError::RateLimit {
|
||||
retry_after: Some(retry_after - Instant::now()),
|
||||
});
|
||||
} else {
|
||||
// Instant retry_after is already reached.
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_auth_token(&self) -> Option<&str> {
|
||||
if self.0.is_auth_token_valid.load(Relaxed) {
|
||||
self.0.auth_token.as_deref()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_gh_token(&self) -> bool {
|
||||
self.get_auth_token().is_some()
|
||||
}
|
||||
|
||||
async fn do_fetch<T, U, GraphQLFn, RestfulFn, GraphQLFut, RestfulFut>(
|
||||
&self,
|
||||
graphql_func: GraphQLFn,
|
||||
restful_func: RestfulFn,
|
||||
data: &T,
|
||||
) -> Result<U, GhApiError>
|
||||
where
|
||||
GraphQLFn: Fn(&remote::Client, &T, &str) -> GraphQLFut,
|
||||
RestfulFn: Fn(&remote::Client, &T) -> RestfulFut,
|
||||
GraphQLFut: Future<Output = Result<U, GhApiError>> + Send + Sync + 'static,
|
||||
RestfulFut: Future<Output = Result<U, GhApiError>> + Send + Sync + 'static,
|
||||
{
|
||||
self.check_retry_after()?;
|
||||
|
||||
if let Some(auth_token) = self.get_auth_token() {
|
||||
match graphql_func(&self.0.client, data, auth_token).await {
|
||||
Err(GhApiError::Unauthorized) => {
|
||||
self.0.is_auth_token_valid.store(false, Relaxed);
|
||||
}
|
||||
res => return res.map_err(|err| err.context("GraphQL API")),
|
||||
}
|
||||
}
|
||||
|
||||
restful_func(&self.0.client, data)
|
||||
.await
|
||||
.map_err(|err| err.context("Restful API"))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self), ret)]
|
||||
pub async fn get_repo_info(&self, repo: &GhRepo) -> Result<Option<RepoInfo>, GhApiError> {
|
||||
match self
|
||||
.do_fetch(
|
||||
repo_info::fetch_repo_info_graphql_api,
|
||||
repo_info::fetch_repo_info_restful_api,
|
||||
repo,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(repo_info) => Ok(repo_info),
|
||||
Err(GhApiError::NotFound) => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
|
||||
pub struct GhReleaseArtifactUrl(Url);
|
||||
|
||||
impl GhApiClient {
|
||||
/// Return `Ok(Some(api_artifact_url))` if exists.
|
||||
///
|
||||
/// The returned future is guaranteed to be pointer size.
|
||||
#[instrument(level = "debug", skip(self), ret)]
|
||||
pub async fn has_release_artifact(
|
||||
&self,
|
||||
GhReleaseArtifact {
|
||||
release,
|
||||
artifact_name,
|
||||
}: GhReleaseArtifact,
|
||||
) -> Result<Option<GhReleaseArtifactUrl>, GhApiError> {
|
||||
let once_cell = self.0.release_artifacts.get(release.clone());
|
||||
let res = once_cell
|
||||
.get_or_try_init(|| {
|
||||
Box::pin(async {
|
||||
match self
|
||||
.do_fetch(
|
||||
release_artifacts::fetch_release_artifacts_graphql_api,
|
||||
release_artifacts::fetch_release_artifacts_restful_api,
|
||||
&release,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(artifacts) => Ok(Some(artifacts)),
|
||||
Err(GhApiError::NotFound) => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
})
|
||||
})
|
||||
.await;
|
||||
|
||||
match res {
|
||||
Ok(Some(artifacts)) => Ok(artifacts
|
||||
.get_artifact_url(&artifact_name)
|
||||
.map(GhReleaseArtifactUrl)),
|
||||
Ok(None) => Ok(None),
|
||||
Err(GhApiError::RateLimit { retry_after }) => {
|
||||
*self.0.retry_after.lock().unwrap() =
|
||||
Some(Instant::now() + retry_after.unwrap_or(DEFAULT_RETRY_DURATION));
|
||||
|
||||
Err(GhApiError::RateLimit { retry_after })
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn download_artifact(
|
||||
&self,
|
||||
artifact_url: GhReleaseArtifactUrl,
|
||||
) -> Result<Download<'static>, GhApiError> {
|
||||
self.check_retry_after()?;
|
||||
|
||||
let Some(auth_token) = self.get_auth_token() else {
|
||||
return Err(GhApiError::Unauthorized);
|
||||
};
|
||||
|
||||
let response = self
|
||||
.0
|
||||
.client
|
||||
.get(artifact_url.0)
|
||||
.header("Accept", "application/octet-stream")
|
||||
.bearer_auth(&auth_token)
|
||||
.send(false)
|
||||
.await?;
|
||||
|
||||
match check_http_status_and_header(&response) {
|
||||
Err(GhApiError::Unauthorized) => {
|
||||
self.0.is_auth_token_valid.store(false, Relaxed);
|
||||
}
|
||||
res => res?,
|
||||
}
|
||||
|
||||
Ok(Download::from_response(response))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use compact_str::{CompactString, ToCompactString};
|
||||
use std::{env, num::NonZeroU16, time::Duration};
|
||||
use tokio::time::sleep;
|
||||
use tracing::subscriber::set_global_default;
|
||||
use tracing_subscriber::{filter::LevelFilter, fmt::fmt};
|
||||
|
||||
static DEFAULT_RETRY_AFTER: Duration = Duration::from_secs(1);
|
||||
|
||||
mod cargo_binstall_v0_20_1 {
|
||||
use super::{CompactString, GhRelease, GhRepo};
|
||||
|
||||
pub(super) const RELEASE: GhRelease = GhRelease {
|
||||
repo: GhRepo {
|
||||
owner: CompactString::new_inline("cargo-bins"),
|
||||
repo: CompactString::new_inline("cargo-binstall"),
|
||||
},
|
||||
tag: CompactString::new_inline("v0.20.1"),
|
||||
};
|
||||
|
||||
pub(super) const ARTIFACTS: &[&str] = &[
|
||||
"cargo-binstall-aarch64-apple-darwin.full.zip",
|
||||
"cargo-binstall-aarch64-apple-darwin.zip",
|
||||
"cargo-binstall-aarch64-pc-windows-msvc.full.zip",
|
||||
"cargo-binstall-aarch64-pc-windows-msvc.zip",
|
||||
"cargo-binstall-aarch64-unknown-linux-gnu.full.tgz",
|
||||
"cargo-binstall-aarch64-unknown-linux-gnu.tgz",
|
||||
"cargo-binstall-aarch64-unknown-linux-musl.full.tgz",
|
||||
"cargo-binstall-aarch64-unknown-linux-musl.tgz",
|
||||
"cargo-binstall-armv7-unknown-linux-gnueabihf.full.tgz",
|
||||
"cargo-binstall-armv7-unknown-linux-gnueabihf.tgz",
|
||||
"cargo-binstall-armv7-unknown-linux-musleabihf.full.tgz",
|
||||
"cargo-binstall-armv7-unknown-linux-musleabihf.tgz",
|
||||
"cargo-binstall-universal-apple-darwin.full.zip",
|
||||
"cargo-binstall-universal-apple-darwin.zip",
|
||||
"cargo-binstall-x86_64-apple-darwin.full.zip",
|
||||
"cargo-binstall-x86_64-apple-darwin.zip",
|
||||
"cargo-binstall-x86_64-pc-windows-msvc.full.zip",
|
||||
"cargo-binstall-x86_64-pc-windows-msvc.zip",
|
||||
"cargo-binstall-x86_64-unknown-linux-gnu.full.tgz",
|
||||
"cargo-binstall-x86_64-unknown-linux-gnu.tgz",
|
||||
"cargo-binstall-x86_64-unknown-linux-musl.full.tgz",
|
||||
"cargo-binstall-x86_64-unknown-linux-musl.tgz",
|
||||
];
|
||||
}
|
||||
|
||||
mod cargo_audit_v_0_17_6 {
|
||||
use super::*;
|
||||
|
||||
pub(super) const RELEASE: GhRelease = GhRelease {
|
||||
repo: GhRepo {
|
||||
owner: CompactString::new_inline("rustsec"),
|
||||
repo: CompactString::new_inline("rustsec"),
|
||||
},
|
||||
tag: CompactString::new_inline("cargo-audit/v0.17.6"),
|
||||
};
|
||||
|
||||
pub(super) const ARTIFACTS: &[&str] = &[
|
||||
"cargo-audit-aarch64-unknown-linux-gnu-v0.17.6.tgz",
|
||||
"cargo-audit-armv7-unknown-linux-gnueabihf-v0.17.6.tgz",
|
||||
"cargo-audit-x86_64-apple-darwin-v0.17.6.tgz",
|
||||
"cargo-audit-x86_64-pc-windows-msvc-v0.17.6.zip",
|
||||
"cargo-audit-x86_64-unknown-linux-gnu-v0.17.6.tgz",
|
||||
"cargo-audit-x86_64-unknown-linux-musl-v0.17.6.tgz",
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn extract_with_escaped_characters() {
|
||||
let release_artifact = try_extract_artifact_from_str(
|
||||
"https://github.com/rustsec/rustsec/releases/download/cargo-audit%2Fv0.17.6/cargo-audit-aarch64-unknown-linux-gnu-v0.17.6.tgz"
|
||||
).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
release_artifact,
|
||||
GhReleaseArtifact {
|
||||
release: RELEASE,
|
||||
artifact_name: CompactString::from(
|
||||
"cargo-audit-aarch64-unknown-linux-gnu-v0.17.6.tgz",
|
||||
)
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gh_repo_extract_from_and_to_url() {
|
||||
[
|
||||
"https://github.com/cargo-bins/cargo-binstall",
|
||||
"https://github.com/rustsec/rustsec",
|
||||
]
|
||||
.into_iter()
|
||||
.for_each(|url| {
|
||||
let url = Url::parse(&url).unwrap();
|
||||
assert_eq!(
|
||||
GhRepo::try_extract_from_url(&url)
|
||||
.unwrap()
|
||||
.repo_url()
|
||||
.unwrap(),
|
||||
url
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
fn try_extract_artifact_from_str(s: &str) -> Option<GhReleaseArtifact> {
|
||||
GhReleaseArtifact::try_extract_from_url(&url::Url::parse(s).unwrap())
|
||||
}
|
||||
|
||||
fn assert_extract_gh_release_artifacts_failures(urls: &[&str]) {
|
||||
for url in urls {
|
||||
assert_eq!(try_extract_artifact_from_str(url), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_gh_release_artifacts_failure() {
|
||||
use cargo_binstall_v0_20_1::*;
|
||||
|
||||
let GhRelease {
|
||||
repo: GhRepo { owner, repo },
|
||||
tag,
|
||||
} = RELEASE;
|
||||
|
||||
assert_extract_gh_release_artifacts_failures(&[
|
||||
"https://examle.com",
|
||||
"https://github.com",
|
||||
&format!("https://github.com/{owner}"),
|
||||
&format!("https://github.com/{owner}/{repo}"),
|
||||
&format!("https://github.com/{owner}/{repo}/123e"),
|
||||
&format!("https://github.com/{owner}/{repo}/releases/21343"),
|
||||
&format!("https://github.com/{owner}/{repo}/releases/download"),
|
||||
&format!("https://github.com/{owner}/{repo}/releases/download/{tag}"),
|
||||
&format!("https://github.com/{owner}/{repo}/releases/download/{tag}/a/23"),
|
||||
&format!("https://github.com/{owner}/{repo}/releases/download/{tag}/a#a=12"),
|
||||
&format!("https://github.com/{owner}/{repo}/releases/download/{tag}/a?page=3"),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_gh_release_artifacts_success() {
|
||||
use cargo_binstall_v0_20_1::*;
|
||||
|
||||
let GhRelease {
|
||||
repo: GhRepo { owner, repo },
|
||||
tag,
|
||||
} = RELEASE;
|
||||
|
||||
for artifact in ARTIFACTS {
|
||||
let GhReleaseArtifact {
|
||||
release,
|
||||
artifact_name,
|
||||
} = try_extract_artifact_from_str(&format!(
|
||||
"https://github.com/{owner}/{repo}/releases/download/{tag}/{artifact}"
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(release, RELEASE);
|
||||
assert_eq!(artifact_name, artifact);
|
||||
}
|
||||
}
|
||||
|
||||
fn init_logger() {
|
||||
// Disable time, target, file, line_num, thread name/ids to make the
|
||||
// output more readable
|
||||
let subscriber = fmt()
|
||||
.without_time()
|
||||
.with_target(false)
|
||||
.with_file(false)
|
||||
.with_line_number(false)
|
||||
.with_thread_names(false)
|
||||
.with_thread_ids(false)
|
||||
.with_test_writer()
|
||||
.with_max_level(LevelFilter::DEBUG)
|
||||
.finish();
|
||||
|
||||
// Setup global subscriber
|
||||
let _ = set_global_default(subscriber);
|
||||
}
|
||||
|
||||
fn create_remote_client() -> remote::Client {
|
||||
remote::Client::new(
|
||||
concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")),
|
||||
None,
|
||||
NonZeroU16::new(10).unwrap(),
|
||||
1.try_into().unwrap(),
|
||||
[],
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Mark this as an async fn so that you won't accidentally use it in
|
||||
/// sync context.
|
||||
fn create_client() -> Vec<GhApiClient> {
|
||||
let client = create_remote_client();
|
||||
|
||||
let mut gh_clients = vec![GhApiClient::new(client.clone(), None)];
|
||||
|
||||
if let Ok(token) = env::var("GITHUB_TOKEN") {
|
||||
gh_clients.push(GhApiClient::new(client, Some(token.into())));
|
||||
}
|
||||
|
||||
gh_clients
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_repo_info() {
|
||||
const PUBLIC_REPOS: [GhRepo; 1] = [GhRepo {
|
||||
owner: CompactString::new_inline("cargo-bins"),
|
||||
repo: CompactString::new_inline("cargo-binstall"),
|
||||
}];
|
||||
const PRIVATE_REPOS: [GhRepo; 1] = [GhRepo {
|
||||
owner: CompactString::new_inline("cargo-bins"),
|
||||
repo: CompactString::new_inline("private-repo-for-testing"),
|
||||
}];
|
||||
const NON_EXISTENT_REPOS: [GhRepo; 1] = [GhRepo {
|
||||
owner: CompactString::new_inline("cargo-bins"),
|
||||
repo: CompactString::new_inline("ttt"),
|
||||
}];
|
||||
|
||||
init_logger();
|
||||
|
||||
let mut tests: Vec<(_, _)> = Vec::new();
|
||||
|
||||
for client in create_client() {
|
||||
for repo in PUBLIC_REPOS {
|
||||
let client = client.clone();
|
||||
|
||||
tests.push((
|
||||
Some(RepoInfo::new(repo.clone(), false)),
|
||||
tokio::spawn(async move { client.get_repo_info(&repo).await }),
|
||||
));
|
||||
}
|
||||
|
||||
for repo in NON_EXISTENT_REPOS {
|
||||
let client = client.clone();
|
||||
|
||||
tests.push((
|
||||
None,
|
||||
tokio::spawn(async move { client.get_repo_info(&repo).await }),
|
||||
));
|
||||
}
|
||||
|
||||
if client.has_gh_token() {
|
||||
for repo in PRIVATE_REPOS {
|
||||
let client = client.clone();
|
||||
|
||||
tests.push((
|
||||
Some(RepoInfo::new(repo.clone(), true)),
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
match client.get_repo_info(&repo).await {
|
||||
Err(GhApiError::RateLimit { retry_after }) => {
|
||||
sleep(retry_after.unwrap_or(DEFAULT_RETRY_AFTER)).await
|
||||
}
|
||||
res => break res,
|
||||
}
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (expected, task) in tests {
|
||||
assert_eq!(task.await.unwrap().unwrap(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_has_release_artifact_and_download_artifacts() {
|
||||
const RELEASES: [(GhRelease, &[&str]); 2] = [
|
||||
(
|
||||
cargo_binstall_v0_20_1::RELEASE,
|
||||
cargo_binstall_v0_20_1::ARTIFACTS,
|
||||
),
|
||||
(
|
||||
cargo_audit_v_0_17_6::RELEASE,
|
||||
cargo_audit_v_0_17_6::ARTIFACTS,
|
||||
),
|
||||
];
|
||||
const NON_EXISTENT_RELEASES: [GhRelease; 1] = [GhRelease {
|
||||
repo: GhRepo {
|
||||
owner: CompactString::new_inline("cargo-bins"),
|
||||
repo: CompactString::new_inline("cargo-binstall"),
|
||||
},
|
||||
// We are currently at v0.20.1 and we would never release
|
||||
// anything older than v0.20.1
|
||||
tag: CompactString::new_inline("v0.18.2"),
|
||||
}];
|
||||
|
||||
init_logger();
|
||||
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for client in create_client() {
|
||||
for (release, artifacts) in RELEASES {
|
||||
for artifact_name in artifacts {
|
||||
let client = client.clone();
|
||||
let release = release.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let artifact = GhReleaseArtifact {
|
||||
release,
|
||||
artifact_name: artifact_name.to_compact_string(),
|
||||
};
|
||||
|
||||
let browser_download_task = client.get_auth_token().map(|_| {
|
||||
tokio::spawn(
|
||||
Download::new(
|
||||
client.remote_client().clone(),
|
||||
Url::parse(&format!(
|
||||
"https://github.com/{}/{}/releases/download/{}/{}",
|
||||
artifact.release.repo.owner,
|
||||
artifact.release.repo.repo,
|
||||
artifact.release.tag,
|
||||
artifact.artifact_name,
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.into_bytes(),
|
||||
)
|
||||
});
|
||||
|
||||
let artifact_url = loop {
|
||||
match client.has_release_artifact(artifact.clone()).await {
|
||||
Err(GhApiError::RateLimit { retry_after }) => {
|
||||
sleep(retry_after.unwrap_or(DEFAULT_RETRY_AFTER)).await
|
||||
}
|
||||
res => break res.unwrap().unwrap(),
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(browser_download_task) = browser_download_task {
|
||||
let artifact_download_data = loop {
|
||||
match client.download_artifact(artifact_url.clone()).await {
|
||||
Err(GhApiError::RateLimit { retry_after }) => {
|
||||
sleep(retry_after.unwrap_or(DEFAULT_RETRY_AFTER)).await
|
||||
}
|
||||
res => break res.unwrap(),
|
||||
}
|
||||
}
|
||||
.into_bytes()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let browser_download_data =
|
||||
browser_download_task.await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(artifact_download_data, browser_download_data);
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
let client = client.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
assert_eq!(
|
||||
client
|
||||
.has_release_artifact(GhReleaseArtifact {
|
||||
release,
|
||||
artifact_name: "123z".to_compact_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
}));
|
||||
}
|
||||
|
||||
for release in NON_EXISTENT_RELEASES {
|
||||
let client = client.clone();
|
||||
|
||||
tasks.push(tokio::spawn(async move {
|
||||
assert_eq!(
|
||||
client
|
||||
.has_release_artifact(GhReleaseArtifact {
|
||||
release,
|
||||
artifact_name: "1234".to_compact_string(),
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
None
|
||||
);
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
for task in tasks {
|
||||
task.await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
143
crates/binstalk-git-repo-api/src/gh_api_client/common.rs
Normal file
143
crates/binstalk-git-repo-api/src/gh_api_client/common.rs
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
use std::{fmt::Debug, future::Future, sync::OnceLock, time::Duration};
|
||||
|
||||
use binstalk_downloader::remote::{self, Response, Url};
|
||||
use compact_str::CompactString;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use serde_json::to_string as to_json_string;
|
||||
use tracing::debug;
|
||||
|
||||
use super::{GhApiError, GhGraphQLErrors};
|
||||
|
||||
pub(super) fn percent_decode_http_url_path(input: &str) -> CompactString {
|
||||
if input.contains('%') {
|
||||
percent_decode_str(input).decode_utf8_lossy().into()
|
||||
} else {
|
||||
// No '%', no need to decode.
|
||||
CompactString::new(input)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn check_http_status_and_header(response: &Response) -> Result<(), GhApiError> {
|
||||
let headers = response.headers();
|
||||
|
||||
match response.status() {
|
||||
remote::StatusCode::FORBIDDEN
|
||||
if headers
|
||||
.get("x-ratelimit-remaining")
|
||||
.map(|val| val == "0")
|
||||
.unwrap_or(false) =>
|
||||
{
|
||||
Err(GhApiError::RateLimit {
|
||||
retry_after: headers.get("x-ratelimit-reset").and_then(|value| {
|
||||
let secs = value.to_str().ok()?.parse().ok()?;
|
||||
Some(Duration::from_secs(secs))
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
remote::StatusCode::UNAUTHORIZED => Err(GhApiError::Unauthorized),
|
||||
remote::StatusCode::NOT_FOUND => Err(GhApiError::NotFound),
|
||||
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_api_endpoint() -> &'static Url {
|
||||
static API_ENDPOINT: OnceLock<Url> = OnceLock::new();
|
||||
|
||||
API_ENDPOINT.get_or_init(|| {
|
||||
Url::parse("https://api.github.com/").expect("Literal provided must be a valid url")
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn issue_restful_api<T>(
|
||||
client: &remote::Client,
|
||||
path: &[&str],
|
||||
) -> impl Future<Output = Result<T, GhApiError>> + Send + Sync + 'static
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
let mut url = get_api_endpoint().clone();
|
||||
|
||||
url.path_segments_mut()
|
||||
.expect("get_api_endpoint() should return a https url")
|
||||
.extend(path);
|
||||
|
||||
debug!("Getting restful API: {url}");
|
||||
|
||||
let future = client
|
||||
.get(url)
|
||||
.header("Accept", "application/vnd.github+json")
|
||||
.header("X-GitHub-Api-Version", "2022-11-28")
|
||||
.send(false);
|
||||
|
||||
async move {
|
||||
let response = future.await?;
|
||||
|
||||
check_http_status_and_header(&response)?;
|
||||
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLResponse<T> {
|
||||
data: T,
|
||||
errors: Option<GhGraphQLErrors>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GraphQLQuery {
|
||||
query: String,
|
||||
}
|
||||
|
||||
fn get_graphql_endpoint() -> Url {
|
||||
let mut graphql_endpoint = get_api_endpoint().clone();
|
||||
|
||||
graphql_endpoint
|
||||
.path_segments_mut()
|
||||
.expect("get_api_endpoint() should return a https url")
|
||||
.push("graphql");
|
||||
|
||||
graphql_endpoint
|
||||
}
|
||||
|
||||
pub(super) fn issue_graphql_query<T>(
|
||||
client: &remote::Client,
|
||||
query: String,
|
||||
auth_token: &str,
|
||||
) -> impl Future<Output = Result<T, GhApiError>> + Send + Sync + 'static
|
||||
where
|
||||
T: DeserializeOwned + Debug,
|
||||
{
|
||||
let res = to_json_string(&GraphQLQuery { query })
|
||||
.map_err(remote::Error::from)
|
||||
.map(|graphql_query| {
|
||||
let graphql_endpoint = get_graphql_endpoint();
|
||||
|
||||
debug!("Sending graphql query to {graphql_endpoint}: '{graphql_query}'");
|
||||
|
||||
let request_builder = client
|
||||
.post(graphql_endpoint, graphql_query)
|
||||
.header("Accept", "application/vnd.github+json")
|
||||
.bearer_auth(&auth_token);
|
||||
|
||||
request_builder.send(false)
|
||||
});
|
||||
|
||||
async move {
|
||||
let response = res?.await?;
|
||||
check_http_status_and_header(&response)?;
|
||||
|
||||
let mut response: GraphQLResponse<T> = response.json().await?;
|
||||
|
||||
debug!("response = {response:?}");
|
||||
|
||||
if let Some(error) = response.errors.take() {
|
||||
Err(error.into())
|
||||
} else {
|
||||
Ok(response.data)
|
||||
}
|
||||
}
|
||||
}
|
||||
203
crates/binstalk-git-repo-api/src/gh_api_client/error.rs
Normal file
203
crates/binstalk-git-repo-api/src/gh_api_client/error.rs
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
use std::{error, fmt, io, time::Duration};
|
||||
|
||||
use binstalk_downloader::remote;
|
||||
use compact_str::{CompactString, ToCompactString};
|
||||
use serde::{de::Deserializer, Deserialize};
|
||||
use thiserror::Error as ThisError;
|
||||
|
||||
#[derive(ThisError, Debug)]
|
||||
#[error("Context: '{context}', err: '{err}'")]
|
||||
pub struct GhApiContextError {
|
||||
context: CompactString,
|
||||
#[source]
|
||||
err: GhApiError,
|
||||
}
|
||||
|
||||
#[derive(ThisError, Debug)]
|
||||
#[non_exhaustive]
|
||||
pub enum GhApiError {
|
||||
#[error("IO Error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
|
||||
#[error("Remote Error: {0}")]
|
||||
Remote(#[from] remote::Error),
|
||||
|
||||
#[error("Failed to parse url: {0}")]
|
||||
InvalidUrl(#[from] url::ParseError),
|
||||
|
||||
/// A wrapped error providing the context the error is about.
|
||||
#[error(transparent)]
|
||||
Context(Box<GhApiContextError>),
|
||||
|
||||
#[error("Remote failed to process GraphQL query: {0}")]
|
||||
GraphQLErrors(GhGraphQLErrors),
|
||||
|
||||
#[error("Hit rate-limit, retry after {retry_after:?}")]
|
||||
RateLimit { retry_after: Option<Duration> },
|
||||
|
||||
#[error("Corresponding resource is not found")]
|
||||
NotFound,
|
||||
|
||||
#[error("Does not have permission to access the API")]
|
||||
Unauthorized,
|
||||
}
|
||||
|
||||
impl GhApiError {
|
||||
/// Attach context to [`GhApiError`]
|
||||
pub fn context(self, context: impl fmt::Display) -> Self {
|
||||
use GhApiError::*;
|
||||
|
||||
if matches!(self, RateLimit { .. } | NotFound | Unauthorized) {
|
||||
self
|
||||
} else {
|
||||
Self::Context(Box::new(GhApiContextError {
|
||||
context: context.to_compact_string(),
|
||||
err: self,
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<GhGraphQLErrors> for GhApiError {
|
||||
fn from(e: GhGraphQLErrors) -> Self {
|
||||
if e.is_rate_limited() {
|
||||
Self::RateLimit { retry_after: None }
|
||||
} else if e.is_not_found_error() {
|
||||
Self::NotFound
|
||||
} else {
|
||||
Self::GraphQLErrors(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct GhGraphQLErrors(Box<[GraphQLError]>);
|
||||
|
||||
impl GhGraphQLErrors {
|
||||
fn is_rate_limited(&self) -> bool {
|
||||
self.0
|
||||
.iter()
|
||||
.any(|error| matches!(error.error_type, GraphQLErrorType::RateLimited))
|
||||
}
|
||||
|
||||
fn is_not_found_error(&self) -> bool {
|
||||
self.0
|
||||
.iter()
|
||||
.any(|error| matches!(&error.error_type, GraphQLErrorType::Other(error_type) if *error_type == "NOT_FOUND"))
|
||||
}
|
||||
}
|
||||
|
||||
impl error::Error for GhGraphQLErrors {}
|
||||
|
||||
impl fmt::Display for GhGraphQLErrors {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let last_error_index = self.0.len() - 1;
|
||||
|
||||
for (i, error) in self.0.iter().enumerate() {
|
||||
write!(
|
||||
f,
|
||||
"type: '{error_type}', msg: '{msg}'",
|
||||
error_type = error.error_type,
|
||||
msg = error.message,
|
||||
)?;
|
||||
|
||||
for location in error.locations.as_deref().into_iter().flatten() {
|
||||
write!(
|
||||
f,
|
||||
", occured on query line {line} col {col}",
|
||||
line = location.line,
|
||||
col = location.column
|
||||
)?;
|
||||
}
|
||||
|
||||
for (k, v) in &error.others {
|
||||
write!(f, ", {k}: {v}")?;
|
||||
}
|
||||
|
||||
if i < last_error_index {
|
||||
f.write_str("\n")?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLError {
|
||||
message: CompactString,
|
||||
locations: Option<Box<[GraphQLLocation]>>,
|
||||
|
||||
#[serde(rename = "type")]
|
||||
error_type: GraphQLErrorType,
|
||||
|
||||
#[serde(flatten, with = "tuple_vec_map")]
|
||||
others: Vec<(CompactString, serde_json::Value)>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) enum GraphQLErrorType {
|
||||
RateLimited,
|
||||
Other(CompactString),
|
||||
}
|
||||
|
||||
impl fmt::Display for GraphQLErrorType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
GraphQLErrorType::RateLimited => "RATE_LIMITED",
|
||||
GraphQLErrorType::Other(s) => s,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for GraphQLErrorType {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s = CompactString::deserialize(deserializer)?;
|
||||
Ok(match &*s {
|
||||
"RATE_LIMITED" => GraphQLErrorType::RateLimited,
|
||||
_ => GraphQLErrorType::Other(s),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLLocation {
|
||||
line: u64,
|
||||
column: u64,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use serde::de::value::{BorrowedStrDeserializer, Error};
|
||||
|
||||
macro_rules! assert_matches {
|
||||
($expression:expr, $pattern:pat $(if $guard:expr)? $(,)?) => {
|
||||
match $expression {
|
||||
$pattern $(if $guard)? => true,
|
||||
expr => {
|
||||
panic!(
|
||||
"assertion failed: `{expr:?}` does not match `{}`",
|
||||
stringify!($pattern $(if $guard)?)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_graph_ql_error_type() {
|
||||
let deserialize = |input: &str| {
|
||||
GraphQLErrorType::deserialize(BorrowedStrDeserializer::<'_, Error>::new(input)).unwrap()
|
||||
};
|
||||
|
||||
assert_matches!(deserialize("RATE_LIMITED"), GraphQLErrorType::RateLimited);
|
||||
assert_matches!(
|
||||
deserialize("rATE_LIMITED"),
|
||||
GraphQLErrorType::Other(val) if val == CompactString::new("rATE_LIMITED")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
use std::{
|
||||
borrow::Borrow,
|
||||
collections::HashSet,
|
||||
fmt,
|
||||
future::Future,
|
||||
hash::{Hash, Hasher},
|
||||
};
|
||||
|
||||
use binstalk_downloader::remote::{self};
|
||||
use compact_str::{CompactString, ToCompactString};
|
||||
use serde::Deserialize;
|
||||
use url::Url;
|
||||
|
||||
use super::{
|
||||
common::{issue_graphql_query, issue_restful_api},
|
||||
GhApiError, GhRelease, GhRepo,
|
||||
};
|
||||
|
||||
// Only include fields we do care about
|
||||
|
||||
#[derive(Eq, Deserialize, Debug)]
|
||||
struct Artifact {
|
||||
name: CompactString,
|
||||
url: Url,
|
||||
}
|
||||
|
||||
// Manually implement PartialEq and Hash to ensure it will always produce the
|
||||
// same hash as a str with the same content, and that the comparison will be
|
||||
// the same to coparing a string.
|
||||
|
||||
impl PartialEq for Artifact {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.name.eq(&other.name)
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for Artifact {
|
||||
fn hash<H>(&self, state: &mut H)
|
||||
where
|
||||
H: Hasher,
|
||||
{
|
||||
let s: &str = self.name.as_str();
|
||||
s.hash(state)
|
||||
}
|
||||
}
|
||||
|
||||
// Implement Borrow so that we can use call
|
||||
// `HashSet::contains::<str>`
|
||||
|
||||
impl Borrow<str> for Artifact {
|
||||
fn borrow(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
pub(super) struct Artifacts {
|
||||
assets: HashSet<Artifact>,
|
||||
}
|
||||
|
||||
impl Artifacts {
|
||||
/// get url for downloading the artifact using GitHub API (for private repository).
|
||||
pub(super) fn get_artifact_url(&self, artifact_name: &str) -> Option<Url> {
|
||||
self.assets
|
||||
.get(artifact_name)
|
||||
.map(|artifact| artifact.url.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fetch_release_artifacts_restful_api(
|
||||
client: &remote::Client,
|
||||
GhRelease {
|
||||
repo: GhRepo { owner, repo },
|
||||
tag,
|
||||
}: &GhRelease,
|
||||
) -> impl Future<Output = Result<Artifacts, GhApiError>> + Send + Sync + 'static {
|
||||
issue_restful_api(client, &["repos", owner, repo, "releases", "tags", tag])
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLData {
|
||||
repository: Option<GraphQLRepo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLRepo {
|
||||
release: Option<GraphQLRelease>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLRelease {
|
||||
#[serde(rename = "releaseAssets")]
|
||||
assets: GraphQLReleaseAssets,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLReleaseAssets {
|
||||
nodes: Vec<Artifact>,
|
||||
#[serde(rename = "pageInfo")]
|
||||
page_info: GraphQLPageInfo,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLPageInfo {
|
||||
#[serde(rename = "endCursor")]
|
||||
end_cursor: Option<CompactString>,
|
||||
#[serde(rename = "hasNextPage")]
|
||||
has_next_page: bool,
|
||||
}
|
||||
|
||||
enum FilterCondition {
|
||||
Init,
|
||||
After(CompactString),
|
||||
}
|
||||
|
||||
impl fmt::Display for FilterCondition {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
// GitHub imposes a limit of 100 for the value passed to param "first"
|
||||
FilterCondition::Init => f.write_str("first:100"),
|
||||
FilterCondition::After(end_cursor) => write!(f, r#"first:100,after:"{end_cursor}""#),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fetch_release_artifacts_graphql_api(
|
||||
client: &remote::Client,
|
||||
GhRelease {
|
||||
repo: GhRepo { owner, repo },
|
||||
tag,
|
||||
}: &GhRelease,
|
||||
auth_token: &str,
|
||||
) -> impl Future<Output = Result<Artifacts, GhApiError>> + Send + Sync + 'static {
|
||||
let client = client.clone();
|
||||
let auth_token = auth_token.to_compact_string();
|
||||
|
||||
let base_query_prefix = format!(
|
||||
r#"
|
||||
query {{
|
||||
repository(owner:"{owner}",name:"{repo}") {{
|
||||
release(tagName:"{tag}") {{"#
|
||||
);
|
||||
|
||||
let base_query_suffix = r#"
|
||||
nodes { name url }
|
||||
pageInfo { endCursor hasNextPage }
|
||||
}}}}"#
|
||||
.trim();
|
||||
|
||||
async move {
|
||||
let mut artifacts = Artifacts::default();
|
||||
let mut cond = FilterCondition::Init;
|
||||
let base_query_prefix = base_query_prefix.trim();
|
||||
|
||||
loop {
|
||||
let query = format!(
|
||||
r#"
|
||||
{base_query_prefix}
|
||||
releaseAssets({cond}) {{
|
||||
{base_query_suffix}"#
|
||||
);
|
||||
|
||||
let data: GraphQLData = issue_graphql_query(&client, query, &auth_token).await?;
|
||||
|
||||
let assets = data
|
||||
.repository
|
||||
.and_then(|repository| repository.release)
|
||||
.map(|release| release.assets);
|
||||
|
||||
if let Some(assets) = assets {
|
||||
artifacts.assets.extend(assets.nodes);
|
||||
|
||||
match assets.page_info {
|
||||
GraphQLPageInfo {
|
||||
end_cursor: Some(end_cursor),
|
||||
has_next_page: true,
|
||||
} => {
|
||||
cond = FilterCondition::After(end_cursor);
|
||||
}
|
||||
_ => break Ok(artifacts),
|
||||
}
|
||||
} else {
|
||||
break Err(GhApiError::NotFound);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
80
crates/binstalk-git-repo-api/src/gh_api_client/repo_info.rs
Normal file
80
crates/binstalk-git-repo-api/src/gh_api_client/repo_info.rs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
use std::future::Future;
|
||||
|
||||
use compact_str::CompactString;
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{
|
||||
common::{issue_graphql_query, issue_restful_api},
|
||||
remote, GhApiError, GhRepo,
|
||||
};
|
||||
|
||||
#[derive(Clone, Eq, PartialEq, Hash, Debug, Deserialize)]
|
||||
struct Owner {
|
||||
login: CompactString,
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, PartialEq, Hash, Debug, Deserialize)]
|
||||
pub struct RepoInfo {
|
||||
owner: Owner,
|
||||
name: CompactString,
|
||||
private: bool,
|
||||
}
|
||||
|
||||
impl RepoInfo {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn new(GhRepo { owner, repo }: GhRepo, private: bool) -> Self {
|
||||
Self {
|
||||
owner: Owner { login: owner },
|
||||
name: repo,
|
||||
private,
|
||||
}
|
||||
}
|
||||
pub fn repo(&self) -> GhRepo {
|
||||
GhRepo {
|
||||
owner: self.owner.login.clone(),
|
||||
repo: self.name.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_private(&self) -> bool {
|
||||
self.private
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn fetch_repo_info_restful_api(
|
||||
client: &remote::Client,
|
||||
GhRepo { owner, repo }: &GhRepo,
|
||||
) -> impl Future<Output = Result<Option<RepoInfo>, GhApiError>> + Send + Sync + 'static {
|
||||
issue_restful_api(client, &["repos", owner, repo])
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GraphQLData {
|
||||
repository: Option<RepoInfo>,
|
||||
}
|
||||
|
||||
pub(super) fn fetch_repo_info_graphql_api(
|
||||
client: &remote::Client,
|
||||
GhRepo { owner, repo }: &GhRepo,
|
||||
auth_token: &str,
|
||||
) -> impl Future<Output = Result<Option<RepoInfo>, GhApiError>> + Send + Sync + 'static {
|
||||
let query = format!(
|
||||
r#"
|
||||
query {{
|
||||
repository(owner:"{owner}",name:"{repo}") {{
|
||||
owner {{
|
||||
login
|
||||
}}
|
||||
name
|
||||
private: isPrivate
|
||||
}}
|
||||
}}"#
|
||||
);
|
||||
|
||||
let future = issue_graphql_query(client, query, auth_token);
|
||||
|
||||
async move {
|
||||
let data: GraphQLData = future.await?;
|
||||
Ok(data.repository)
|
||||
}
|
||||
}
|
||||
1
crates/binstalk-git-repo-api/src/lib.rs
Normal file
1
crates/binstalk-git-repo-api/src/lib.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod gh_api_client;
|
||||
Loading…
Add table
Add a link
Reference in a new issue