cargo-binstall/crates/binstalk-registry/src/common.rs
Félix Saparelli 32beba507b
Initial signing support (#1345)
* Add CLI options

* Add manifest types

* Thread signature policy through to fetchers

* Thread signing section through from metadata

* Implement signing validation

* Clippy

* Attempt testing

* Yes and

* Why

* fmt

* Update crates/bin/src/args.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Update crates/binstalk-fetchers/src/gh_crate_meta.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Update crates/bin/src/args.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Update crates/binstalk-fetchers/src/signing.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Update crates/binstalk-fetchers/src/signing.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Update crates/binstalk-fetchers/src/signing.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Update crates/binstalk-fetchers/src/signing.rs

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* fixes

* Finish feature

* Document

* Include all fields in the signing.file template

* Readme document

* Review fixes

* Fail on non-utf8 sig

* Thank goodness for tests

* Run test in ci

* Add rsign2 commands

* Log utf8 error

* Update e2e-tests/signing.sh

Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fix `e2e-tests/signing.sh` MacOS CI failure

Move the tls cert creation into `signing.sh` and sleep for 10s to wait
for https server to start.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Refactor e2e-tests-signing files

 - Use a tempdir generated by `mktemp` for all certificates-related
   files
 - Put other checked-in files into `e2e-tests/signing`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fixed `e2e-tests-signing` connection err in MacOS CI

Wait for server to start up by trying to connect to it.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fix `e2e-tests-signing` passing `-subj` to `openssl` on Windows

Use single quote instead of double quote to avoid automatic expansion
from bash

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fix `e2e-tests-signing` waiting for server to startup

Remove `timeout` since it is not supported on MacOS.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Try to fix windows CI by setting `MSYS_NO_PATHCONV=1` on `openssl` cmds

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fixed `e2e-tests-signing` on windows

By using double `//` for the value passed to option `-subj`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fixed infinite loop in `signing/wait-for-server` on Windows

Pass `--ssl-revoke-best-effort` to prevent schannel from checking ssl
revocation status.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Add cap on retry attempt in `signing/wait-for-server.sh`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Let `singing/server.py` print output to stderr

so that we can see the error message there.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

* Fix running `signing/server.py` on MacOS CI

use `python3` since macos-latest still has python2 installed and
`python` is a symlink to `python2` there.

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>

---------

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
Co-authored-by: Jiahao XU <Jiahao_XU@outlook.com>
2023-09-23 04:02:56 +00:00

211 lines
6.2 KiB
Rust

use std::borrow::Cow;
use base16::{decode as decode_base16, encode_lower as encode_base16};
use binstalk_downloader::{
bytes::Bytes,
download::{DataVerifier, Download},
remote::{Client, Url},
};
use binstalk_types::cargo_toml_binstall::{Meta, TarBasedFmt};
use cargo_toml_workspace::cargo_toml::Manifest;
use compact_str::{format_compact, CompactString, ToCompactString};
use leon::{Template, Values};
use semver::{Version, VersionReq};
use serde::Deserialize;
use serde_json::Error as JsonError;
use sha2::{Digest, Sha256};
use tracing::{debug, instrument};
use crate::{visitor::ManifestVisitor, RegistryError};
#[derive(Deserialize)]
pub(super) struct RegistryConfig {
pub(super) dl: CompactString,
}
struct Sha256Digest {
expected: Vec<u8>,
actual: Option<Vec<u8>>,
state: Option<Sha256>,
}
impl Sha256Digest {
fn new(checksum: Vec<u8>) -> Self {
Self {
expected: checksum,
actual: None,
state: Some(Sha256::new()),
}
}
}
impl DataVerifier for Sha256Digest {
fn update(&mut self, data: &Bytes) {
if let Some(ref mut state) = &mut self.state {
state.update(data);
}
}
fn validate(&mut self) -> bool {
if let Some(state) = self.state.take() {
self.actual = Some(state.finalize().to_vec());
}
self.actual.as_ref().unwrap() == &self.expected
}
}
#[instrument]
pub(super) async fn parse_manifest(
client: Client,
crate_name: &str,
crate_url: Url,
MatchedVersion { version, cksum }: MatchedVersion,
) -> Result<Manifest<Meta>, RegistryError> {
debug!("Fetching crate from: {crate_url} and extracting Cargo.toml from it");
let mut manifest_visitor = ManifestVisitor::new(format!("{crate_name}-{version}").into());
let checksum = decode_base16(cksum.as_bytes()).map_err(RegistryError::from)?;
let mut digest = Sha256Digest::new(checksum);
Download::new_with_data_verifier(client, crate_url, &mut digest)
.and_visit_tar(TarBasedFmt::Tgz, &mut manifest_visitor)
.await?;
if !digest.validate() {
Err(RegistryError::UnmatchedChecksum {
expected: encode_base16(digest.expected.as_slice()).into(),
actual: encode_base16(digest.actual.unwrap().as_slice()).into(),
})
} else {
manifest_visitor.load_manifest()
}
}
/// Return components of crate prefix
pub(super) fn crate_prefix_components(
crate_name: &str,
) -> Result<(CompactString, Option<CompactString>), RegistryError> {
let mut chars = crate_name.chars();
match (chars.next(), chars.next(), chars.next(), chars.next()) {
(None, None, None, None) => Err(RegistryError::NotFound(crate_name.into())),
(Some(_), None, None, None) => Ok((CompactString::new("1"), None)),
(Some(_), Some(_), None, None) => Ok((CompactString::new("2"), None)),
(Some(ch), Some(_), Some(_), None) => Ok((
CompactString::new("3"),
Some(ch.to_lowercase().to_compact_string()),
)),
(Some(a), Some(b), Some(c), Some(d)) => Ok((
format_compact!("{}{}", a.to_lowercase(), b.to_lowercase()),
Some(format_compact!("{}{}", c.to_lowercase(), d.to_lowercase())),
)),
_ => unreachable!(),
}
}
pub(super) fn render_dl_template(
dl_template: &str,
crate_name: &str,
(c1, c2): &(CompactString, Option<CompactString>),
MatchedVersion { version, cksum }: &MatchedVersion,
) -> Result<String, RegistryError> {
let template = Template::parse(dl_template)?;
if template.keys().next().is_some() {
let mut crate_prefix = c1.clone();
if let Some(c2) = c2 {
crate_prefix.push('/');
crate_prefix.push_str(c2);
}
struct Context<'a> {
crate_name: &'a str,
crate_prefix: CompactString,
crate_lowerprefix: String,
version: &'a str,
cksum: &'a str,
}
impl Values for Context<'_> {
fn get_value(&self, key: &str) -> Option<Cow<'_, str>> {
match key {
"crate" => Some(Cow::Borrowed(self.crate_name)),
"version" => Some(Cow::Borrowed(self.version)),
"prefix" => Some(Cow::Borrowed(&self.crate_prefix)),
"lowerprefix" => Some(Cow::Borrowed(&self.crate_lowerprefix)),
"sha256-checksum" => Some(Cow::Borrowed(self.cksum)),
_ => None,
}
}
}
Ok(template.render(&Context {
crate_name,
crate_lowerprefix: crate_prefix.to_lowercase(),
crate_prefix,
version,
cksum,
})?)
} else {
Ok(format!("{dl_template}/{crate_name}/{version}/download"))
}
}
#[derive(Deserialize)]
pub(super) struct RegistryIndexEntry {
vers: CompactString,
yanked: bool,
cksum: String,
}
pub(super) struct MatchedVersion {
pub(super) version: CompactString,
/// sha256 checksum encoded in base16
pub(super) cksum: String,
}
impl MatchedVersion {
pub(super) fn find(
it: &mut dyn Iterator<Item = Result<RegistryIndexEntry, JsonError>>,
version_req: &VersionReq,
) -> Result<Self, RegistryError> {
let mut ret = Option::<(Self, Version)>::None;
for res in it {
let entry = res.map_err(RegistryError::from)?;
if entry.yanked {
continue;
}
let num = entry.vers;
// Parse out version
let Ok(ver) = Version::parse(&num) else {
continue;
};
// Filter by version match
if !version_req.matches(&ver) {
continue;
}
let matched = Self {
version: num,
cksum: entry.cksum,
};
if let Some((_, max_ver)) = &ret {
if ver > *max_ver {
ret = Some((matched, ver));
}
} else {
ret = Some((matched, ver));
}
}
ret.map(|(num, _)| num)
.ok_or_else(|| RegistryError::VersionMismatch {
req: version_req.clone(),
})
}
}