Refactor: Extract new crate binstalk-types plus other misc refactor and optimization (#535)

* Refactor: Extract new crate binstalk-types
* Optimize: Rm field `CrateInfo::other`
   which also removes dep serde-tuple-vec-map and serde-json from
   binstalk-types.
   
   This also makes `CrateInfo` easier to use, more generic and can be used
   over any `Serializer`, not just `serde_json::Value`.
* Mark all errors in `binstalk-manifests` as non_exhaustive
* Reduce size of `CvsParseError` by using `Box<str>`
   instead of `String` for variant `UnknownSourceType`.
* Reduce size of `CratesTomlParseError` to 16 bytes on 64bit platform
   by boxing variants `TomlWrite` and `CvsParse` as these two fields are
   significantly larger than other variants.
* Unify import style in mod `binstall_crates_v1`
* Replace dep binstalk-manifests with binstalk-types in binstalk-downloader
   to reduce its transitive dependencies and enables binstalk-downloader to
   be built in parallel to binstak-manifests.
* Replace dep binstalk-manifests with binstalk-types in binstalk
   to reduce transitive dependencies and enables binstalk to be built in
   parallel to binstalk-manifests.
   
   This is benefitial because binstalk-manifests pulls in toml_edit, which
   could takes up to 15s to be built on M1 (7-9s for codegen).
* Add dep binstalk-manifests to crates/bin
* Update dependabot and GHA release-pr

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
This commit is contained in:
Jiahao XU 2022-11-17 11:46:27 +11:00 committed by GitHub
parent 58326a6085
commit d9cc3ce219
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 372 additions and 76 deletions

View file

@ -10,17 +10,15 @@ edition = "2021"
license = "Apache-2.0 OR MIT"
[dependencies]
binstalk-types = { version = "0.1.0", path = "../binstalk-types" }
compact_str = { version = "0.6.0", features = ["serde"] }
fs-lock = { version = "0.1.0", path = "../fs-lock" }
home = "0.5.4"
miette = "5.4.1"
once_cell = "1.16.0"
semver = { version = "1.0.14", features = ["serde"] }
serde = { version = "1.0.147", features = ["derive"] }
serde-tuple-vec-map = "1.0.1"
serde_json = "1.0.87"
strum = "0.24.1"
strum_macros = "0.24.3"
thiserror = "1.0.37"
toml_edit = { version = "0.15.0", features = ["easy"] }
url = { version = "2.3.1", features = ["serde"] }

View file

@ -7,6 +7,8 @@
//! NLJSON to the file will be understood fine.
use std::{
borrow::Borrow,
cmp,
collections::{btree_set, BTreeSet},
fs,
io::{self, Seek, Write},
@ -14,17 +16,17 @@ use std::{
path::{Path, PathBuf},
};
use compact_str::CompactString;
use fs_lock::FileLock;
use home::cargo_home;
use miette::Diagnostic;
use serde::Serialize;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::helpers::create_if_not_exist;
use super::crate_info::CrateInfo;
use crate::{crate_info::CrateInfo, helpers::create_if_not_exist};
#[derive(Debug, Diagnostic, Error)]
#[non_exhaustive]
pub enum Error {
#[error(transparent)]
Io(#[from] io::Error),
@ -33,28 +35,27 @@ pub enum Error {
SerdeJsonParse(#[from] serde_json::Error),
}
pub fn append_to_path<Iter>(path: impl AsRef<Path>, iter: Iter) -> Result<(), Error>
pub fn append_to_path<Iter, T>(path: impl AsRef<Path>, iter: Iter) -> Result<(), Error>
where
Iter: IntoIterator<Item = CrateInfo>,
Iter: IntoIterator<Item = T>,
Data: From<T>,
{
let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?;
// Move the cursor to EOF
file.seek(io::SeekFrom::End(0))?;
write_to(&mut file, &mut iter.into_iter())
write_to(&mut file, &mut iter.into_iter().map(Data::from))
}
pub fn append<Iter>(iter: Iter) -> Result<(), Error>
pub fn append<Iter, T>(iter: Iter) -> Result<(), Error>
where
Iter: IntoIterator<Item = CrateInfo>,
Iter: IntoIterator<Item = T>,
Data: From<T>,
{
append_to_path(default_path()?, iter)
}
pub fn write_to(
file: &mut FileLock,
iter: &mut dyn Iterator<Item = CrateInfo>,
) -> Result<(), Error> {
pub fn write_to(file: &mut FileLock, iter: &mut dyn Iterator<Item = Data>) -> Result<(), Error> {
let writer = io::BufWriter::with_capacity(512, file);
let mut ser = serde_json::Serializer::new(writer);
@ -76,11 +77,74 @@ pub fn default_path() -> Result<PathBuf, Error> {
Ok(dir.join("crates-v1.json"))
}
#[derive(Debug, Deserialize, Serialize)]
pub struct Data {
#[serde(flatten)]
pub crate_info: CrateInfo,
/// Forwards compatibility. Unknown keys from future versions
/// will be stored here and retained when the file is saved.
///
/// We use an `Vec` here since it is never accessed in Rust.
#[serde(flatten, with = "tuple_vec_map")]
pub other: Vec<(CompactString, serde_json::Value)>,
}
impl From<CrateInfo> for Data {
fn from(crate_info: CrateInfo) -> Self {
Self {
crate_info,
other: Vec::new(),
}
}
}
impl From<Data> for CrateInfo {
fn from(data: Data) -> Self {
data.crate_info
}
}
impl Borrow<str> for Data {
fn borrow(&self) -> &str {
&self.crate_info.name
}
}
impl PartialEq for Data {
fn eq(&self, other: &Self) -> bool {
self.crate_info.name == other.crate_info.name
}
}
impl PartialEq<CrateInfo> for Data {
fn eq(&self, other: &CrateInfo) -> bool {
self.crate_info.name == other.name
}
}
impl PartialEq<Data> for CrateInfo {
fn eq(&self, other: &Data) -> bool {
self.name == other.crate_info.name
}
}
impl Eq for Data {}
impl PartialOrd for Data {
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for Data {
fn cmp(&self, other: &Self) -> cmp::Ordering {
self.crate_info.name.cmp(&other.crate_info.name)
}
}
#[derive(Debug)]
pub struct Records {
file: FileLock,
/// Use BTreeSet to dedup the metadata
data: BTreeSet<CrateInfo>,
data: BTreeSet<Data>,
}
impl Records {
@ -122,7 +186,7 @@ impl Records {
}
pub fn get(&self, value: impl AsRef<str>) -> Option<&CrateInfo> {
self.data.get(value.as_ref())
self.data.get(value.as_ref()).map(|data| &data.crate_info)
}
pub fn contains(&self, value: impl AsRef<str>) -> bool {
@ -134,11 +198,12 @@ impl Records {
/// If the set did have an equal element present, false is returned,
/// and the entry is not updated.
pub fn insert(&mut self, value: CrateInfo) -> bool {
self.data.insert(value)
self.data.insert(Data::from(value))
}
/// Return the previous `CrateInfo` for the package if there is any.
pub fn replace(&mut self, value: CrateInfo) -> Option<CrateInfo> {
self.data.replace(value)
self.data.replace(Data::from(value)).map(CrateInfo::from)
}
pub fn remove(&mut self, value: impl AsRef<str>) -> bool {
@ -146,7 +211,7 @@ impl Records {
}
pub fn take(&mut self, value: impl AsRef<str>) -> Option<CrateInfo> {
self.data.take(value.as_ref())
self.data.take(value.as_ref()).map(CrateInfo::from)
}
pub fn len(&self) -> usize {
@ -159,9 +224,9 @@ impl Records {
}
impl<'a> IntoIterator for &'a Records {
type Item = &'a CrateInfo;
type Item = &'a Data;
type IntoIter = btree_set::Iter<'a, CrateInfo>;
type IntoIter = btree_set::Iter<'a, Data>;
fn into_iter(self) -> Self::IntoIter {
self.data.iter()
@ -202,7 +267,6 @@ mod test {
source: CrateSource::cratesio_registry(),
target: target.clone(),
bins: vec!["1".into(), "2".into()],
other: Default::default(),
},
CrateInfo {
name: "b".into(),
@ -211,7 +275,6 @@ mod test {
source: CrateSource::cratesio_registry(),
target: target.clone(),
bins: vec!["1".into(), "2".into()],
other: Default::default(),
},
CrateInfo {
name: "a".into(),
@ -220,7 +283,6 @@ mod test {
source: CrateSource::cratesio_registry(),
target: target.clone(),
bins: vec!["1".into()],
other: Default::default(),
},
];
@ -234,11 +296,11 @@ mod test {
let mut records = Records::load_from_path(path).unwrap();
assert_records_eq!(&records, &metadata_set);
records.remove("b");
assert_eq!(records.len(), metadata_set.len() - 1);
assert!(records.remove("b"));
metadata_set.remove("b");
assert_eq!(records.len(), metadata_set.len());
records.overwrite().unwrap();
metadata_set.remove("b");
let records = Records::load_from_path(path).unwrap();
assert_records_eq!(&records, &metadata_set);
// Drop the exclusive file lock
@ -251,7 +313,6 @@ mod test {
source: CrateSource::cratesio_registry(),
target,
bins: vec!["1".into(), "2".into()],
other: Default::default(),
};
append_to_path(path, [new_metadata.clone()]).unwrap();
metadata_set.insert(new_metadata);

View file

@ -114,6 +114,7 @@ impl CratesToml {
}
#[derive(Debug, Diagnostic, Error)]
#[non_exhaustive]
pub enum CratesTomlParseError {
#[error(transparent)]
Io(#[from] io::Error),
@ -122,10 +123,22 @@ pub enum CratesTomlParseError {
TomlParse(#[from] toml_edit::easy::de::Error),
#[error(transparent)]
TomlWrite(#[from] toml_edit::easy::ser::Error),
TomlWrite(Box<toml_edit::easy::ser::Error>),
#[error(transparent)]
CvsParse(#[from] CvsParseError),
CvsParse(Box<CvsParseError>),
}
impl From<CvsParseError> for CratesTomlParseError {
fn from(e: CvsParseError) -> Self {
CratesTomlParseError::CvsParse(Box::new(e))
}
}
impl From<toml_edit::easy::ser::Error> for CratesTomlParseError {
fn from(e: toml_edit::easy::ser::Error) -> Self {
CratesTomlParseError::TomlWrite(Box::new(e))
}
}
#[cfg(test)]
@ -151,7 +164,6 @@ mod tests {
source: CrateSource::cratesio_registry(),
target: TARGET.into(),
bins: vec!["cargo-binstall".into()],
other: Default::default(),
}],
)
.unwrap();

View file

@ -1,5 +1,6 @@
use std::{borrow::Cow, fmt, str::FromStr};
use binstalk_types::crate_info::cratesio_url;
use compact_str::CompactString;
use miette::Diagnostic;
use semver::Version;
@ -7,10 +8,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
use thiserror::Error;
use url::Url;
use crate::{
crate_info::{CrateInfo, CrateSource, SourceType},
helpers::cratesio_url,
};
use crate::crate_info::{CrateInfo, CrateSource, SourceType};
#[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)]
pub struct CrateVersionSource {
@ -72,8 +70,8 @@ impl FromStr for CrateVersionSource {
["registry", url] => Source::Registry(Url::parse(url)?),
[kind, arg] => {
return Err(CvsParseError::UnknownSourceType {
kind: kind.to_string(),
arg: arg.to_string(),
kind: kind.to_string().into_boxed_str(),
arg: arg.to_string().into_boxed_str(),
})
}
_ => return Err(CvsParseError::BadSource),
@ -90,6 +88,7 @@ impl FromStr for CrateVersionSource {
}
#[derive(Debug, Diagnostic, Error)]
#[non_exhaustive]
pub enum CvsParseError {
#[error(transparent)]
UrlParse(#[from] url::ParseError),
@ -98,7 +97,7 @@ pub enum CvsParseError {
VersionParse(#[from] semver::Error),
#[error("unknown source type {kind}+{arg}")]
UnknownSourceType { kind: String, arg: String },
UnknownSourceType { kind: Box<str>, arg: Box<str> },
#[error("bad source format")]
BadSource,

View file

@ -1,113 +0,0 @@
//! The format of the `[package.metadata.binstall]` manifest.
//!
//! This manifest defines how a particular binary crate may be installed by Binstall.
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
#[doc(inline)]
pub use package_formats::*;
mod package_formats;
/// `binstall` metadata container
///
/// Required to nest metadata under `package.metadata.binstall`
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct Meta {
pub binstall: Option<PkgMeta>,
}
/// Metadata for binary installation use.
///
/// Exposed via `[package.metadata]` in `Cargo.toml`
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", default)]
pub struct PkgMeta {
/// URL template for package downloads
pub pkg_url: Option<String>,
/// Format for package downloads
pub pkg_fmt: Option<PkgFmt>,
/// Path template for binary files in packages
pub bin_dir: Option<String>,
/// Public key for package verification (base64 encoded)
pub pub_key: Option<String>,
/// Target specific overrides
pub overrides: BTreeMap<String, PkgOverride>,
}
impl PkgMeta {
/// Merge configuration overrides into object
pub fn merge(&mut self, pkg_override: &PkgOverride) {
if let Some(o) = &pkg_override.pkg_url {
self.pkg_url = Some(o.clone());
}
if let Some(o) = &pkg_override.pkg_fmt {
self.pkg_fmt = Some(*o);
}
if let Some(o) = &pkg_override.bin_dir {
self.bin_dir = Some(o.clone());
}
}
/// Merge configuration overrides into object
///
/// * `pkg_overrides` - ordered in preference
pub fn merge_overrides<'a, It>(&self, pkg_overrides: It) -> Self
where
It: IntoIterator<Item = &'a PkgOverride> + Clone,
{
Self {
pkg_url: pkg_overrides
.clone()
.into_iter()
.find_map(|pkg_override| pkg_override.pkg_url.clone())
.or_else(|| self.pkg_url.clone()),
pkg_fmt: pkg_overrides
.clone()
.into_iter()
.find_map(|pkg_override| pkg_override.pkg_fmt)
.or(self.pkg_fmt),
bin_dir: pkg_overrides
.into_iter()
.find_map(|pkg_override| pkg_override.bin_dir.clone())
.or_else(|| self.bin_dir.clone()),
pub_key: self.pub_key.clone(),
overrides: Default::default(),
}
}
}
/// Target specific overrides for binary installation
///
/// Exposed via `[package.metadata.TARGET]` in `Cargo.toml`
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", default)]
pub struct PkgOverride {
/// URL template override for package downloads
pub pkg_url: Option<String>,
/// Format override for package downloads
pub pkg_fmt: Option<PkgFmt>,
/// Path template override for binary files in packages
pub bin_dir: Option<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct BinMeta {
/// Binary name
pub name: String,
/// Binary template path (within package)
pub path: String,
}

View file

@ -1,93 +0,0 @@
use serde::{Deserialize, Serialize};
use strum_macros::{Display, EnumIter, EnumString};
/// Binary format enumeration
#[derive(
Debug, Display, Copy, Clone, Eq, PartialEq, Serialize, Deserialize, EnumString, EnumIter,
)]
#[serde(rename_all = "snake_case")]
#[strum(ascii_case_insensitive)]
pub enum PkgFmt {
/// Download format is TAR (uncompressed)
Tar,
/// Download format is TAR + Bzip2
Tbz2,
/// Download format is TGZ (TAR + GZip)
Tgz,
/// Download format is TAR + XZ
Txz,
/// Download format is TAR + Zstd
Tzstd,
/// Download format is Zip
Zip,
/// Download format is raw / binary
Bin,
}
impl Default for PkgFmt {
fn default() -> Self {
Self::Tgz
}
}
impl PkgFmt {
/// If self is one of the tar based formats, return Some.
pub fn decompose(self) -> PkgFmtDecomposed {
match self {
PkgFmt::Tar => PkgFmtDecomposed::Tar(TarBasedFmt::Tar),
PkgFmt::Tbz2 => PkgFmtDecomposed::Tar(TarBasedFmt::Tbz2),
PkgFmt::Tgz => PkgFmtDecomposed::Tar(TarBasedFmt::Tgz),
PkgFmt::Txz => PkgFmtDecomposed::Tar(TarBasedFmt::Txz),
PkgFmt::Tzstd => PkgFmtDecomposed::Tar(TarBasedFmt::Tzstd),
PkgFmt::Bin => PkgFmtDecomposed::Bin,
PkgFmt::Zip => PkgFmtDecomposed::Zip,
}
}
/// List of possible file extensions for the format
/// (with prefix `.`).
pub fn extensions(self) -> &'static [&'static str] {
match self {
PkgFmt::Tar => &[".tar"],
PkgFmt::Tbz2 => &[".tbz2", ".tar.bz2"],
PkgFmt::Tgz => &[".tgz", ".tar.gz"],
PkgFmt::Txz => &[".txz", ".tar.xz"],
PkgFmt::Tzstd => &[".tzstd", ".tzst", ".tar.zst"],
PkgFmt::Bin => &[".bin", ".exe", ""],
PkgFmt::Zip => &[".zip"],
}
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum PkgFmtDecomposed {
Tar(TarBasedFmt),
Bin,
Zip,
}
#[derive(Debug, Display, Copy, Clone, Eq, PartialEq)]
pub enum TarBasedFmt {
/// Download format is TAR (uncompressed)
Tar,
/// Download format is TAR + Bzip2
Tbz2,
/// Download format is TGZ (TAR + GZip)
Tgz,
/// Download format is TAR + XZ
Txz,
/// Download format is TAR + Zstd
Tzstd,
}
impl From<TarBasedFmt> for PkgFmt {
fn from(fmt: TarBasedFmt) -> Self {
match fmt {
TarBasedFmt::Tar => PkgFmt::Tar,
TarBasedFmt::Tbz2 => PkgFmt::Tbz2,
TarBasedFmt::Tgz => PkgFmt::Tgz,
TarBasedFmt::Txz => PkgFmt::Txz,
TarBasedFmt::Tzstd => PkgFmt::Tzstd,
}
}
}

View file

@ -1,83 +0,0 @@
//! Common structure for crate information for post-install manifests.
use std::{borrow, cmp, hash};
use compact_str::CompactString;
use semver::Version;
use serde::{Deserialize, Serialize};
use url::Url;
use crate::helpers::cratesio_url;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CrateInfo {
pub name: CompactString,
pub version_req: CompactString,
pub current_version: Version,
pub source: CrateSource,
pub target: CompactString,
pub bins: Vec<CompactString>,
/// Forwards compatibility. Unknown keys from future versions
/// will be stored here and retained when the file is saved.
///
/// We use an `Vec` here since it is never accessed in Rust.
#[serde(flatten, with = "tuple_vec_map")]
pub other: Vec<(CompactString, serde_json::Value)>,
}
impl borrow::Borrow<str> for CrateInfo {
fn borrow(&self) -> &str {
&self.name
}
}
impl PartialEq for CrateInfo {
fn eq(&self, other: &Self) -> bool {
self.name == other.name
}
}
impl Eq for CrateInfo {}
impl PartialOrd for CrateInfo {
fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
self.name.partial_cmp(&other.name)
}
}
impl Ord for CrateInfo {
fn cmp(&self, other: &Self) -> cmp::Ordering {
self.name.cmp(&other.name)
}
}
impl hash::Hash for CrateInfo {
fn hash<H>(&self, state: &mut H)
where
H: hash::Hasher,
{
self.name.hash(state)
}
}
#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
pub enum SourceType {
Git,
Path,
Registry,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CrateSource {
pub source_type: SourceType,
pub url: Url,
}
impl CrateSource {
pub fn cratesio_registry() -> CrateSource {
Self {
source_type: SourceType::Registry,
url: cratesio_url().clone(),
}
}
}

View file

@ -1,8 +1,5 @@
use std::{fs, io, path::Path};
use once_cell::sync::Lazy;
use url::Url;
/// Returned file is readable and writable.
pub(crate) fn create_if_not_exist(path: impl AsRef<Path>) -> io::Result<fs::File> {
let path = path.as_ref();
@ -16,10 +13,3 @@ pub(crate) fn create_if_not_exist(path: impl AsRef<Path>) -> io::Result<fs::File
.open(path)
.or_else(|_| options.open(path))
}
pub(crate) fn cratesio_url() -> &'static Url {
static CRATESIO: Lazy<Url, fn() -> Url> =
Lazy::new(|| Url::parse("https://github.com/rust-lang/crates.io-index").unwrap());
&CRATESIO
}

View file

@ -12,5 +12,5 @@ mod helpers;
pub mod binstall_crates_v1;
pub mod cargo_crates_v1;
pub mod cargo_toml_binstall;
pub mod crate_info;
pub use binstalk_types::{cargo_toml_binstall, crate_info};