diff --git a/Cargo.lock b/Cargo.lock index ba311b77..a90b580e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,8 @@ dependencies = [ "scopeguard", "semver", "serde", + "serde-tuple-vec-map", + "serde_json", "simplelog", "strum", "strum_macros", @@ -1375,6 +1377,9 @@ name = "semver" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -1385,6 +1390,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-tuple-vec-map" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a04d0ebe0de77d7d445bb729a895dcb0a288854b267ca85f030ce51cdc578c82" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.140" @@ -1811,6 +1825,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 410d4a13..eb99d341 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,8 +38,10 @@ mimalloc = { version = "0.1.29", default-features = false, optional = true } once_cell = "1.13.0" reqwest = { version = "0.11.11", features = ["stream"], default-features = false } scopeguard = "1.1.0" -semver = "1.0.12" +semver = { version = "1.0.12", features = ["serde"] } serde = { version = "1.0.140", features = ["derive"] } +serde-tuple-vec-map = "1.0.1" +serde_json = "1.0.82" simplelog = "0.12.0" strum = "0.24.1" strum_macros = "0.24.2" @@ -49,7 +51,7 @@ thiserror = "1.0.31" tinytemplate = "1.2.1" tokio = { version = "1.20.0", features = ["rt-multi-thread", "process", "sync"], default-features = false } toml_edit = { version = "0.14.4", features = ["easy"] } -url = "2.2.2" +url = { version = "2.2.2", features = ["serde"] } xz2 = "0.1.7" # Disable all features of zip except for features of compression algorithms: diff --git a/src/binstall.rs b/src/binstall.rs index f28c0e5d..c6fcc569 100644 --- a/src/binstall.rs +++ b/src/binstall.rs @@ -1,8 +1,6 @@ use std::path::PathBuf; -use compact_str::CompactString; - -use crate::{metafiles, DesiredTargets, PkgOverride}; +use crate::{metafiles::binstall_v1::MetaData, DesiredTargets, PkgOverride}; mod resolve; pub use resolve::*; @@ -18,11 +16,3 @@ pub struct Options { pub cli_overrides: PkgOverride, pub desired_targets: DesiredTargets, } - -/// MetaData required to update MetaFiles. -pub struct MetaData { - pub bins: Vec, - pub cvs: metafiles::CrateVersionSource, - pub version_req: String, - pub target: String, -} diff --git a/src/binstall/install.rs b/src/binstall/install.rs index e7f12a36..365dd0de 100644 --- a/src/binstall/install.rs +++ b/src/binstall/install.rs @@ -1,12 +1,13 @@ use std::{path::PathBuf, process, sync::Arc}; use cargo_toml::Package; +use compact_str::CompactString; use log::{debug, error, info}; use miette::{miette, IntoDiagnostic, Result, WrapErr}; use tokio::{process::Command, task::block_in_place}; use super::{MetaData, Options, Resolution}; -use crate::{bins, fetchers::Fetcher, *}; +use crate::{bins, fetchers::Fetcher, metafiles::binstall_v1::Source, *}; pub async fn install( resolution: Resolution, @@ -22,13 +23,22 @@ pub async fn install( bin_path, bin_files, } => { - let cvs = metafiles::CrateVersionSource { - name, - version: package.version.parse().into_diagnostic()?, - source: metafiles::Source::cratesio_registry(), - }; + let current_version = package.version.parse().into_diagnostic()?; + let target = fetcher.target().into(); - install_from_package(fetcher, opts, cvs, version, bin_path, bin_files).await + install_from_package(fetcher, opts, bin_path, bin_files) + .await + .map(|option| { + option.map(|bins| MetaData { + name: name.into(), + version_req: version.into(), + current_version, + source: Source::cratesio_registry(), + target, + bins, + other: Default::default(), + }) + }) } Resolution::InstallFromSource { package } => { let desired_targets = opts.desired_targets.get().await; @@ -54,11 +64,9 @@ pub async fn install( async fn install_from_package( fetcher: Arc, opts: Arc, - cvs: metafiles::CrateVersionSource, - version: String, bin_path: PathBuf, bin_files: Vec, -) -> Result> { +) -> Result>> { // Download package if opts.dry_run { info!("Dry run, not downloading package"); @@ -108,12 +116,9 @@ async fn install_from_package( } } - Ok(Some(MetaData { - bins: bin_files.into_iter().map(|bin| bin.base_name).collect(), - cvs, - version_req: version, - target: fetcher.target().to_string(), - })) + Ok(Some( + bin_files.into_iter().map(|bin| bin.base_name).collect(), + )) }) } diff --git a/src/helpers.rs b/src/helpers.rs index 269557fc..7a7e565d 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -10,7 +10,7 @@ use bytes::Bytes; use cargo_toml::Manifest; use futures_util::stream::Stream; use log::debug; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use reqwest::{tls, Client, ClientBuilder, Method, Response}; use serde::Serialize; use tempfile::NamedTempFile; @@ -55,6 +55,13 @@ pub fn cargo_home() -> Result<&'static Path, io::Error> { .map(ops::Deref::deref) } +pub fn cratesio_url() -> &'static Url { + static CRATESIO: Lazy Url> = + Lazy::new(|| url::Url::parse("https://github.com/rust-lang/crates.io-index").unwrap()); + + &*CRATESIO +} + /// Returned file is readable and writable. pub fn create_if_not_exist(path: impl AsRef) -> io::Result { let path = path.as_ref(); diff --git a/src/helpers/flock.rs b/src/helpers/flock.rs index 2c0707b5..f8b4514a 100644 --- a/src/helpers/flock.rs +++ b/src/helpers/flock.rs @@ -43,3 +43,39 @@ impl ops::DerefMut for FileLock { &mut self.0 } } + +impl io::Write for FileLock { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + self.0.write_vectored(bufs) + } +} + +impl io::Read for FileLock { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0.read(buf) + } + + fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result { + self.0.read_vectored(bufs) + } +} + +impl io::Seek for FileLock { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + self.0.seek(pos) + } + + fn rewind(&mut self) -> io::Result<()> { + self.0.rewind() + } + fn stream_position(&mut self) -> io::Result { + self.0.stream_position() + } +} diff --git a/src/main.rs b/src/main.rs index ab652f26..307e66ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -344,11 +344,10 @@ async fn entry(jobserver_client: LazyJobserverClient) -> Result<()> { block_in_place(|| { if !custom_install_path { debug!("Writing .crates.toml"); - metafiles::v1::CratesToml::append( - metadata_vec - .iter() - .map(|metadata| (&metadata.cvs, metadata.bins.clone())), - )?; + metafiles::v1::CratesToml::append(metadata_vec.iter())?; + + debug!("Writing binstall/crates-v1.json"); + metafiles::binstall_v1::append(metadata_vec)?; } if opts.no_cleanup { diff --git a/src/metafiles.rs b/src/metafiles.rs index 75ad0250..b8a2b4ee 100644 --- a/src/metafiles.rs +++ b/src/metafiles.rs @@ -2,3 +2,5 @@ mod cvs; pub use cvs::*; pub mod v1; + +pub mod binstall_v1; diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs new file mode 100644 index 00000000..2d429ec2 --- /dev/null +++ b/src/metafiles/binstall_v1.rs @@ -0,0 +1,324 @@ +use std::{ + borrow, cmp, + collections::{btree_set, BTreeSet}, + fs, hash, + io::{self, Seek, Write}, + iter::{IntoIterator, Iterator}, + path::{Path, PathBuf}, +}; + +use compact_str::CompactString; +use miette::Diagnostic; +use semver::Version; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use url::Url; + +use crate::{cargo_home, cratesio_url, create_if_not_exist, FileLock}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MetaData { + pub name: CompactString, + pub version_req: CompactString, + pub current_version: Version, + pub source: Source, + pub target: CompactString, + pub bins: Vec, + + /// Forwards compatibility. Unknown keys from future versions + /// will be stored here and retained when the file is saved. + /// + /// We use an `Vec` here since it is never accessed in Rust. + #[serde(flatten, with = "tuple_vec_map")] + pub other: Vec<(CompactString, serde_json::Value)>, +} + +impl borrow::Borrow for MetaData { + fn borrow(&self) -> &str { + &self.name + } +} + +impl PartialEq for MetaData { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} +impl Eq for MetaData {} + +impl PartialOrd for MetaData { + fn partial_cmp(&self, other: &Self) -> Option { + self.name.partial_cmp(&other.name) + } +} + +impl Ord for MetaData { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.name.cmp(&other.name) + } +} + +impl hash::Hash for MetaData { + fn hash(&self, state: &mut H) + where + H: hash::Hasher, + { + self.name.hash(state) + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +pub enum SourceType { + Git, + Path, + Registry, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Source { + pub source_type: SourceType, + pub url: Url, +} + +impl Source { + pub fn cratesio_registry() -> Source { + Self { + source_type: SourceType::Registry, + url: cratesio_url().clone(), + } + } +} + +#[derive(Debug, Diagnostic, Error)] +pub enum Error { + #[error(transparent)] + Io(#[from] io::Error), + + #[error(transparent)] + SerdeJsonParse(#[from] serde_json::Error), +} + +pub fn append_to_path(path: impl AsRef, iter: Iter) -> Result<(), Error> +where + Iter: IntoIterator, +{ + let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?; + // Move the cursor to EOF + file.seek(io::SeekFrom::End(0))?; + + write_to(&mut file, &mut iter.into_iter()) +} + +pub fn append(iter: Iter) -> Result<(), Error> +where + Iter: IntoIterator, +{ + append_to_path(default_path()?, iter) +} + +pub fn write_to( + file: &mut FileLock, + iter: &mut dyn Iterator, +) -> Result<(), Error> { + let writer = io::BufWriter::with_capacity(512, file); + + let mut ser = serde_json::Serializer::new(writer); + + for item in iter { + item.serialize(&mut ser)?; + } + + ser.into_inner().flush()?; + + Ok(()) +} + +pub fn default_path() -> Result { + let dir = cargo_home()?.join("binstall"); + + fs::create_dir_all(&dir)?; + + Ok(dir.join("crates-v1.json")) +} + +#[derive(Debug)] +pub struct Records { + file: FileLock, + /// Use BTreeSet to dedup the metadata + data: BTreeSet, +} + +impl Records { + fn load_impl(&mut self) -> Result<(), Error> { + let reader = io::BufReader::with_capacity(1024, &mut self.file); + let stream_deser = serde_json::Deserializer::from_reader(reader).into_iter(); + + for res in stream_deser { + let item = res?; + + self.data.replace(item); + } + + Ok(()) + } + + pub fn load_from_path(path: impl AsRef) -> Result { + let mut this = Self { + file: FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?, + data: BTreeSet::default(), + }; + this.load_impl()?; + Ok(this) + } + + pub fn load() -> Result { + Self::load_from_path(default_path()?) + } + + /// **Warning: This will overwrite all existing records!** + pub fn overwrite(mut self) -> Result<(), Error> { + self.file.rewind()?; + write_to(&mut self.file, &mut self.data.into_iter())?; + + let len = self.file.stream_position()?; + self.file.set_len(len)?; + + Ok(()) + } + + pub fn get(&self, value: impl AsRef) -> Option<&MetaData> { + self.data.get(value.as_ref()) + } + + pub fn contains(&self, value: impl AsRef) -> bool { + self.data.contains(value.as_ref()) + } + + /// Adds a value to the set. + /// If the set did not have an equal element present, true is returned. + /// If the set did have an equal element present, false is returned, + /// and the entry is not updated. + pub fn insert(&mut self, value: MetaData) -> bool { + self.data.insert(value) + } + + pub fn replace(&mut self, value: MetaData) -> Option { + self.data.replace(value) + } + + pub fn remove(&mut self, value: impl AsRef) -> bool { + self.data.remove(value.as_ref()) + } + + pub fn take(&mut self, value: impl AsRef) -> Option { + self.data.take(value.as_ref()) + } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } +} + +impl<'a> IntoIterator for &'a Records { + type Item = &'a MetaData; + + type IntoIter = btree_set::Iter<'a, MetaData>; + + fn into_iter(self) -> Self::IntoIter { + self.data.iter() + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::target::TARGET; + + use tempfile::NamedTempFile; + + macro_rules! assert_records_eq { + ($records:expr, $metadata_set:expr) => { + assert_eq!($records.len(), $metadata_set.len()); + for (record, metadata) in $records.into_iter().zip($metadata_set.iter()) { + assert_eq!(record, metadata); + } + }; + } + + #[test] + fn rw_test() { + let target = CompactString::from(TARGET); + + let named_tempfile = NamedTempFile::new().unwrap(); + let path = named_tempfile.path(); + + let metadata_vec = [ + MetaData { + name: "a".into(), + version_req: "*".into(), + current_version: Version::new(0, 1, 0), + source: Source::cratesio_registry(), + target: target.clone(), + bins: vec!["1".into(), "2".into()], + other: Default::default(), + }, + MetaData { + name: "b".into(), + version_req: "0.1.0".into(), + current_version: Version::new(0, 1, 0), + source: Source::cratesio_registry(), + target: target.clone(), + bins: vec!["1".into(), "2".into()], + other: Default::default(), + }, + MetaData { + name: "a".into(), + version_req: "*".into(), + current_version: Version::new(0, 2, 0), + source: Source::cratesio_registry(), + target: target.clone(), + bins: vec!["1".into()], + other: Default::default(), + }, + ]; + + append_to_path(&path, metadata_vec.clone()).unwrap(); + + let mut iter = metadata_vec.into_iter(); + iter.next().unwrap(); + + let mut metadata_set: BTreeSet<_> = iter.collect(); + + let mut records = Records::load_from_path(&path).unwrap(); + assert_records_eq!(&records, &metadata_set); + + records.remove("b"); + assert_eq!(records.len(), metadata_set.len() - 1); + records.overwrite().unwrap(); + + metadata_set.remove("b"); + let records = Records::load_from_path(&path).unwrap(); + assert_records_eq!(&records, &metadata_set); + // Drop the exclusive file lock + drop(records); + + let new_metadata = MetaData { + name: "b".into(), + version_req: "0.1.0".into(), + current_version: Version::new(0, 1, 1), + source: Source::cratesio_registry(), + target, + bins: vec!["1".into(), "2".into()], + other: Default::default(), + }; + append_to_path(&path, [new_metadata.clone()]).unwrap(); + metadata_set.insert(new_metadata); + + let records = Records::load_from_path(&path).unwrap(); + assert_records_eq!(&records, &metadata_set); + } +} diff --git a/src/metafiles/cvs.rs b/src/metafiles/cvs.rs index b56ab5fc..f8bd7777 100644 --- a/src/metafiles/cvs.rs +++ b/src/metafiles/cvs.rs @@ -1,19 +1,31 @@ use std::{borrow::Cow, fmt, str::FromStr}; +use compact_str::CompactString; use miette::Diagnostic; -use once_cell::sync::Lazy; use semver::Version; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; use url::Url; +use crate::cratesio_url; + #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] pub struct CrateVersionSource { - pub name: String, + pub name: CompactString, pub version: Version, pub source: Source, } +impl From<&super::binstall_v1::MetaData> for CrateVersionSource { + fn from(metadata: &super::binstall_v1::MetaData) -> Self { + super::CrateVersionSource { + name: metadata.name.clone(), + version: metadata.current_version.clone(), + source: Source::from(&metadata.source), + } + } +} + #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] pub enum Source { Git(Url), @@ -23,10 +35,21 @@ pub enum Source { impl Source { pub fn cratesio_registry() -> Source { - static CRATESIO: Lazy Url> = - Lazy::new(|| url::Url::parse("https://github.com/rust-lang/crates.io-index").unwrap()); + Self::Registry(cratesio_url().clone()) + } +} - Self::Registry(CRATESIO.clone()) +impl From<&super::binstall_v1::Source> for Source { + fn from(source: &super::binstall_v1::Source) -> Self { + use super::binstall_v1::SourceType::*; + + let url = source.url.clone(); + + match source.source_type { + Git => Self::Git(url), + Path => Self::Path(url), + Registry => Self::Registry(url), + } } } @@ -53,7 +76,7 @@ impl FromStr for CrateVersionSource { _ => return Err(CvsParseError::BadSource), }; Ok(Self { - name: name.to_string(), + name: name.into(), version, source, }) diff --git a/src/metafiles/v1.rs b/src/metafiles/v1.rs index 374188ba..25ef6c3f 100644 --- a/src/metafiles/v1.rs +++ b/src/metafiles/v1.rs @@ -11,7 +11,7 @@ use miette::Diagnostic; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::CrateVersionSource; +use super::{binstall_v1::MetaData, CrateVersionSource}; use crate::{cargo_home, create_if_not_exist, FileLock}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -71,13 +71,13 @@ impl CratesToml { iter: Iter, ) -> Result<(), CratesTomlParseError> where - Iter: IntoIterator)>, + Iter: IntoIterator, { let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?; let mut c1 = Self::load_from_reader(&mut *file)?; - for (cvs, bins) in iter { - c1.insert(cvs, bins); + for metadata in iter { + c1.insert(&CrateVersionSource::from(metadata), metadata.bins.clone()); } file.rewind()?; @@ -88,7 +88,7 @@ impl CratesToml { pub fn append<'a, Iter>(iter: Iter) -> Result<(), CratesTomlParseError> where - Iter: IntoIterator)>, + Iter: IntoIterator, { Self::append_to_path(Self::default_path()?, iter) }