Merge pull request #252 from NobodyXu/feature/binstall-format

Feature: Create metafile format for cargo-binstall

Cargo.lock

@@ -148,6 +148,8 @@ dependencies = [
  "scopeguard",
  "semver",
  "serde",
+ "serde-tuple-vec-map",
+ "serde_json",
  "simplelog",
  "strum",
  "strum_macros",
@@ -1375,6 +1377,9 @@ name = "semver"
 version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "serde"
@@ -1385,6 +1390,15 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-tuple-vec-map"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a04d0ebe0de77d7d445bb729a895dcb0a288854b267ca85f030ce51cdc578c82"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.140"
@@ -1811,6 +1825,7 @@ dependencies = [
  "idna",
  "matches",
  "percent-encoding",
+ "serde",
 ]
 
 [[package]]

Cargo.toml

@@ -38,8 +38,10 @@ mimalloc = { version = "0.1.29", default-features = false, optional = true }
 once_cell = "1.13.0"
 reqwest = { version = "0.11.11", features = ["stream"], default-features = false }
 scopeguard = "1.1.0"
-semver = "1.0.12"
+semver = { version = "1.0.12", features = ["serde"] }
 serde = { version = "1.0.140", features = ["derive"] }
+serde-tuple-vec-map = "1.0.1"
+serde_json = "1.0.82"
 simplelog = "0.12.0"
 strum = "0.24.1"
 strum_macros = "0.24.2"
@@ -49,7 +51,7 @@ thiserror = "1.0.31"
 tinytemplate = "1.2.1"
 tokio = { version = "1.20.0", features = ["rt-multi-thread", "process", "sync"], default-features = false }
 toml_edit = { version = "0.14.4", features = ["easy"] }
-url = "2.2.2"
+url = { version = "2.2.2", features = ["serde"] }
 xz2 = "0.1.7"
 
 # Disable all features of zip except for features of compression algorithms:

src/binstall.rs

@@ -1,8 +1,6 @@
 use std::path::PathBuf;
 
-use compact_str::CompactString;
-
-use crate::{metafiles, DesiredTargets, PkgOverride};
+use crate::{metafiles::binstall_v1::MetaData, DesiredTargets, PkgOverride};
 
 mod resolve;
 pub use resolve::*;
@@ -18,11 +16,3 @@ pub struct Options {
     pub cli_overrides: PkgOverride,
     pub desired_targets: DesiredTargets,
 }
-
-/// MetaData required to update MetaFiles.
-pub struct MetaData {
-    pub bins: Vec<CompactString>,
-    pub cvs: metafiles::CrateVersionSource,
-    pub version_req: String,
-    pub target: String,
-}

src/binstall/install.rs

@@ -1,12 +1,13 @@
 use std::{path::PathBuf, process, sync::Arc};
 
 use cargo_toml::Package;
+use compact_str::CompactString;
 use log::{debug, error, info};
 use miette::{miette, IntoDiagnostic, Result, WrapErr};
 use tokio::{process::Command, task::block_in_place};
 
 use super::{MetaData, Options, Resolution};
-use crate::{bins, fetchers::Fetcher, *};
+use crate::{bins, fetchers::Fetcher, metafiles::binstall_v1::Source, *};
 
 pub async fn install(
     resolution: Resolution,
@@ -22,13 +23,22 @@ pub async fn install(
             bin_path,
             bin_files,
         } => {
-            let cvs = metafiles::CrateVersionSource {
-                name,
-                version: package.version.parse().into_diagnostic()?,
-                source: metafiles::Source::cratesio_registry(),
-            };
+            let current_version = package.version.parse().into_diagnostic()?;
+            let target = fetcher.target().into();
 
-            install_from_package(fetcher, opts, cvs, version, bin_path, bin_files).await
+            install_from_package(fetcher, opts, bin_path, bin_files)
+                .await
+                .map(|option| {
+                    option.map(|bins| MetaData {
+                        name: name.into(),
+                        version_req: version.into(),
+                        current_version,
+                        source: Source::cratesio_registry(),
+                        target,
+                        bins,
+                        other: Default::default(),
+                    })
+                })
         }
         Resolution::InstallFromSource { package } => {
             let desired_targets = opts.desired_targets.get().await;
@@ -54,11 +64,9 @@ pub async fn install(
 async fn install_from_package(
     fetcher: Arc<dyn Fetcher>,
     opts: Arc<Options>,
-    cvs: metafiles::CrateVersionSource,
-    version: String,
     bin_path: PathBuf,
     bin_files: Vec<bins::BinFile>,
-) -> Result<Option<MetaData>> {
+) -> Result<Option<Vec<CompactString>>> {
     // Download package
     if opts.dry_run {
         info!("Dry run, not downloading package");
@@ -108,12 +116,9 @@ async fn install_from_package(
             }
         }
 
-        Ok(Some(MetaData {
-            bins: bin_files.into_iter().map(|bin| bin.base_name).collect(),
-            cvs,
-            version_req: version,
-            target: fetcher.target().to_string(),
-        }))
+        Ok(Some(
+            bin_files.into_iter().map(|bin| bin.base_name).collect(),
+        ))
     })
 }
 

src/helpers.rs

@@ -10,7 +10,7 @@ use bytes::Bytes;
 use cargo_toml::Manifest;
 use futures_util::stream::Stream;
 use log::debug;
-use once_cell::sync::OnceCell;
+use once_cell::sync::{Lazy, OnceCell};
 use reqwest::{tls, Client, ClientBuilder, Method, Response};
 use serde::Serialize;
 use tempfile::NamedTempFile;
@@ -55,6 +55,13 @@ pub fn cargo_home() -> Result<&'static Path, io::Error> {
         .map(ops::Deref::deref)
 }
 
+pub fn cratesio_url() -> &'static Url {
+    static CRATESIO: Lazy<Url, fn() -> Url> =
+        Lazy::new(|| url::Url::parse("https://github.com/rust-lang/crates.io-index").unwrap());
+
+    &*CRATESIO
+}
+
 /// Returned file is readable and writable.
 pub fn create_if_not_exist(path: impl AsRef<Path>) -> io::Result<fs::File> {
     let path = path.as_ref();

src/helpers/flock.rs

@@ -43,3 +43,39 @@ impl ops::DerefMut for FileLock {
         &mut self.0
     }
 }
+
+impl io::Write for FileLock {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        self.0.write(buf)
+    }
+    fn flush(&mut self) -> io::Result<()> {
+        self.0.flush()
+    }
+
+    fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result<usize> {
+        self.0.write_vectored(bufs)
+    }
+}
+
+impl io::Read for FileLock {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.0.read(buf)
+    }
+
+    fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result<usize> {
+        self.0.read_vectored(bufs)
+    }
+}
+
+impl io::Seek for FileLock {
+    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
+        self.0.seek(pos)
+    }
+
+    fn rewind(&mut self) -> io::Result<()> {
+        self.0.rewind()
+    }
+    fn stream_position(&mut self) -> io::Result<u64> {
+        self.0.stream_position()
+    }
+}

src/main.rs

@@ -344,11 +344,10 @@ async fn entry(jobserver_client: LazyJobserverClient) -> Result<()> {
     block_in_place(|| {
         if !custom_install_path {
             debug!("Writing .crates.toml");
-            metafiles::v1::CratesToml::append(
-                metadata_vec
-                    .iter()
-                    .map(|metadata| (&metadata.cvs, metadata.bins.clone())),
-            )?;
+            metafiles::v1::CratesToml::append(metadata_vec.iter())?;
+
+            debug!("Writing binstall/crates-v1.json");
+            metafiles::binstall_v1::append(metadata_vec)?;
         }
 
         if opts.no_cleanup {

src/metafiles.rs

@@ -2,3 +2,5 @@ mod cvs;
 pub use cvs::*;
 
 pub mod v1;
+
+pub mod binstall_v1;

src/metafiles/binstall_v1.rs

@@ -0,0 +1,324 @@
+use std::{
+    borrow, cmp,
+    collections::{btree_set, BTreeSet},
+    fs, hash,
+    io::{self, Seek, Write},
+    iter::{IntoIterator, Iterator},
+    path::{Path, PathBuf},
+};
+
+use compact_str::CompactString;
+use miette::Diagnostic;
+use semver::Version;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+use url::Url;
+
+use crate::{cargo_home, cratesio_url, create_if_not_exist, FileLock};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct MetaData {
+    pub name: CompactString,
+    pub version_req: CompactString,
+    pub current_version: Version,
+    pub source: Source,
+    pub target: CompactString,
+    pub bins: Vec<CompactString>,
+
+    /// Forwards compatibility. Unknown keys from future versions
+    /// will be stored here and retained when the file is saved.
+    ///
+    /// We use an `Vec` here since it is never accessed in Rust.
+    #[serde(flatten, with = "tuple_vec_map")]
+    pub other: Vec<(CompactString, serde_json::Value)>,
+}
+
+impl borrow::Borrow<str> for MetaData {
+    fn borrow(&self) -> &str {
+        &self.name
+    }
+}
+
+impl PartialEq for MetaData {
+    fn eq(&self, other: &Self) -> bool {
+        self.name == other.name
+    }
+}
+impl Eq for MetaData {}
+
+impl PartialOrd for MetaData {
+    fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
+        self.name.partial_cmp(&other.name)
+    }
+}
+
+impl Ord for MetaData {
+    fn cmp(&self, other: &Self) -> cmp::Ordering {
+        self.name.cmp(&other.name)
+    }
+}
+
+impl hash::Hash for MetaData {
+    fn hash<H>(&self, state: &mut H)
+    where
+        H: hash::Hasher,
+    {
+        self.name.hash(state)
+    }
+}
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
+pub enum SourceType {
+    Git,
+    Path,
+    Registry,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Source {
+    pub source_type: SourceType,
+    pub url: Url,
+}
+
+impl Source {
+    pub fn cratesio_registry() -> Source {
+        Self {
+            source_type: SourceType::Registry,
+            url: cratesio_url().clone(),
+        }
+    }
+}
+
+#[derive(Debug, Diagnostic, Error)]
+pub enum Error {
+    #[error(transparent)]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    SerdeJsonParse(#[from] serde_json::Error),
+}
+
+pub fn append_to_path<Iter>(path: impl AsRef<Path>, iter: Iter) -> Result<(), Error>
+where
+    Iter: IntoIterator<Item = MetaData>,
+{
+    let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?;
+    // Move the cursor to EOF
+    file.seek(io::SeekFrom::End(0))?;
+
+    write_to(&mut file, &mut iter.into_iter())
+}
+
+pub fn append<Iter>(iter: Iter) -> Result<(), Error>
+where
+    Iter: IntoIterator<Item = MetaData>,
+{
+    append_to_path(default_path()?, iter)
+}
+
+pub fn write_to(
+    file: &mut FileLock,
+    iter: &mut dyn Iterator<Item = MetaData>,
+) -> Result<(), Error> {
+    let writer = io::BufWriter::with_capacity(512, file);
+
+    let mut ser = serde_json::Serializer::new(writer);
+
+    for item in iter {
+        item.serialize(&mut ser)?;
+    }
+
+    ser.into_inner().flush()?;
+
+    Ok(())
+}
+
+pub fn default_path() -> Result<PathBuf, Error> {
+    let dir = cargo_home()?.join("binstall");
+
+    fs::create_dir_all(&dir)?;
+
+    Ok(dir.join("crates-v1.json"))
+}
+
+#[derive(Debug)]
+pub struct Records {
+    file: FileLock,
+    /// Use BTreeSet to dedup the metadata
+    data: BTreeSet<MetaData>,
+}
+
+impl Records {
+    fn load_impl(&mut self) -> Result<(), Error> {
+        let reader = io::BufReader::with_capacity(1024, &mut self.file);
+        let stream_deser = serde_json::Deserializer::from_reader(reader).into_iter();
+
+        for res in stream_deser {
+            let item = res?;
+
+            self.data.replace(item);
+        }
+
+        Ok(())
+    }
+
+    pub fn load_from_path(path: impl AsRef<Path>) -> Result<Self, Error> {
+        let mut this = Self {
+            file: FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?,
+            data: BTreeSet::default(),
+        };
+        this.load_impl()?;
+        Ok(this)
+    }
+
+    pub fn load() -> Result<Self, Error> {
+        Self::load_from_path(default_path()?)
+    }
+
+    /// **Warning: This will overwrite all existing records!**
+    pub fn overwrite(mut self) -> Result<(), Error> {
+        self.file.rewind()?;
+        write_to(&mut self.file, &mut self.data.into_iter())?;
+
+        let len = self.file.stream_position()?;
+        self.file.set_len(len)?;
+
+        Ok(())
+    }
+
+    pub fn get(&self, value: impl AsRef<str>) -> Option<&MetaData> {
+        self.data.get(value.as_ref())
+    }
+
+    pub fn contains(&self, value: impl AsRef<str>) -> bool {
+        self.data.contains(value.as_ref())
+    }
+
+    /// Adds a value to the set.
+    /// If the set did not have an equal element present, true is returned.
+    /// If the set did have an equal element present, false is returned,
+    /// and the entry is not updated.
+    pub fn insert(&mut self, value: MetaData) -> bool {
+        self.data.insert(value)
+    }
+
+    pub fn replace(&mut self, value: MetaData) -> Option<MetaData> {
+        self.data.replace(value)
+    }
+
+    pub fn remove(&mut self, value: impl AsRef<str>) -> bool {
+        self.data.remove(value.as_ref())
+    }
+
+    pub fn take(&mut self, value: impl AsRef<str>) -> Option<MetaData> {
+        self.data.take(value.as_ref())
+    }
+
+    pub fn len(&self) -> usize {
+        self.data.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.data.is_empty()
+    }
+}
+
+impl<'a> IntoIterator for &'a Records {
+    type Item = &'a MetaData;
+
+    type IntoIter = btree_set::Iter<'a, MetaData>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.data.iter()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::target::TARGET;
+
+    use tempfile::NamedTempFile;
+
+    macro_rules! assert_records_eq {
+        ($records:expr, $metadata_set:expr) => {
+            assert_eq!($records.len(), $metadata_set.len());
+            for (record, metadata) in $records.into_iter().zip($metadata_set.iter()) {
+                assert_eq!(record, metadata);
+            }
+        };
+    }
+
+    #[test]
+    fn rw_test() {
+        let target = CompactString::from(TARGET);
+
+        let named_tempfile = NamedTempFile::new().unwrap();
+        let path = named_tempfile.path();
+
+        let metadata_vec = [
+            MetaData {
+                name: "a".into(),
+                version_req: "*".into(),
+                current_version: Version::new(0, 1, 0),
+                source: Source::cratesio_registry(),
+                target: target.clone(),
+                bins: vec!["1".into(), "2".into()],
+                other: Default::default(),
+            },
+            MetaData {
+                name: "b".into(),
+                version_req: "0.1.0".into(),
+                current_version: Version::new(0, 1, 0),
+                source: Source::cratesio_registry(),
+                target: target.clone(),
+                bins: vec!["1".into(), "2".into()],
+                other: Default::default(),
+            },
+            MetaData {
+                name: "a".into(),
+                version_req: "*".into(),
+                current_version: Version::new(0, 2, 0),
+                source: Source::cratesio_registry(),
+                target: target.clone(),
+                bins: vec!["1".into()],
+                other: Default::default(),
+            },
+        ];
+
+        append_to_path(&path, metadata_vec.clone()).unwrap();
+
+        let mut iter = metadata_vec.into_iter();
+        iter.next().unwrap();
+
+        let mut metadata_set: BTreeSet<_> = iter.collect();
+
+        let mut records = Records::load_from_path(&path).unwrap();
+        assert_records_eq!(&records, &metadata_set);
+
+        records.remove("b");
+        assert_eq!(records.len(), metadata_set.len() - 1);
+        records.overwrite().unwrap();
+
+        metadata_set.remove("b");
+        let records = Records::load_from_path(&path).unwrap();
+        assert_records_eq!(&records, &metadata_set);
+        // Drop the exclusive file lock
+        drop(records);
+
+        let new_metadata = MetaData {
+            name: "b".into(),
+            version_req: "0.1.0".into(),
+            current_version: Version::new(0, 1, 1),
+            source: Source::cratesio_registry(),
+            target,
+            bins: vec!["1".into(), "2".into()],
+            other: Default::default(),
+        };
+        append_to_path(&path, [new_metadata.clone()]).unwrap();
+        metadata_set.insert(new_metadata);
+
+        let records = Records::load_from_path(&path).unwrap();
+        assert_records_eq!(&records, &metadata_set);
+    }
+}

src/metafiles/cvs.rs

@@ -1,19 +1,31 @@
 use std::{borrow::Cow, fmt, str::FromStr};
 
+use compact_str::CompactString;
 use miette::Diagnostic;
-use once_cell::sync::Lazy;
 use semver::Version;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use thiserror::Error;
 use url::Url;
 
+use crate::cratesio_url;
+
 #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)]
 pub struct CrateVersionSource {
-    pub name: String,
+    pub name: CompactString,
     pub version: Version,
     pub source: Source,
 }
 
+impl From<&super::binstall_v1::MetaData> for CrateVersionSource {
+    fn from(metadata: &super::binstall_v1::MetaData) -> Self {
+        super::CrateVersionSource {
+            name: metadata.name.clone(),
+            version: metadata.current_version.clone(),
+            source: Source::from(&metadata.source),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)]
 pub enum Source {
     Git(Url),
@@ -23,10 +35,21 @@ pub enum Source {
 
 impl Source {
     pub fn cratesio_registry() -> Source {
-        static CRATESIO: Lazy<Url, fn() -> Url> =
-            Lazy::new(|| url::Url::parse("https://github.com/rust-lang/crates.io-index").unwrap());
+        Self::Registry(cratesio_url().clone())
+    }
+}
 
-        Self::Registry(CRATESIO.clone())
+impl From<&super::binstall_v1::Source> for Source {
+    fn from(source: &super::binstall_v1::Source) -> Self {
+        use super::binstall_v1::SourceType::*;
+
+        let url = source.url.clone();
+
+        match source.source_type {
+            Git => Self::Git(url),
+            Path => Self::Path(url),
+            Registry => Self::Registry(url),
+        }
     }
 }
 
@@ -53,7 +76,7 @@ impl FromStr for CrateVersionSource {
                     _ => return Err(CvsParseError::BadSource),
                 };
                 Ok(Self {
-                    name: name.to_string(),
+                    name: name.into(),
                     version,
                     source,
                 })

src/metafiles/v1.rs

@@ -11,7 +11,7 @@ use miette::Diagnostic;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 
-use super::CrateVersionSource;
+use super::{binstall_v1::MetaData, CrateVersionSource};
 use crate::{cargo_home, create_if_not_exist, FileLock};
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
@@ -71,13 +71,13 @@ impl CratesToml {
         iter: Iter,
     ) -> Result<(), CratesTomlParseError>
     where
-        Iter: IntoIterator<Item = (&'a CrateVersionSource, Vec<CompactString>)>,
+        Iter: IntoIterator<Item = &'a MetaData>,
     {
         let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?;
         let mut c1 = Self::load_from_reader(&mut *file)?;
 
-        for (cvs, bins) in iter {
-            c1.insert(cvs, bins);
+        for metadata in iter {
+            c1.insert(&CrateVersionSource::from(metadata), metadata.bins.clone());
         }
 
         file.rewind()?;
@@ -88,7 +88,7 @@ impl CratesToml {
 
     pub fn append<'a, Iter>(iter: Iter) -> Result<(), CratesTomlParseError>
     where
-        Iter: IntoIterator<Item = (&'a CrateVersionSource, Vec<CompactString>)>,
+        Iter: IntoIterator<Item = &'a MetaData>,
     {
         Self::append_to_path(Self::default_path()?, iter)
     }