From e9901519190403c72269cfa5211425a69258e93d Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 21:31:58 +1000 Subject: [PATCH 01/44] Add new dep serde_json v1.0.82 Signed-off-by: Jiahao XU --- Cargo.lock | 1 + Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ba311b77..2477b9e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,7 @@ dependencies = [ "scopeguard", "semver", "serde", + "serde_json", "simplelog", "strum", "strum_macros", diff --git a/Cargo.toml b/Cargo.toml index 410d4a13..da5d0c18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ reqwest = { version = "0.11.11", features = ["stream"], default-features = false scopeguard = "1.1.0" semver = "1.0.12" serde = { version = "1.0.140", features = ["derive"] } +serde_json = "1.0.82" simplelog = "0.12.0" strum = "0.24.1" strum_macros = "0.24.2" From f7c798352aa379331b0ad3ef53a7d06389b08e0b Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 21:57:57 +1000 Subject: [PATCH 02/44] Enable feature serde of semver Signed-off-by: Jiahao XU --- Cargo.lock | 3 +++ Cargo.toml | 2 +- src/metafiles/binstall_v1.rs | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 src/metafiles/binstall_v1.rs diff --git a/Cargo.lock b/Cargo.lock index 2477b9e2..1879a402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1376,6 +1376,9 @@ name = "semver" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2333e6df6d6598f2b1974829f853c2b4c5f4a6e503c10af918081aa6f8564e1" +dependencies = [ + "serde", +] [[package]] name = "serde" diff --git a/Cargo.toml b/Cargo.toml index da5d0c18..913a1fd7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ mimalloc = { version = "0.1.29", default-features = false, optional = true } once_cell = "1.13.0" reqwest = { version = "0.11.11", features = ["stream"], default-features = false } scopeguard = "1.1.0" -semver = "1.0.12" +semver = { version = "1.0.12", features = ["serde"] } serde = { version = "1.0.140", features = ["derive"] } serde_json = "1.0.82" simplelog = "0.12.0" diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs new file mode 100644 index 00000000..e3f059f7 --- /dev/null +++ b/src/metafiles/binstall_v1.rs @@ -0,0 +1,15 @@ +/* +{ + "name": "cargo-binstall", + "version_req": none, + "current_version": "0.10.0" + "source": { + "type": "Registry", + "url": "https://crates.io", + }, + "target": none, + "bins": [ + "cargo-binstall", + ], +} + */ From 93d4dbcd1ab715ea01285f81986f295cd9a468d8 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:12:04 +1000 Subject: [PATCH 03/44] Enable feature serde of dep url Signed-off-by: Jiahao XU --- Cargo.lock | 1 + Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1879a402..55b7e882 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1815,6 +1815,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 913a1fd7..8907e68f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ thiserror = "1.0.31" tinytemplate = "1.2.1" tokio = { version = "1.20.0", features = ["rt-multi-thread", "process", "sync"], default-features = false } toml_edit = { version = "0.14.4", features = ["easy"] } -url = "2.2.2" +url = { version = "2.2.2", features = ["serde"] } xz2 = "0.1.7" # Disable all features of zip except for features of compression algorithms: From 172af54cd8ae0e39df978eb780aafbb0ceb0613d Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:29:10 +1000 Subject: [PATCH 04/44] Impl `metafiles::binstall_v1::{Entry, Source}` Signed-off-by: Jiahao XU --- src/metafiles.rs | 2 + src/metafiles/binstall_v1.rs | 80 +++++++++++++++++++++++++++++------- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/metafiles.rs b/src/metafiles.rs index 75ad0250..b8a2b4ee 100644 --- a/src/metafiles.rs +++ b/src/metafiles.rs @@ -2,3 +2,5 @@ mod cvs; pub use cvs::*; pub mod v1; + +pub mod binstall_v1; diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index e3f059f7..35e78ed8 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,15 +1,67 @@ -/* -{ - "name": "cargo-binstall", - "version_req": none, - "current_version": "0.10.0" - "source": { - "type": "Registry", - "url": "https://crates.io", - }, - "target": none, - "bins": [ - "cargo-binstall", - ], +use compact_str::CompactString; +use semver::Version; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::binstall::MetaData; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Entry { + pub name: CompactString, + pub version_req: CompactString, + pub current_version: Version, + pub source: Source, + pub target: CompactString, + pub bins: Vec, +} +impl Entry { + pub fn new(metadata: MetaData) -> Self { + let MetaData { + bins, + cvs: + super::CrateVersionSource { + name, + version, + source, + }, + version_req, + target, + } = metadata; + + Self { + name: name.into(), + version_req: version_req.into(), + current_version: version, + source: source.into(), + target: target.into(), + bins, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Source { + source_type: CompactString, + url: Url, +} + +impl From for Source { + fn from(src: super::Source) -> Self { + use super::Source::*; + + match src { + Git(url) => Source { + source_type: "Git".into(), + url, + }, + Path(url) => Source { + source_type: "Path".into(), + url, + }, + Registry(url) => Source { + source_type: "Registry".into(), + url, + }, + } + } } - */ From 46c4d6f406efa1994d7cf5841cbaf65b07c61a95 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:39:15 +1000 Subject: [PATCH 05/44] Impl `io::Write` for `helpers::FileLock` Signed-off-by: Jiahao XU --- src/helpers/flock.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/helpers/flock.rs b/src/helpers/flock.rs index 2c0707b5..285c357d 100644 --- a/src/helpers/flock.rs +++ b/src/helpers/flock.rs @@ -43,3 +43,16 @@ impl ops::DerefMut for FileLock { &mut self.0 } } + +impl io::Write for FileLock { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + self.0.write_vectored(bufs) + } +} From 461571075dc2ac868e2fba92df6bb8fb1d906d59 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:45:48 +1000 Subject: [PATCH 06/44] Impl `binstall_v1::Error` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 35e78ed8..3ed2a1bc 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,9 +1,19 @@ +use std::{ + fs, + io::{self, Write}, + iter::IntoIterator, + path::Path, +}; + use compact_str::CompactString; +use miette::Diagnostic; use semver::Version; use serde::{Deserialize, Serialize}; +use thiserror::Error; use url::Url; use crate::binstall::MetaData; +use crate::FileLock; #[derive(Debug, Serialize, Deserialize)] pub struct Entry { @@ -65,3 +75,12 @@ impl From for Source { } } } + +#[derive(Debug, Diagnostic, Error)] +pub enum Error { + #[error(transparent)] + Io(#[from] io::Error), + + #[error(transparent)] + SerdeJsonParse(#[from] serde_json::Error), +} From f0b6b7b1afcf43a1018492c4c7a037a990fe4540 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:45:59 +1000 Subject: [PATCH 07/44] Impl `binstall_v1::append_to_path` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 3ed2a1bc..0089c5bb 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -84,3 +84,27 @@ pub enum Error { #[error(transparent)] SerdeJsonParse(#[from] serde_json::Error), } + +pub fn append_to_path(path: impl AsRef, iter: Iter) -> Result<(), Error> +where + Iter: IntoIterator, +{ + let file = FileLock::new_exclusive( + fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?, + )?; + + let writer = io::BufWriter::with_capacity(512, file); + + let mut ser = serde_json::Serializer::new(writer); + + for item in iter { + item.serialize(&mut ser)?; + } + + ser.into_inner().flush()?; + + Ok(()) +} From c6c5dcd79fd779c5413927d3ba3fcee1f3445318 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:47:57 +1000 Subject: [PATCH 08/44] Rename `binstall_v1::Entry` to `Item` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 0089c5bb..f93916ce 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -16,7 +16,7 @@ use crate::binstall::MetaData; use crate::FileLock; #[derive(Debug, Serialize, Deserialize)] -pub struct Entry { +pub struct Item { pub name: CompactString, pub version_req: CompactString, pub current_version: Version, @@ -24,7 +24,7 @@ pub struct Entry { pub target: CompactString, pub bins: Vec, } -impl Entry { +impl Item { pub fn new(metadata: MetaData) -> Self { let MetaData { bins, @@ -87,7 +87,7 @@ pub enum Error { pub fn append_to_path(path: impl AsRef, iter: Iter) -> Result<(), Error> where - Iter: IntoIterator, + Iter: IntoIterator, { let file = FileLock::new_exclusive( fs::OpenOptions::new() From ebb2d5d0c37ea36e6a328d451a1d67424eee15f9 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:51:20 +1000 Subject: [PATCH 09/44] Impl `io::Read` for `FileLock` Signed-off-by: Jiahao XU --- src/helpers/flock.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/helpers/flock.rs b/src/helpers/flock.rs index 285c357d..f0dbb9f2 100644 --- a/src/helpers/flock.rs +++ b/src/helpers/flock.rs @@ -56,3 +56,13 @@ impl io::Write for FileLock { self.0.write_vectored(bufs) } } + +impl io::Read for FileLock { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0.read(buf) + } + + fn read_vectored(&mut self, bufs: &mut [io::IoSliceMut<'_>]) -> io::Result { + self.0.read_vectored(bufs) + } +} From ce50186f4a64304a6f94c108e3efc5408b983efa Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 26 Jul 2022 22:52:25 +1000 Subject: [PATCH 10/44] Impl `io::Seek` for `FileLock` Signed-off-by: Jiahao XU --- src/helpers/flock.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/helpers/flock.rs b/src/helpers/flock.rs index f0dbb9f2..f8b4514a 100644 --- a/src/helpers/flock.rs +++ b/src/helpers/flock.rs @@ -66,3 +66,16 @@ impl io::Read for FileLock { self.0.read_vectored(bufs) } } + +impl io::Seek for FileLock { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + self.0.seek(pos) + } + + fn rewind(&mut self) -> io::Result<()> { + self.0.rewind() + } + fn stream_position(&mut self) -> io::Result { + self.0.stream_position() + } +} From 951a0f8b9f1be17257b450c1151fc5f4e8f735c4 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 18:38:26 +1000 Subject: [PATCH 11/44] Impl `binstall_v1::default_path` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index f93916ce..dd0428ba 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -2,7 +2,7 @@ use std::{ fs, io::{self, Write}, iter::IntoIterator, - path::Path, + path::{Path, PathBuf}, }; use compact_str::CompactString; @@ -13,7 +13,7 @@ use thiserror::Error; use url::Url; use crate::binstall::MetaData; -use crate::FileLock; +use crate::{cargo_home, FileLock}; #[derive(Debug, Serialize, Deserialize)] pub struct Item { @@ -108,3 +108,7 @@ where Ok(()) } + +pub fn default_path() -> Result { + Ok(cargo_home()?.join(".binstall-crates.toml")) +} From cbd64b039d6ce9b3575e1b09a75a09e4b5dc4ab3 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 18:49:24 +1000 Subject: [PATCH 12/44] Rename `binstall_v1::Item` to `MetaData` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index dd0428ba..386b2929 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -12,11 +12,10 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use url::Url; -use crate::binstall::MetaData; use crate::{cargo_home, FileLock}; #[derive(Debug, Serialize, Deserialize)] -pub struct Item { +pub struct MetaData { pub name: CompactString, pub version_req: CompactString, pub current_version: Version, @@ -24,9 +23,9 @@ pub struct Item { pub target: CompactString, pub bins: Vec, } -impl Item { - pub fn new(metadata: MetaData) -> Self { - let MetaData { +impl MetaData { + pub fn new(metadata: crate::binstall::MetaData) -> Self { + let crate::binstall::MetaData { bins, cvs: super::CrateVersionSource { @@ -87,7 +86,7 @@ pub enum Error { pub fn append_to_path(path: impl AsRef, iter: Iter) -> Result<(), Error> where - Iter: IntoIterator, + Iter: IntoIterator, { let file = FileLock::new_exclusive( fs::OpenOptions::new() From 5ca1278c2228efcee0eb3b879c75fd26abc270a3 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 19:18:44 +1000 Subject: [PATCH 13/44] Add new enum `binstall_v1::SourceType` Use it to represent source type instead of using `CompactString`. Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 386b2929..9704719f 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -48,9 +48,16 @@ impl MetaData { } } +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +enum SourceType { + Git, + Path, + Registry, +} + #[derive(Debug, Serialize, Deserialize)] pub struct Source { - source_type: CompactString, + source_type: SourceType, url: Url, } @@ -60,15 +67,15 @@ impl From for Source { match src { Git(url) => Source { - source_type: "Git".into(), + source_type: SourceType::Git, url, }, Path(url) => Source { - source_type: "Path".into(), + source_type: SourceType::Path, url, }, Registry(url) => Source { - source_type: "Registry".into(), + source_type: SourceType::Registry, url, }, } From cc13aa911fee77938e90717078eda4256b53b544 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 19:22:45 +1000 Subject: [PATCH 14/44] Refactor: Extract `helpers::cratesio_url` Signed-off-by: Jiahao XU --- src/helpers.rs | 9 ++++++++- src/metafiles/cvs.rs | 8 +++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/helpers.rs b/src/helpers.rs index 269557fc..7a7e565d 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -10,7 +10,7 @@ use bytes::Bytes; use cargo_toml::Manifest; use futures_util::stream::Stream; use log::debug; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use reqwest::{tls, Client, ClientBuilder, Method, Response}; use serde::Serialize; use tempfile::NamedTempFile; @@ -55,6 +55,13 @@ pub fn cargo_home() -> Result<&'static Path, io::Error> { .map(ops::Deref::deref) } +pub fn cratesio_url() -> &'static Url { + static CRATESIO: Lazy Url> = + Lazy::new(|| url::Url::parse("https://github.com/rust-lang/crates.io-index").unwrap()); + + &*CRATESIO +} + /// Returned file is readable and writable. pub fn create_if_not_exist(path: impl AsRef) -> io::Result { let path = path.as_ref(); diff --git a/src/metafiles/cvs.rs b/src/metafiles/cvs.rs index b56ab5fc..15121c69 100644 --- a/src/metafiles/cvs.rs +++ b/src/metafiles/cvs.rs @@ -1,12 +1,13 @@ use std::{borrow::Cow, fmt, str::FromStr}; use miette::Diagnostic; -use once_cell::sync::Lazy; use semver::Version; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; use url::Url; +use crate::cratesio_url; + #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] pub struct CrateVersionSource { pub name: String, @@ -23,10 +24,7 @@ pub enum Source { impl Source { pub fn cratesio_registry() -> Source { - static CRATESIO: Lazy Url> = - Lazy::new(|| url::Url::parse("https://github.com/rust-lang/crates.io-index").unwrap()); - - Self::Registry(CRATESIO.clone()) + Self::Registry(cratesio_url().clone()) } } From 1a8fda1f5ebe1e48f5b3365f4e8cb1ee7fec2fd2 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 19:23:15 +1000 Subject: [PATCH 15/44] Impl `binstall_v1::Source::cratesio_registry` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 9704719f..4ba95dae 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use url::Url; -use crate::{cargo_home, FileLock}; +use crate::{cargo_home, cratesio_url, FileLock}; #[derive(Debug, Serialize, Deserialize)] pub struct MetaData { @@ -61,6 +61,15 @@ pub struct Source { url: Url, } +impl Source { + pub fn cratesio_registry() -> Source { + Self { + source_type: SourceType::Registry, + url: cratesio_url().clone(), + } + } +} + impl From for Source { fn from(src: super::Source) -> Self { use super::Source::*; From 05c0d5fcae8e54df788cfde2e5a7897d5eabc59f Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 19:24:01 +1000 Subject: [PATCH 16/44] Mark `binstall_v1::{Source, SourceType}` pub Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 4ba95dae..a99e9d89 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -49,7 +49,7 @@ impl MetaData { } #[derive(Debug, Copy, Clone, Serialize, Deserialize)] -enum SourceType { +pub enum SourceType { Git, Path, Registry, @@ -57,8 +57,8 @@ enum SourceType { #[derive(Debug, Serialize, Deserialize)] pub struct Source { - source_type: SourceType, - url: Url, + pub source_type: SourceType, + pub url: Url, } impl Source { From 2f27a5fd93fe03dee744a8d0f0a879a2d4697a2c Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 19:44:16 +1000 Subject: [PATCH 17/44] Refactor: Use `binstall_v1::MetaData` in mod `binstall` Signed-off-by: Jiahao XU --- src/binstall.rs | 12 +--------- src/binstall/install.rs | 36 ++++++++++++++++------------- src/main.rs | 6 +---- src/metafiles/binstall_v1.rs | 45 ------------------------------------ src/metafiles/cvs.rs | 24 +++++++++++++++++++ src/metafiles/v1.rs | 10 ++++---- 6 files changed, 51 insertions(+), 82 deletions(-) diff --git a/src/binstall.rs b/src/binstall.rs index f28c0e5d..c6fcc569 100644 --- a/src/binstall.rs +++ b/src/binstall.rs @@ -1,8 +1,6 @@ use std::path::PathBuf; -use compact_str::CompactString; - -use crate::{metafiles, DesiredTargets, PkgOverride}; +use crate::{metafiles::binstall_v1::MetaData, DesiredTargets, PkgOverride}; mod resolve; pub use resolve::*; @@ -18,11 +16,3 @@ pub struct Options { pub cli_overrides: PkgOverride, pub desired_targets: DesiredTargets, } - -/// MetaData required to update MetaFiles. -pub struct MetaData { - pub bins: Vec, - pub cvs: metafiles::CrateVersionSource, - pub version_req: String, - pub target: String, -} diff --git a/src/binstall/install.rs b/src/binstall/install.rs index e7f12a36..21691323 100644 --- a/src/binstall/install.rs +++ b/src/binstall/install.rs @@ -1,12 +1,13 @@ use std::{path::PathBuf, process, sync::Arc}; use cargo_toml::Package; +use compact_str::CompactString; use log::{debug, error, info}; use miette::{miette, IntoDiagnostic, Result, WrapErr}; use tokio::{process::Command, task::block_in_place}; use super::{MetaData, Options, Resolution}; -use crate::{bins, fetchers::Fetcher, *}; +use crate::{bins, fetchers::Fetcher, metafiles::binstall_v1::Source, *}; pub async fn install( resolution: Resolution, @@ -22,13 +23,21 @@ pub async fn install( bin_path, bin_files, } => { - let cvs = metafiles::CrateVersionSource { - name, - version: package.version.parse().into_diagnostic()?, - source: metafiles::Source::cratesio_registry(), - }; + let current_version = package.version.parse().into_diagnostic()?; + let target = fetcher.target().into(); - install_from_package(fetcher, opts, cvs, version, bin_path, bin_files).await + install_from_package(fetcher, opts, bin_path, bin_files) + .await + .map(|option| { + option.map(|bins| MetaData { + name: name.into(), + version_req: version.into(), + current_version, + source: Source::cratesio_registry(), + target, + bins, + }) + }) } Resolution::InstallFromSource { package } => { let desired_targets = opts.desired_targets.get().await; @@ -54,11 +63,9 @@ pub async fn install( async fn install_from_package( fetcher: Arc, opts: Arc, - cvs: metafiles::CrateVersionSource, - version: String, bin_path: PathBuf, bin_files: Vec, -) -> Result> { +) -> Result>> { // Download package if opts.dry_run { info!("Dry run, not downloading package"); @@ -108,12 +115,9 @@ async fn install_from_package( } } - Ok(Some(MetaData { - bins: bin_files.into_iter().map(|bin| bin.base_name).collect(), - cvs, - version_req: version, - target: fetcher.target().to_string(), - })) + Ok(Some( + bin_files.into_iter().map(|bin| bin.base_name).collect(), + )) }) } diff --git a/src/main.rs b/src/main.rs index ab652f26..51d86863 100644 --- a/src/main.rs +++ b/src/main.rs @@ -344,11 +344,7 @@ async fn entry(jobserver_client: LazyJobserverClient) -> Result<()> { block_in_place(|| { if !custom_install_path { debug!("Writing .crates.toml"); - metafiles::v1::CratesToml::append( - metadata_vec - .iter() - .map(|metadata| (&metadata.cvs, metadata.bins.clone())), - )?; + metafiles::v1::CratesToml::append(metadata_vec.iter())?; } if opts.no_cleanup { diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index a99e9d89..475e1274 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -23,30 +23,6 @@ pub struct MetaData { pub target: CompactString, pub bins: Vec, } -impl MetaData { - pub fn new(metadata: crate::binstall::MetaData) -> Self { - let crate::binstall::MetaData { - bins, - cvs: - super::CrateVersionSource { - name, - version, - source, - }, - version_req, - target, - } = metadata; - - Self { - name: name.into(), - version_req: version_req.into(), - current_version: version, - source: source.into(), - target: target.into(), - bins, - } - } -} #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SourceType { @@ -70,27 +46,6 @@ impl Source { } } -impl From for Source { - fn from(src: super::Source) -> Self { - use super::Source::*; - - match src { - Git(url) => Source { - source_type: SourceType::Git, - url, - }, - Path(url) => Source { - source_type: SourceType::Path, - url, - }, - Registry(url) => Source { - source_type: SourceType::Registry, - url, - }, - } - } -} - #[derive(Debug, Diagnostic, Error)] pub enum Error { #[error(transparent)] diff --git a/src/metafiles/cvs.rs b/src/metafiles/cvs.rs index 15121c69..f0068328 100644 --- a/src/metafiles/cvs.rs +++ b/src/metafiles/cvs.rs @@ -15,6 +15,16 @@ pub struct CrateVersionSource { pub source: Source, } +impl From<&super::binstall_v1::MetaData> for CrateVersionSource { + fn from(metadata: &super::binstall_v1::MetaData) -> Self { + super::CrateVersionSource { + name: metadata.name.clone().to_string(), + version: metadata.current_version.clone(), + source: Source::from(&metadata.source), + } + } +} + #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] pub enum Source { Git(Url), @@ -28,6 +38,20 @@ impl Source { } } +impl From<&super::binstall_v1::Source> for Source { + fn from(source: &super::binstall_v1::Source) -> Self { + use super::binstall_v1::SourceType::*; + + let url = source.url.clone(); + + match source.source_type { + Git => Self::Git(url), + Path => Self::Path(url), + Registry => Self::Registry(url), + } + } +} + impl FromStr for CrateVersionSource { type Err = CvsParseError; fn from_str(s: &str) -> Result { diff --git a/src/metafiles/v1.rs b/src/metafiles/v1.rs index 374188ba..25ef6c3f 100644 --- a/src/metafiles/v1.rs +++ b/src/metafiles/v1.rs @@ -11,7 +11,7 @@ use miette::Diagnostic; use serde::{Deserialize, Serialize}; use thiserror::Error; -use super::CrateVersionSource; +use super::{binstall_v1::MetaData, CrateVersionSource}; use crate::{cargo_home, create_if_not_exist, FileLock}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -71,13 +71,13 @@ impl CratesToml { iter: Iter, ) -> Result<(), CratesTomlParseError> where - Iter: IntoIterator)>, + Iter: IntoIterator, { let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?; let mut c1 = Self::load_from_reader(&mut *file)?; - for (cvs, bins) in iter { - c1.insert(cvs, bins); + for metadata in iter { + c1.insert(&CrateVersionSource::from(metadata), metadata.bins.clone()); } file.rewind()?; @@ -88,7 +88,7 @@ impl CratesToml { pub fn append<'a, Iter>(iter: Iter) -> Result<(), CratesTomlParseError> where - Iter: IntoIterator)>, + Iter: IntoIterator, { Self::append_to_path(Self::default_path()?, iter) } From 4114b6e7c4bc8a05b5db98d0b3cd8d34a9ad2f80 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 20:19:05 +1000 Subject: [PATCH 18/44] Impl `PartialEq`, `Eq` and `Hash` fo `MetaData` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 475e1274..a9cb446f 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,5 +1,5 @@ use std::{ - fs, + fs, hash, io::{self, Write}, iter::IntoIterator, path::{Path, PathBuf}, @@ -23,6 +23,21 @@ pub struct MetaData { pub target: CompactString, pub bins: Vec, } +impl PartialEq for MetaData { + fn eq(&self, other: &MetaData) -> bool { + self.name == other.name + } +} +impl Eq for MetaData {} + +impl hash::Hash for MetaData { + fn hash(&self, state: &mut H) + where + H: hash::Hasher, + { + self.name.hash(state) + } +} #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SourceType { From 90203dd4671268c9e4fc09ecafd095d803013a07 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 20:33:40 +1000 Subject: [PATCH 19/44] Impl `PartialOrd` and `Ord` for `MetaData` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index a9cb446f..8ce59a4a 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,5 +1,5 @@ use std::{ - fs, hash, + cmp, fs, hash, io::{self, Write}, iter::IntoIterator, path::{Path, PathBuf}, @@ -24,12 +24,24 @@ pub struct MetaData { pub bins: Vec, } impl PartialEq for MetaData { - fn eq(&self, other: &MetaData) -> bool { + fn eq(&self, other: &Self) -> bool { self.name == other.name } } impl Eq for MetaData {} +impl PartialOrd for MetaData { + fn partial_cmp(&self, other: &Self) -> Option { + self.name.partial_cmp(&other.name) + } +} + +impl Ord for MetaData { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.name.cmp(&other.name) + } +} + impl hash::Hash for MetaData { fn hash(&self, state: &mut H) where From 1047a782e5f1a0323f24e132c1b899ba1da0e51f Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 20:36:13 +1000 Subject: [PATCH 20/44] Impl newtype `Records` and methods `load{_from_path}` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 40 ++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 8ce59a4a..e2ecac5d 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,5 +1,7 @@ use std::{ - cmp, fs, hash, + cmp, + collections::BTreeSet, + fs, hash, io::{self, Write}, iter::IntoIterator, path::{Path, PathBuf}, @@ -12,7 +14,7 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; use url::Url; -use crate::{cargo_home, cratesio_url, FileLock}; +use crate::{cargo_home, cratesio_url, create_if_not_exist, FileLock}; #[derive(Debug, Serialize, Deserialize)] pub struct MetaData { @@ -109,3 +111,37 @@ where pub fn default_path() -> Result { Ok(cargo_home()?.join(".binstall-crates.toml")) } + +#[derive(Debug)] +pub struct Records { + file: FileLock, + data: BTreeSet, +} + +impl Records { + fn load_impl(&mut self) -> Result<(), Error> { + let reader = io::BufReader::with_capacity(1024, &mut self.file); + let stream_deser = serde_json::Deserializer::from_reader(reader).into_iter(); + + for res in stream_deser { + let item = res?; + + self.data.replace(item); + } + + Ok(()) + } + + pub fn load_from_path(path: impl AsRef) -> Result { + let mut this = Self { + file: FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?, + data: BTreeSet::default(), + }; + this.load_impl()?; + Ok(this) + } + + pub fn load() -> Result { + Self::load_from_path(default_path()?) + } +} From df3af9727efda5e4e98155a65976896b8ce7dd9c Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:15:21 +1000 Subject: [PATCH 21/44] Impl new fn `Records::overwrite` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index e2ecac5d..ae1c3bc8 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -95,6 +95,10 @@ where .open(path)?, )?; + write_to(file, &mut iter.into_iter()) +} + +pub fn write_to(file: FileLock, iter: &mut dyn Iterator) -> Result<(), Error> { let writer = io::BufWriter::with_capacity(512, file); let mut ser = serde_json::Serializer::new(writer); @@ -115,6 +119,7 @@ pub fn default_path() -> Result { #[derive(Debug)] pub struct Records { file: FileLock, + /// Use BTreeSet to dedup the metadata data: BTreeSet, } @@ -144,4 +149,9 @@ impl Records { pub fn load() -> Result { Self::load_from_path(default_path()?) } + + /// **Warning: This will overwrite all existing records!** + pub fn overwrite(self) -> Result<(), Error> { + write_to(self.file, &mut self.data.into_iter()) + } } From aba2c87d6d8e1271ba47e3b2675113ae4c15fb71 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:22:07 +1000 Subject: [PATCH 22/44] Impl `IntoIterator` for `&Records` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index ae1c3bc8..5d2d6e06 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,9 +1,9 @@ use std::{ cmp, - collections::BTreeSet, + collections::{btree_set, BTreeSet}, fs, hash, io::{self, Write}, - iter::IntoIterator, + iter::{IntoIterator, Iterator}, path::{Path, PathBuf}, }; @@ -155,3 +155,13 @@ impl Records { write_to(self.file, &mut self.data.into_iter()) } } + +impl<'a> IntoIterator for &'a Records { + type Item = &'a MetaData; + + type IntoIter = btree_set::Iter<'a, MetaData>; + + fn into_iter(self) -> Self::IntoIter { + self.data.iter() + } +} From 04f167491aa452bc6d597fe564bfa8e62710eaae Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:25:36 +1000 Subject: [PATCH 23/44] Impl `Borrow` for `MetaData` for `Records` so that we can use `&str` to query `BTreeSet` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 5d2d6e06..edb9c61c 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,5 +1,5 @@ use std::{ - cmp, + borrow, cmp, collections::{btree_set, BTreeSet}, fs, hash, io::{self, Write}, @@ -25,6 +25,13 @@ pub struct MetaData { pub target: CompactString, pub bins: Vec, } + +impl borrow::Borrow for MetaData { + fn borrow(&self) -> &str { + &self.name + } +} + impl PartialEq for MetaData { fn eq(&self, other: &Self) -> bool { self.name == other.name From 8ec33c5b6c058556bbce0a9fb5882c4c31a40581 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:30:58 +1000 Subject: [PATCH 24/44] Impl get, contains, insert, replace, remove & take for `binstall_v1::Records` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index edb9c61c..02fdf167 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -161,6 +161,34 @@ impl Records { pub fn overwrite(self) -> Result<(), Error> { write_to(self.file, &mut self.data.into_iter()) } + + pub fn get(&self, value: impl AsRef) -> Option<&MetaData> { + self.data.get(value.as_ref()) + } + + pub fn contains(&self, value: impl AsRef) -> bool { + self.data.contains(value.as_ref()) + } + + /// Adds a value to the set. + /// If the set did not have an equal element present, true is returned. + /// + /// If the set did have an equal element present, false is returned, and the entry is not updated. See the module-level documentation for more. + pub fn insert(&mut self, value: MetaData) -> bool { + self.data.insert(value) + } + + pub fn replace(&mut self, value: MetaData) -> Option { + self.data.replace(value) + } + + pub fn remove(&mut self, value: impl AsRef) -> bool { + self.data.remove(value.as_ref()) + } + + pub fn take(&mut self, value: impl AsRef) -> Option { + self.data.take(value.as_ref()) + } } impl<'a> IntoIterator for &'a Records { From e6ee49377151dd62681a48afd9ed99432a4e36e4 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:31:51 +1000 Subject: [PATCH 25/44] Impl `binstall_v1::Records::{len, is_empty}` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 02fdf167..b85cbf41 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -189,6 +189,14 @@ impl Records { pub fn take(&mut self, value: impl AsRef) -> Option { self.data.take(value.as_ref()) } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } } impl<'a> IntoIterator for &'a Records { From cef4dfee5c6b4057b832ca2f5641907f77d16158 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:32:20 +1000 Subject: [PATCH 26/44] Update doc of `binstall_v1::Records::insert` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index b85cbf41..63a01ad6 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -172,8 +172,8 @@ impl Records { /// Adds a value to the set. /// If the set did not have an equal element present, true is returned. - /// - /// If the set did have an equal element present, false is returned, and the entry is not updated. See the module-level documentation for more. + /// If the set did have an equal element present, false is returned, + /// and the entry is not updated. pub fn insert(&mut self, value: MetaData) -> bool { self.data.insert(value) } From 671b2fb6f0c5a20b04b2406f08fd47a23892f0ae Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:54:32 +1000 Subject: [PATCH 27/44] Derive `Clone` for `MetaData` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 63a01ad6..0dba0dea 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -16,7 +16,7 @@ use url::Url; use crate::{cargo_home, cratesio_url, create_if_not_exist, FileLock}; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct MetaData { pub name: CompactString, pub version_req: CompactString, From 5987acbf958d0e14a627afc6a3e56d7e8b19f8f6 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 22:55:16 +1000 Subject: [PATCH 28/44] Derive `Clone` for `Source` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 0dba0dea..a2fad509 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -67,7 +67,7 @@ pub enum SourceType { Registry, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct Source { pub source_type: SourceType, pub url: Url, From c9139fec68e486005e4b805ec243df5ad23c1943 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 23:09:52 +1000 Subject: [PATCH 29/44] Add unit tests for `metafiles::binstall_v1` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 73 ++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index a2fad509..26d46f57 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -208,3 +208,76 @@ impl<'a> IntoIterator for &'a Records { self.data.iter() } } + +#[cfg(test)] +mod test { + use super::*; + use crate::target::TARGET; + + use miette::Result; + use tempfile::TempDir; + + macro_rules! assert_records_eq { + ($records:expr, $metadata_set:expr) => { + assert_eq!($records.len(), $metadata_set.len()); + for (record, metadata) in $records.into_iter().zip($metadata_set.iter()) { + assert_eq!(record, metadata); + } + }; + } + + #[test] + fn rw_test() -> Result<()> { + let target = CompactString::from(TARGET); + + let tempdir = TempDir::new().unwrap(); + let path = tempdir.path().join("binstall-tests.json"); + + let metadata_vec = [ + MetaData { + name: "a".into(), + version_req: "*".into(), + current_version: Version::new(0, 1, 0), + source: Source::cratesio_registry(), + target: target.clone(), + bins: vec!["1".into(), "2".into()], + }, + MetaData { + name: "b".into(), + version_req: "0.1.0".into(), + current_version: Version::new(0, 1, 0), + source: Source::cratesio_registry(), + target: target.clone(), + bins: vec!["1".into(), "2".into()], + }, + MetaData { + name: "a".into(), + version_req: "*".into(), + current_version: Version::new(0, 2, 0), + source: Source::cratesio_registry(), + target, + bins: vec!["1".into()], + }, + ]; + + append_to_path(&path, metadata_vec.clone())?; + + let mut iter = metadata_vec.into_iter(); + iter.next().unwrap(); + + let mut metadata_set: BTreeSet<_> = iter.collect(); + + let mut records = Records::load_from_path(&path)?; + assert_records_eq!(&records, &metadata_set); + + records.remove("b"); + assert_eq!(records.len(), metadata_set.len() - 1); + records.overwrite()?; + + metadata_set.remove("b"); + let records = Records::load_from_path(&path)?; + assert_records_eq!(&records, &metadata_set); + + Ok(()) + } +} From d796424199b080ba14e8cd5bad545f80f87e1b06 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 23:13:41 +1000 Subject: [PATCH 30/44] Fix `Records::overwrite`: Update file len Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 26d46f57..8f95e7a1 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -2,7 +2,7 @@ use std::{ borrow, cmp, collections::{btree_set, BTreeSet}, fs, hash, - io::{self, Write}, + io::{self, Seek, Write}, iter::{IntoIterator, Iterator}, path::{Path, PathBuf}, }; @@ -95,17 +95,20 @@ pub fn append_to_path(path: impl AsRef, iter: Iter) -> Result<(), Er where Iter: IntoIterator, { - let file = FileLock::new_exclusive( + let mut file = FileLock::new_exclusive( fs::OpenOptions::new() .create(true) .append(true) .open(path)?, )?; - write_to(file, &mut iter.into_iter()) + write_to(&mut file, &mut iter.into_iter()) } -pub fn write_to(file: FileLock, iter: &mut dyn Iterator) -> Result<(), Error> { +pub fn write_to( + file: &mut FileLock, + iter: &mut dyn Iterator, +) -> Result<(), Error> { let writer = io::BufWriter::with_capacity(512, file); let mut ser = serde_json::Serializer::new(writer); @@ -158,8 +161,14 @@ impl Records { } /// **Warning: This will overwrite all existing records!** - pub fn overwrite(self) -> Result<(), Error> { - write_to(self.file, &mut self.data.into_iter()) + pub fn overwrite(mut self) -> Result<(), Error> { + self.file.rewind()?; + write_to(&mut self.file, &mut self.data.into_iter())?; + + let len = self.file.stream_position()?; + self.file.set_len(len)?; + + Ok(()) } pub fn get(&self, value: impl AsRef) -> Option<&MetaData> { From 7ccbdb2356e549b77580134edbb8ce686b2efbfc Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 23:15:46 +1000 Subject: [PATCH 31/44] Impl `binstall_v1::append` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 8f95e7a1..9a253f83 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -105,6 +105,13 @@ where write_to(&mut file, &mut iter.into_iter()) } +pub fn append(iter: Iter) -> Result<(), Error> +where + Iter: IntoIterator, +{ + append_to_path(default_path()?, iter) +} + pub fn write_to( file: &mut FileLock, iter: &mut dyn Iterator, From 2490c08840b9616fe884c6860ff0604389d4d633 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Wed, 27 Jul 2022 23:15:54 +1000 Subject: [PATCH 32/44] Write to `.binstall-crates.toml` in `entry` Signed-off-by: Jiahao XU --- src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.rs b/src/main.rs index 51d86863..7fb17bb1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -345,6 +345,9 @@ async fn entry(jobserver_client: LazyJobserverClient) -> Result<()> { if !custom_install_path { debug!("Writing .crates.toml"); metafiles::v1::CratesToml::append(metadata_vec.iter())?; + + debug!("Writing .binstall-crates.toml"); + metafiles::binstall_v1::append(metadata_vec)?; } if opts.no_cleanup { From 3cd9866d322554c3661d938fc840e9ccb91096a5 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 01:24:16 +1000 Subject: [PATCH 33/44] Use `unwrap` in unit test for `binstall_v1` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 9a253f83..2ed1d6c7 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -230,7 +230,6 @@ mod test { use super::*; use crate::target::TARGET; - use miette::Result; use tempfile::TempDir; macro_rules! assert_records_eq { @@ -243,7 +242,7 @@ mod test { } #[test] - fn rw_test() -> Result<()> { + fn rw_test() { let target = CompactString::from(TARGET); let tempdir = TempDir::new().unwrap(); @@ -276,24 +275,22 @@ mod test { }, ]; - append_to_path(&path, metadata_vec.clone())?; + append_to_path(&path, metadata_vec.clone()).unwrap(); let mut iter = metadata_vec.into_iter(); iter.next().unwrap(); let mut metadata_set: BTreeSet<_> = iter.collect(); - let mut records = Records::load_from_path(&path)?; + let mut records = Records::load_from_path(&path).unwrap(); assert_records_eq!(&records, &metadata_set); records.remove("b"); assert_eq!(records.len(), metadata_set.len() - 1); - records.overwrite()?; + records.overwrite().unwrap(); metadata_set.remove("b"); - let records = Records::load_from_path(&path)?; + let records = Records::load_from_path(&path).unwrap(); assert_records_eq!(&records, &metadata_set); - - Ok(()) } } From 3f72e9b81ac1a1dcbe900163957ad28cffcffd81 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 01:45:54 +1000 Subject: [PATCH 34/44] Use `tempfile::NamedTempFile` in unit test of `binstall_v1` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 2ed1d6c7..d20ed712 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -230,7 +230,7 @@ mod test { use super::*; use crate::target::TARGET; - use tempfile::TempDir; + use tempfile::NamedTempFile; macro_rules! assert_records_eq { ($records:expr, $metadata_set:expr) => { @@ -245,8 +245,8 @@ mod test { fn rw_test() { let target = CompactString::from(TARGET); - let tempdir = TempDir::new().unwrap(); - let path = tempdir.path().join("binstall-tests.json"); + let named_tempfile = NamedTempFile::new().unwrap(); + let path = named_tempfile.path(); let metadata_vec = [ MetaData { From 96d90c0376aba052b719fd9c2a5d9b7f061f38d5 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 18:06:39 +1000 Subject: [PATCH 35/44] Fix `append_to_path` on windows: Use `create_if_not_exist` to open the file. Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index d20ed712..5bb0e61c 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -95,12 +95,9 @@ pub fn append_to_path(path: impl AsRef, iter: Iter) -> Result<(), Er where Iter: IntoIterator, { - let mut file = FileLock::new_exclusive( - fs::OpenOptions::new() - .create(true) - .append(true) - .open(path)?, - )?; + let mut file = FileLock::new_exclusive(create_if_not_exist(path.as_ref())?)?; + // Move the cursor to EOF + file.seek(io::SeekFrom::End(0))?; write_to(&mut file, &mut iter.into_iter()) } From 73af5b2824002efdb9245a99304dd3d6a66ba7a1 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 18:09:38 +1000 Subject: [PATCH 36/44] Test appending behavior of `append_to_path` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 5bb0e61c..4681dd0f 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -267,7 +267,7 @@ mod test { version_req: "*".into(), current_version: Version::new(0, 2, 0), source: Source::cratesio_registry(), - target, + target: target.clone(), bins: vec!["1".into()], }, ]; @@ -289,5 +289,21 @@ mod test { metadata_set.remove("b"); let records = Records::load_from_path(&path).unwrap(); assert_records_eq!(&records, &metadata_set); + // Drop the exclusive file lock + drop(records); + + let new_metadata = MetaData { + name: "b".into(), + version_req: "0.1.0".into(), + current_version: Version::new(0, 1, 1), + source: Source::cratesio_registry(), + target, + bins: vec!["1".into(), "2".into()], + }; + append_to_path(&path, [new_metadata.clone()]).unwrap(); + metadata_set.insert(new_metadata); + + let records = Records::load_from_path(&path).unwrap(); + assert_records_eq!(&records, &metadata_set); } } From 00fb2528af878dfb6a7d83eb8d21359c1a996623 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 18:09:53 +1000 Subject: [PATCH 37/44] Rm unused import `std::fs` in `binstall_v1` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 4681dd0f..f76f3ef6 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,7 +1,7 @@ use std::{ borrow, cmp, collections::{btree_set, BTreeSet}, - fs, hash, + hash, io::{self, Seek, Write}, iter::{IntoIterator, Iterator}, path::{Path, PathBuf}, From 98556cb2af95665a7dd2b610b0165432513bc456 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 19:21:11 +1000 Subject: [PATCH 38/44] Add field `MetaData::other` for forward compatibility Signed-off-by: Jiahao XU --- src/binstall/install.rs | 1 + src/metafiles/binstall_v1.rs | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/binstall/install.rs b/src/binstall/install.rs index 21691323..365dd0de 100644 --- a/src/binstall/install.rs +++ b/src/binstall/install.rs @@ -36,6 +36,7 @@ pub async fn install( source: Source::cratesio_registry(), target, bins, + other: Default::default(), }) }) } diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index f76f3ef6..35c3901a 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,6 +1,6 @@ use std::{ borrow, cmp, - collections::{btree_set, BTreeSet}, + collections::{btree_set, BTreeMap, BTreeSet}, hash, io::{self, Seek, Write}, iter::{IntoIterator, Iterator}, @@ -24,6 +24,11 @@ pub struct MetaData { pub source: Source, pub target: CompactString, pub bins: Vec, + + /// Forwards compatibility. Unknown keys from future versions of Cargo + /// will be stored here and retained when the file is saved. + #[serde(flatten)] + pub other: BTreeMap, } impl borrow::Borrow for MetaData { @@ -253,6 +258,7 @@ mod test { source: Source::cratesio_registry(), target: target.clone(), bins: vec!["1".into(), "2".into()], + other: Default::default(), }, MetaData { name: "b".into(), @@ -261,6 +267,7 @@ mod test { source: Source::cratesio_registry(), target: target.clone(), bins: vec!["1".into(), "2".into()], + other: Default::default(), }, MetaData { name: "a".into(), @@ -269,6 +276,7 @@ mod test { source: Source::cratesio_registry(), target: target.clone(), bins: vec!["1".into()], + other: Default::default(), }, ]; @@ -299,6 +307,7 @@ mod test { source: Source::cratesio_registry(), target, bins: vec!["1".into(), "2".into()], + other: Default::default(), }; append_to_path(&path, [new_metadata.clone()]).unwrap(); metadata_set.insert(new_metadata); From 4ad8e4f46e040e914fbf98f09922614973508a79 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 19:43:58 +1000 Subject: [PATCH 39/44] Add new dep serde-tuple-vec-map v1.0.1 Signed-off-by: Jiahao XU --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + 2 files changed, 11 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 55b7e882..a90b580e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,7 @@ dependencies = [ "scopeguard", "semver", "serde", + "serde-tuple-vec-map", "serde_json", "simplelog", "strum", @@ -1389,6 +1390,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-tuple-vec-map" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a04d0ebe0de77d7d445bb729a895dcb0a288854b267ca85f030ce51cdc578c82" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.140" diff --git a/Cargo.toml b/Cargo.toml index 8907e68f..eb99d341 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ reqwest = { version = "0.11.11", features = ["stream"], default-features = false scopeguard = "1.1.0" semver = { version = "1.0.12", features = ["serde"] } serde = { version = "1.0.140", features = ["derive"] } +serde-tuple-vec-map = "1.0.1" serde_json = "1.0.82" simplelog = "0.12.0" strum = "0.24.1" From c5db0fefb5b2d3b1e872fb00c56186fcb2306f5d Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 19:45:47 +1000 Subject: [PATCH 40/44] Use `Vec` for field `MetaData::other` by using `tuple_vec_map` to make the `Vec<(CompactString, serde_json::Value)>` appears as a map. This is OK because we don't access that part anyway and only add such field to avoid losing any information when deserialize, modify it and serialize it again to overwrite existing metafile. Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 35c3901a..cebdb722 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,6 +1,6 @@ use std::{ borrow, cmp, - collections::{btree_set, BTreeMap, BTreeSet}, + collections::{btree_set, BTreeSet}, hash, io::{self, Seek, Write}, iter::{IntoIterator, Iterator}, @@ -27,8 +27,8 @@ pub struct MetaData { /// Forwards compatibility. Unknown keys from future versions of Cargo /// will be stored here and retained when the file is saved. - #[serde(flatten)] - pub other: BTreeMap, + #[serde(flatten, with = "tuple_vec_map")] + pub other: Vec<(CompactString, serde_json::Value)>, } impl borrow::Borrow for MetaData { From 0d8b865ba4889a5e21e3972e7853858cf3bfcbc2 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 19:48:45 +1000 Subject: [PATCH 41/44] Update `binstall_v1::default_path` and fix typo in it The postfix should be `.json`, not `.toml`. Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index cebdb722..2cfc58f8 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -1,7 +1,7 @@ use std::{ borrow, cmp, collections::{btree_set, BTreeSet}, - hash, + fs, hash, io::{self, Seek, Write}, iter::{IntoIterator, Iterator}, path::{Path, PathBuf}, @@ -132,7 +132,11 @@ pub fn write_to( } pub fn default_path() -> Result { - Ok(cargo_home()?.join(".binstall-crates.toml")) + let dir = cargo_home()?.join("binstall"); + + fs::create_dir_all(&dir)?; + + Ok(dir.join("crates-v1.json")) } #[derive(Debug)] From cb4cffd0ab227c5f54b94f490942b5e30b2ddac7 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 20:33:24 +1000 Subject: [PATCH 42/44] Fix confusing comment for field `MetaData::other` Signed-off-by: Jiahao XU --- src/metafiles/binstall_v1.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/metafiles/binstall_v1.rs b/src/metafiles/binstall_v1.rs index 2cfc58f8..2d429ec2 100644 --- a/src/metafiles/binstall_v1.rs +++ b/src/metafiles/binstall_v1.rs @@ -25,8 +25,10 @@ pub struct MetaData { pub target: CompactString, pub bins: Vec, - /// Forwards compatibility. Unknown keys from future versions of Cargo + /// Forwards compatibility. Unknown keys from future versions /// will be stored here and retained when the file is saved. + /// + /// We use an `Vec` here since it is never accessed in Rust. #[serde(flatten, with = "tuple_vec_map")] pub other: Vec<(CompactString, serde_json::Value)>, } From 241b763477a2bf22840ce02abd10b242612559b5 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 23:13:47 +1000 Subject: [PATCH 43/44] Use `CompactString` for field `CrateVersionSource::name` Signed-off-by: Jiahao XU --- src/metafiles/cvs.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/metafiles/cvs.rs b/src/metafiles/cvs.rs index f0068328..f8bd7777 100644 --- a/src/metafiles/cvs.rs +++ b/src/metafiles/cvs.rs @@ -1,5 +1,6 @@ use std::{borrow::Cow, fmt, str::FromStr}; +use compact_str::CompactString; use miette::Diagnostic; use semver::Version; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -10,7 +11,7 @@ use crate::cratesio_url; #[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq)] pub struct CrateVersionSource { - pub name: String, + pub name: CompactString, pub version: Version, pub source: Source, } @@ -18,7 +19,7 @@ pub struct CrateVersionSource { impl From<&super::binstall_v1::MetaData> for CrateVersionSource { fn from(metadata: &super::binstall_v1::MetaData) -> Self { super::CrateVersionSource { - name: metadata.name.clone().to_string(), + name: metadata.name.clone(), version: metadata.current_version.clone(), source: Source::from(&metadata.source), } @@ -75,7 +76,7 @@ impl FromStr for CrateVersionSource { _ => return Err(CvsParseError::BadSource), }; Ok(Self { - name: name.to_string(), + name: name.into(), version, source, }) From 6c6d0fe9c229e61cb2a087a7a463167b1a35bc9a Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 28 Jul 2022 23:15:26 +1000 Subject: [PATCH 44/44] Update logging for `binstall_v1::append` Signed-off-by: Jiahao XU --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 7fb17bb1..307e66ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -346,7 +346,7 @@ async fn entry(jobserver_client: LazyJobserverClient) -> Result<()> { debug!("Writing .crates.toml"); metafiles::v1::CratesToml::append(metadata_vec.iter())?; - debug!("Writing .binstall-crates.toml"); + debug!("Writing binstall/crates-v1.json"); metafiles::binstall_v1::append(metadata_vec)?; }