From 9e35efbd10a49a396b1304bd70386ac77025235e Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 16:42:28 +1000 Subject: [PATCH 01/12] Add new dep zstd v0.10.0 Signed-off-by: Jiahao XU --- Cargo.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 4db7ace7..6b4d83f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,13 @@ tokio = { version = "1.18.0", features = [ "full" ] } url = "2.2.2" xz2 = "0.1.6" zip = "0.6.2" +# zstd is also depended by zip. +# Since zip 0.6.2 depends on zstd 0.10.0, we also have to use 0.10.0 here, +# otherwise there will be a link conflict. +# +# Enable feature bindgen to generate C bindings. +# Enable feature zstdmt to enable multithreading in libzstd. +zstd = { version = "0.10.0", features = [ "bindgen", "zstdmt" ], default-features = false } [dev-dependencies] env_logger = "0.9.0" From f555c75e23d03bc0d38564b18f8ab8e2e2e0929c Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 16:50:36 +1000 Subject: [PATCH 02/12] Update Cargo.lock Signed-off-by: Jiahao XU --- Cargo.lock | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 232f4789..389cf211 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,6 +84,29 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b" +[[package]] +name = "bindgen" +version = "0.59.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "clap", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -174,6 +197,7 @@ dependencies = [ "url", "xz2", "zip", + "zstd", ] [[package]] @@ -218,6 +242,15 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -244,6 +277,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "clang-sys" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a050e2153c5be08febd6734e29298e844fdb0fa21aeddd63b4eb7baa106c69b" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "2.34.0" @@ -351,6 +395,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -530,6 +580,12 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "h2" version = "0.3.13" @@ -731,12 +787,28 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +[[package]] +name = "libloading" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "lock_api" version = "0.4.7" @@ -785,6 +857,12 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.5.3" @@ -806,6 +884,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -902,6 +990,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1073,6 +1167,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustls" version = "0.20.6" @@ -1196,6 +1296,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1711,6 +1817,17 @@ dependencies = [ "webpki", ] +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1857,6 +1974,7 @@ version = "1.6.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" dependencies = [ + "bindgen", "cc", "libc", ] From 5b6634def6d53f2f439cefcb9bd0005fc474ba32 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 16:50:55 +1000 Subject: [PATCH 03/12] Add new variant `PkgFmt::Tzstd` and update `helpers::extract` Signed-off-by: Jiahao XU --- src/helpers.rs | 20 ++++++++++++++++++++ src/lib.rs | 2 ++ 2 files changed, 22 insertions(+) diff --git a/src/helpers.rs b/src/helpers.rs index 3b6c7505..64647f18 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -9,6 +9,7 @@ use tar::Archive; use tinytemplate::TinyTemplate; use xz2::read::XzDecoder; use zip::read::ZipArchive; +use zstd::stream::Decoder as ZstdDecoder; use crate::Meta; @@ -102,6 +103,25 @@ pub fn extract, P: AsRef>( txz.unpack(path)?; } + PkgFmt::Tzstd => { + // Extract to install dir + debug!( + "Decompressing from tzstd archive '{:?}' to `{:?}`", + source.as_ref(), + path.as_ref() + ); + + let dat = std::fs::File::open(source)?; + + // The error can only come from raw::Decoder::with_dictionary + // as of zstd 0.10.2 and 0.11.2, which is specified + // as &[] by ZstdDecoder::new, thus ZstdDecoder::new + // should not return any error. + let tar = ZstdDecoder::new(dat)?; + let mut txz = Archive::new(tar); + + txz.unpack(path)?; + } PkgFmt::Zip => { // Extract to install dir debug!( diff --git a/src/lib.rs b/src/lib.rs index a5389054..dc7e4569 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,8 @@ pub enum PkgFmt { Tgz, /// Download format is TAR + XZ Txz, + /// Download format is TAR + Zstd + Tzstd, /// Download format is Zip Zip, /// Download format is raw / binary From a59057b9321ab331c467ea830d1d5070b0e9815e Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 16:59:00 +1000 Subject: [PATCH 04/12] Update `README.md`: `Tar.zst` is now supported by `cargo-binstall` Signed-off-by: Jiahao XU --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b72a5d56..57b337df 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ $ binstall \ - Package formats - [x] Tgz - [x] Txz - - [ ] Tar.zst + - [x] Tar.zst - [x] Tar - [x] Zip - [x] Bin From 69a23e477a945615d05e860f90c912b56b309a0e Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 18:19:09 +1000 Subject: [PATCH 05/12] Add workaround `libclang.so` not found error when cross bulding Signed-off-by: Jiahao XU --- Cross.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Cross.toml diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..21e128e9 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,11 @@ +[target.armv7-unknown-linux-gnueabihf] +image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf:edge" + +[target.armv7-unknown-linux-musleabihf] +image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf:edge" + +[target.aarch64-unknown-linux-gnu] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge" + +[target.aarch64-unknown-linux-musl] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:edge" From ba5a81d7923b50da6d79d60290da37f2d2fa4560 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 19:09:26 +1000 Subject: [PATCH 06/12] Pin image version in `Cross.toml` Signed-off-by: Jiahao XU --- Cross.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cross.toml b/Cross.toml index 21e128e9..58846039 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,11 +1,11 @@ [target.armv7-unknown-linux-gnueabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf:edge" +image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.armv7-unknown-linux-musleabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf:edge" +image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.aarch64-unknown-linux-gnu] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:edge" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.aarch64-unknown-linux-musl] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:edge" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" From ca6469ca666e553555990724aa2526f7d6fe9352 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 19:23:20 +1000 Subject: [PATCH 07/12] Fix image ver format in `Cross.toml` Signed-off-by: Jiahao XU --- Cross.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cross.toml b/Cross.toml index 58846039..0587881a 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,11 +1,11 @@ [target.armv7-unknown-linux-gnueabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" +image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.armv7-unknown-linux-musleabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" +image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.aarch64-unknown-linux-gnu] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.aarch64-unknown-linux-musl] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl:sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" From 5f3396a886da70e73186bac4112f7ea0257b908a Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 19:25:44 +1000 Subject: [PATCH 08/12] Rm image specification for musl targets in `Cross.toml` musl targets work fine, no workaound is required. Signed-off-by: Jiahao XU --- Cross.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Cross.toml b/Cross.toml index 0587881a..e220ab60 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,11 +1,5 @@ [target.armv7-unknown-linux-gnueabihf] image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" -[target.armv7-unknown-linux-musleabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" - [target.aarch64-unknown-linux-gnu] image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" - -[target.aarch64-unknown-linux-musl] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" From e0f67973ae7f924c1f08331e85e5ae0e72448f6c Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 19:31:20 +1000 Subject: [PATCH 09/12] Update Cross.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Emil Gardström --- Cross.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cross.toml b/Cross.toml index e220ab60..876285aa 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,5 +1,5 @@ [target.armv7-unknown-linux-gnueabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" +image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf@sha256:379eee7aa254912a803b558fe0742450dd32ca9749bc8b931927dafa2ce6e8d3" [target.aarch64-unknown-linux-gnu] image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" From 66a3e9f416954dc84001a450a60e6e8c5607efa4 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 19:32:02 +1000 Subject: [PATCH 10/12] Add back workaround for musl targets Signed-off-by: Jiahao XU --- Cross.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Cross.toml b/Cross.toml index 876285aa..4bebaaaf 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,5 +1,11 @@ [target.armv7-unknown-linux-gnueabihf] image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf@sha256:379eee7aa254912a803b558fe0742450dd32ca9749bc8b931927dafa2ce6e8d3" +[target.armv7-unknown-linux-musleabihf] +image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf@sha256:379eee7aa254912a803b558fe0742450dd32ca9749bc8b931927dafa2ce6e8d3" + [target.aarch64-unknown-linux-gnu] image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" + +[target.aarch64-unknown-linux-musl] +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" From 393e808951553082032b82f77ba9d9268210c91d Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 19:36:39 +1000 Subject: [PATCH 11/12] Update Cross.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Emil Gardström --- Cross.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cross.toml b/Cross.toml index 4bebaaaf..a384d49c 100644 --- a/Cross.toml +++ b/Cross.toml @@ -2,10 +2,10 @@ image = "ghcr.io/cross-rs/armv7-unknown-linux-gnueabihf@sha256:379eee7aa254912a803b558fe0742450dd32ca9749bc8b931927dafa2ce6e8d3" [target.armv7-unknown-linux-musleabihf] -image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf@sha256:379eee7aa254912a803b558fe0742450dd32ca9749bc8b931927dafa2ce6e8d3" +image = "ghcr.io/cross-rs/armv7-unknown-linux-musleabihf@sha256:00088ebbb98f5d9895c2b6e7f713cd3028331185fe82f60d93efc03bdff8ea65" [target.aarch64-unknown-linux-gnu] image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" [target.aarch64-unknown-linux-musl] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl@sha256:5db2a92c73d1251c5fb9de895928ef2551c08d0ec30b969cf7340ca3dd62ea81" +image = "ghcr.io/cross-rs/aarch64-unknown-linux-musl@sha256:341256c185132fff9265ee7c8d94ba9b4fde3e8cfa8b39047c0e64ba4c1faf9a" From 5d62f7f61511c16186a70d88d8dab4ea0521fcd9 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Tue, 31 May 2022 20:40:39 +1000 Subject: [PATCH 12/12] Rm target aarch64-unknown-linux-gnu & armv7-unknown-linux-gnueabihf Since they are the only two targets that crate `zstd-sys` failed to build. Signed-off-by: Jiahao XU --- .github/workflows/rust.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9ff74d56..3b26bb66 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -34,11 +34,6 @@ jobs: output: cargo-binstall archive: zip use-cross: false - - target: armv7-unknown-linux-gnueabihf - os: ubuntu-20.04 - output: cargo-binstall - archive: tgz - use-cross: true - target: x86_64-pc-windows-msvc os: windows-latest output: cargo-binstall.exe @@ -54,11 +49,6 @@ jobs: output: cargo-binstall archive: tgz use-cross: true - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - output: cargo-binstall - archive: tgz - use-cross: true - target: aarch64-unknown-linux-musl os: ubuntu-latest output: cargo-binstall