Rough first attempt

.github/dependabot.yml

@@ -0,0 +1,9 @@
+# Dependabot dependency version checks / updates
+
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"
+      

.github/workflows/rust.yml

@@ -0,0 +1,102 @@
+name: Rust
+
+on:
+  push:
+    branches: [ master ]
+    tags: [ 'v*' ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+            output: cargo-binstall
+            experimental: false
+          - target: x86_64-apple-darwin
+            os: macos-latest
+            output: cargo-binstall
+            experimental: false
+          - target: armv7-unknown-linux-gnueabihf 
+            os: ubuntu-latest
+            output: cargo-binstall
+            experimental: true
+          - target: x86_64-pc-windows-msvc
+            os: windows-latest
+            output: cargo-binstall.exe
+            experimental: true
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: FranzDiebold/github-env-vars-action@v1.2.1
+      
+    - name: Configure toolchain
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        target:  ${{ matrix.target }}
+        override: true
+    
+    - name: Configure caching 
+      uses: actions/cache@v2
+      with:
+        key: ${{ matrix.os }}-${{ matrix.target }}
+        path: |
+          ${{ env.HOME }}/.cargo"
+          target
+
+    - name: Build release
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --target ${{ matrix.target }} --release
+
+    - name: Copy / Rename utility
+      run: |
+        cp target/${{ matrix.target }}/release/${{ matrix.output }} ${{ matrix.output }}-${{ matrix.target }}
+        tar -czvf cargo-binstall-${{ matrix.target }}.tgz ${{ matrix.output }}-${{ matrix.target }}
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v1
+      with:
+        name: cargo-binstall-${{ matrix.target }}
+        path: cargo-binstall-${{ matrix.target }}
+
+    - name: Upload binary to release
+      if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: cargo-binstall-${{ matrix.target }}.tgz
+        asset_name: cargo-binstall-${{ matrix.target }}.tgz
+        tag: ${{ github.ref }}
+        overwrite: true
+
+
+  release:
+    name: Upload firmware artifacts to release
+    runs-on: ubuntu-latest
+    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
+    steps:
+
+      - name: Create Release
+        uses: actions/create-release@v1
+        id: create_release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          body: Release ${{ github.ref }}
+

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "cargo-binstall"
+repository = "https://github.com/ryankurte/cargo-binutil"
+version = "0.1.0"
+authors = ["ryan <ryan@kurte.nz>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+crates_io_api = "0.6.1"
+cargo_metadata = "0.12.1"
+tinytemplate = "1.1.0"
+tokio = { version = "0.2.24", features = [ "macros" ] }
+log = "0.4.11"
+structopt = "0.3.21"
+simplelog = "0.8.0"
+anyhow = "1.0.35"
+reqwest = "0.10.9"
+tempdir = "0.3.7"
+flate2 = "1.0.19"
+tar = "0.4.30"
+cargo_toml = "0.8.1"
+serde = { version = "1.0.118", features = [ "derive" ] }
+strum_macros = "0.20.1"
+strum = "0.20.0"
+dirs = "3.0.1"

README.md

@@ -0,0 +1,59 @@
+# Cargo B(inary) Install
+
+A helper for distributing / installing pre-built binaries in a pseudo-distributed and maybe-one-day secure manner
+
+
+## Status
+
+TODO
+
+## Getting Started
+
+First you'll need to install `cargo-binstall` either via `cargo install cargo-binstall` (and it'll have to compile, sorry...), or by grabbing a pre-compiled version from the [releases](https://github.com/ryankurte/cargo-binstall/releases) page and putting that on your path. It's like there's a problem we're trying to solve?
+
+If a project supports `binstall` you can then install binaries via `cargo binstall NAME` where `NAME` is the name of the crate. We hope the defaults will work without configuration in some cases, however, different projects have wildly different configurations so some further work may be required to support `binstall` in your project, see [Usage](#Usage) for details.
+
+
+
+## Features
+
+- Manifest discovery
+  - [x] Fetch manifest from crates.io
+  - [ ] Fetch manifest using git
+  - [x] Use local manifest (`--manifest-path`)
+- Package formats
+  - [x] Tgz
+  - [x] Tar
+  - [x] Bin
+- Cryptography
+  - [ ] Package signing
+  - [ ] Package verification
+
+## Usage
+
+Packages are located first by locating or querying for a manifest (to allow configuration of the tool), then by interpolating a templated string to download the required package. Where possible defaults are provided to avoid any need for additional configuration, these can generally be overridden via `[package.metadata]` keys at a project level, or on the command line as required (and for debugging), see `cargo binstall -- help` for details.
+
+
+By default `binstall` will look for pre-built packages at `{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ format }`, where `repo`, `name`, and `version` are those specified in the crate manifest (`Cargo.toml`).
+`target` defaults to your architecture, but can be overridden using the `--target` command line option _if required_, and `format` defaults to `tgz` and can be specified via the `pkg-fmt` key (you may need this if you have sneaky `tgz` files that are actually not gzipped).
+
+To support projects with different binary URLs you can override these via the following mechanisms:
+
+To replace _only_ the the package name, specify (`pkg-name`) under `[package.metadata]`. This is useful if you're using github, and your binary paths mostly match except that output package names differ from your crate name. As an example, the `ryankurte/radio-sx128x` crate produces a `sx128x-util` package, and can be configured using the following:
+
+```
+[package.metadata]
+pkg-name = "sx128x-util"
+```
+
+To replace the entire URL, with all the benefits of interpolation, specify (`pkg-url`) under `[package.metadata]`.
+This lets you customise the URL for completely different paths (or different services!). Using the same example as above, this becomes:
+
+```
+[package.metadata]
+pkg-url = "https://github.com/ryankurte/rust-radio-sx128x/releases/download/v{ version }/sx128x-util-{ target }-v{ version }.tgz"
+```
+
+---
+
+If anything is not working the way you expect, add a `--log-level debug` to see debug information, and feel free to open an issue or PR.

build.rs

@@ -0,0 +1,8 @@
+
+// Fetch build target and define this for the compiler
+fn main() {
+    println!(
+        "cargo:rustc-env=TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

src/main.rs

@@ -0,0 +1,379 @@
+use std::time::Duration;
+use std::path::{PathBuf, Path};
+
+use log::{debug, info, error, LevelFilter};
+use simplelog::{TermLogger, ConfigBuilder, TerminalMode};
+
+use structopt::StructOpt;
+use serde::{Serialize, Deserialize};
+
+use crates_io_api::AsyncClient;
+use cargo_toml::Manifest;
+
+use tempdir::TempDir;
+use flate2::read::GzDecoder;
+use tar::Archive;
+
+use tinytemplate::TinyTemplate;
+
+/// Compiled target triple, used as default for binary fetching
+const TARGET: &'static str = env!("TARGET");
+
+/// Default binary path for use if no path is specified
+const DEFAULT_BIN_PATH: &'static str = "{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.{ format }";
+
+/// Binary format enumeration
+#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(strum_macros::Display, strum_macros::EnumString, strum_macros::EnumVariantNames)]
+#[strum(serialize_all = "snake_case")]
+#[serde(rename_all = "snake_case")]
+pub enum PkgFmt {
+    /// Download format is TAR (uncompressed)
+    Tar,
+    /// Download format is TGZ (TAR + GZip)
+    Tgz,
+    /// Download format is raw / binary
+    Bin,
+}
+
+#[derive(Debug, StructOpt)]
+struct Options {
+    /// Crate name to install
+    #[structopt()]
+    name: String,
+
+    /// Crate version to install
+    #[structopt(long)]
+    version: Option<String>,
+
+    /// Override the package path template.
+    /// If no `metadata.pkg_url` key is set or `--pkg-url` argument provided, this
+    /// defaults to `{ repo }/releases/download/v{ version }/{ name }-{ target }-v{ version }.tgz`
+    #[structopt(long)]
+    pkg_url: Option<String>,
+
+    /// Override format for binary file download.
+    /// Defaults to `tgz`
+    #[structopt(long)]
+    pkg_fmt: Option<PkgFmt>,
+
+    /// Override the package name. 
+    /// This is only useful for diagnostics when using the default `pkg_url`
+    /// as you can otherwise customise this in the path.
+    /// Defaults to the crate name.
+    #[structopt(long)]
+    pkg_name: Option<String>,
+
+    /// Override install path for downloaded binary.
+    /// Defaults to `$HOME/.cargo/bin`
+    #[structopt(long)]
+    install_path: Option<String>,
+
+    /// Override binary target, ignoring compiled version
+    #[structopt(long, default_value = TARGET)]
+    target: String,
+
+    /// Override manifest source.
+    /// This skips searching crates.io for a manifest and uses
+    /// the specified path directly, useful for debugging
+    #[structopt(long)]
+    manifest_path: Option<PathBuf>,
+
+    /// Utility log level
+    #[structopt(long, default_value = "info")]
+    log_level: LevelFilter,
+
+    /// Do not cleanup temporary files on success
+    #[structopt(long)]
+    no_cleanup: bool,
+}
+
+
+/// Metadata for cargo-binstall exposed via cargo.toml
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Meta {
+    /// Path template override for binary downloads
+    pub pkg_url: Option<String>,
+    /// Package name override for binary downloads
+    pub pkg_name: Option<String>,
+    /// Format override for binary downloads
+    pub pkg_fmt: Option<PkgFmt>,
+}
+
+/// Template for constructing download paths
+#[derive(Clone, Debug, Serialize)]
+pub struct Context {
+    name: String,
+    repo: Option<String>,
+    target: String,
+    version: String,
+    format: PkgFmt,
+}
+
+
+#[tokio::main]
+async fn main() -> Result<(), anyhow::Error> {
+
+    // Filter extraneous arg when invoked by cargo
+    // `cargo run -- --help` gives ["target/debug/cargo-binstall", "--help"]
+    // `cargo binstall --help` gives ["/home/ryan/.cargo/bin/cargo-binstall", "binstall", "--help"]
+    let mut args: Vec<String> = std::env::args().collect();
+    if args.len() > 1 && args[1] == "binstall" {
+        args.remove(1);
+    }
+
+    // Load options
+    let opts = Options::from_iter(args.iter());
+
+    // Setup logging
+    let mut log_config = ConfigBuilder::new();
+    log_config.add_filter_ignore("hyper".to_string());
+    log_config.add_filter_ignore("reqwest".to_string());
+    log_config.set_location_level(LevelFilter::Off);
+    TermLogger::init(opts.log_level, log_config.build(), TerminalMode::Mixed).unwrap();
+
+    // Create a temporary directory for downloads etc.
+    let temp_dir = TempDir::new("cargo-binstall")?;
+
+    // Fetch crate via crates.io, git, or use a local manifest path
+    // TODO: work out which of these to do based on `opts.name`
+    let crate_path = match opts.manifest_path {
+        Some(p) => p,
+        None => fetch_crate_cratesio(&opts.name, opts.version.as_deref(), temp_dir.path()).await?,
+    };
+
+    // Read cargo manifest
+    let manifest_path = crate_path.join("Cargo.toml");
+
+    debug!("Reading manifest: {}", manifest_path.to_str().unwrap());
+    let package = match Manifest::<Meta>::from_path_with_metadata(&manifest_path) {
+        Ok(m) => m.package.unwrap(),
+        Err(e) => {
+            error!("Error reading manifest '{}': {:?}", manifest_path.to_str().unwrap(), e);
+            return Err(e.into());
+        },
+    };
+
+    let meta = package.metadata;
+    debug!("Retrieved metadata: {:?}", meta);
+
+    // Select which binary path to use
+    let pkg_url = match (opts.pkg_url, meta.as_ref().map(|m| m.pkg_url.clone() ).flatten()) {
+        (Some(p), _) => {
+            info!("Using package url override: '{}'", p);
+            p
+        },
+        (_, Some(m)) => {
+            info!("Using package url: '{}'", &m);
+            m
+        },
+        _ => {
+            info!("No `pkg-url` key found in Cargo.toml or `--pkg-url` argument provided");
+            info!("Using default url: {}", DEFAULT_BIN_PATH);
+            DEFAULT_BIN_PATH.to_string()
+        },
+    };
+
+    // Select bin format to use
+    let pkg_fmt = match (opts.pkg_fmt, meta.as_ref().map(|m| m.pkg_fmt.clone() ).flatten()) {
+        (Some(o), _) => o,
+        (_, Some(m)) => m.clone(),
+        _ => PkgFmt::Tgz,
+    };
+
+    // Override package name if required
+    let pkg_name = match (&opts.pkg_name, meta.as_ref().map(|m| m.pkg_name.clone() ).flatten()) {
+        (Some(o), _) => o.clone(),
+        (_, Some(m)) => m,
+        _ => opts.name.clone(),
+    };
+
+    // Generate context for interpolation
+    let ctx = Context { 
+        name: pkg_name.to_string(), 
+        repo: package.repository, 
+        target: opts.target.clone(), 
+        version: package.version.clone(),
+        format: pkg_fmt.clone(),
+    };
+
+    debug!("Using context: {:?}", ctx);
+    
+    // Interpolate version / target / etc.
+    let mut tt = TinyTemplate::new();
+    tt.add_template("path", &pkg_url)?;
+    let rendered = tt.render("path", &ctx)?;
+
+    info!("Downloading package from: '{}'", rendered);
+
+    // Download package
+    let pkg_path = temp_dir.path().join(format!("pkg-{}.{}", pkg_name, pkg_fmt));
+    download(&rendered, pkg_path.to_str().unwrap()).await?;
+
+
+    if opts.no_cleanup {
+        // Do not delete temporary directory
+        let _ = temp_dir.into_path();
+    }
+
+    // TODO: check signature
+
+    // Compute install directory
+    let install_path = match get_install_path(opts.install_path) {
+        Some(p) => p,
+        None => {
+            error!("No viable install path found of specified, try `--install-path`");
+            return Err(anyhow::anyhow!("No install path found or specified"));
+        }
+    };
+
+    // Install package
+    info!("Installing to: '{}'", install_path);
+    extract(&pkg_path, pkg_fmt, &install_path)?;
+
+    
+    info!("Installation done!");
+
+    Ok(())
+}
+
+/// Download a file from the provided URL to the provided path
+async fn download<P: AsRef<Path>>(url: &str, path: P) -> Result<(), anyhow::Error> {
+
+    debug!("Downloading from: '{}'", url);
+
+    let resp = reqwest::get(url).await?;
+
+    if !resp.status().is_success() {
+        error!("Download error: {}", resp.status());
+        return Err(anyhow::anyhow!(resp.status()));
+    }
+
+    let bytes = resp.bytes().await?;
+
+    debug!("Download OK, writing to file: '{:?}'", path.as_ref());
+
+    std::fs::write(&path, bytes)?;
+
+    Ok(())
+}
+
+fn extract<S: AsRef<Path>, P: AsRef<Path>>(source: S, fmt: PkgFmt, path: P) -> Result<(), anyhow::Error> {
+    match fmt {
+        PkgFmt::Tar => {
+            // Extract to install dir
+            debug!("Extracting from archive '{:?}' to `{:?}`", source.as_ref(), path.as_ref());
+
+            let dat = std::fs::File::open(source)?;
+            let mut tar = Archive::new(dat);
+
+            tar.unpack(path)?;
+        },
+        PkgFmt::Tgz => {
+            // Extract to install dir
+            debug!("Decompressing from archive '{:?}' to `{:?}`", source.as_ref(), path.as_ref());
+
+            let dat = std::fs::File::open(source)?;
+            let tar = GzDecoder::new(dat);
+            let mut tgz = Archive::new(tar);
+
+            tgz.unpack(path)?;
+        },
+        PkgFmt::Bin => {
+            debug!("Copying data from archive '{:?}' to `{:?}`", source.as_ref(), path.as_ref());
+            // Copy to install dir
+            std::fs::copy(source, path)?;
+        },
+    };
+
+    Ok(())
+}
+
+/// Fetch a crate by name and version from crates.io
+async fn fetch_crate_cratesio(name: &str, version: Option<&str>, temp_dir: &Path) -> Result<PathBuf, anyhow::Error> {
+    // Build crates.io api client and fetch info
+    // TODO: support git-based fetches (whole repo name rather than just crate name)
+    let api_client = AsyncClient::new("cargo-binstall (https://github.com/ryankurte/cargo-binstall)", Duration::from_millis(100))?;
+
+    info!("Fetching information for crate: '{}'", name);
+
+    // Fetch overall crate info
+    let info = match api_client.get_crate(name.as_ref()).await {
+        Ok(i) => i,
+        Err(e) => {
+            error!("Error fetching information for crate {}: {}", name, e);
+            return Err(e.into())
+        }
+    };
+
+    // Use specified or latest version
+    let version_num = match version {
+        Some(v) => v.to_string(),
+        None => info.crate_data.max_version,
+    };
+
+    // Fetch crates.io information for the specified version
+    // TODO: could do a semver match and sort here?
+    let version = match info.versions.iter().find(|v| v.num == version_num) {
+        Some(v) => v,
+        None => {
+            error!("No crates.io information found for crate: '{}' version: '{}'", 
+                    name, version_num);
+            return Err(anyhow::anyhow!("No crate information found"));
+        }
+    };
+
+    info!("Found information for crate version: '{}'", version.num);
+    
+    // Download crate to temporary dir (crates.io or git?)
+    let crate_url = format!("https://crates.io/{}", version.dl_path);
+    let tgz_path = temp_dir.join(format!("{}.tgz", name));
+
+    debug!("Fetching crate from: {}", crate_url);    
+
+    // Download crate
+    download(&crate_url, &tgz_path).await?;
+
+    // Decompress downloaded tgz
+    debug!("Decompressing crate archive");
+    extract(&tgz_path, PkgFmt::Tgz, &temp_dir)?;
+    let crate_path = temp_dir.join(format!("{}-{}", name, version_num));
+
+    // Return crate directory
+    Ok(crate_path)
+}
+
+/// Fetch install path
+/// roughly follows https://doc.rust-lang.org/cargo/commands/cargo-install.html#description
+fn get_install_path(opt: Option<String>) -> Option<String> {
+    // Command line override first first
+    if let Some(p) = opt {
+        return Some(p)
+    }
+
+    // Environmental variables
+    if let Ok(p) = std::env::var("CARGO_INSTALL_ROOT") {
+        return Some(format!("{}/bin", p))
+    }
+    if let Ok(p) = std::env::var("CARGO_HOME") {
+        return Some(format!("{}/bin", p))
+    }
+
+    // Standard $HOME/.cargo/bin
+    if let Some(mut d) = dirs::home_dir() {
+        d.push(".cargo/bin");
+        let p = d.as_path();
+
+        if p.exists() {
+            return Some(p.to_str().unwrap().to_owned());
+        }
+    }
+
+    // Local executable dir if no cargo is found
+    if let Some(d) = dirs::executable_dir() {
+        return Some(d.to_str().unwrap().to_owned());
+    }
+
+    None
+}