Merge pull request #65 from PLSysSec/rm-cargo-download

🗑️ remove dependence on cargo download (closes #64)

Cargo.toml

@@ -52,6 +52,7 @@ ra_ap_project_model = "0.0.185"
 ra_ap_syntax = "0.0.185"
 ra_ap_vfs = "0.0.185"
 ra_ap_cfg = "0.0.185"
+regex = "1.10.6"
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 serde_with.workspace = true
@@ -73,4 +74,4 @@ serde = { version = "1.0.192", features = ["derive"] }
 anyhow = "1.0.75"
 env_logger = "0.10.1"
 home = "0.5.5"
-serde_with = "3.4.0"
+serde_with = "3.4.0"

Makefile

@@ -4,10 +4,7 @@
 SCAN_ALL := cargo run --release --bin scan_all --
 UPDATE_TEST_CRATES_CSV := ./scripts/update_test_crates_csv.py
 
-dependencies:
- - cargo install cargo-download
-
-install: dependencies
+install:
  cargo build && cargo build --release
 
 checks:

README.md

@@ -7,11 +7,8 @@ Cargo Scan is a tool for auditing Rust crates.
 ## Installation
 
 1. Clone this repository.
-2. Make sure you have [Rust](https://www.rust-lang.org/tools/install).
-3. Run `rustup update` to ensure you have the latest version of Rust.
-4. Run `make install`.
-
-This installs [cargo-download](https://crates.io/crates/cargo-download) and builds the Rust source.
+2. Run `rustup update` to ensure you have the latest version of Rust (or install it via the [official website]((https://www.rust-lang.org/tools/install))).
+3. Run `cargo build`.
 
 Known working builds:
 

src/bin/scan_all.rs

@@ -5,6 +5,7 @@
 //!
 //! See README for current usage information.
 
+use cargo_scan::download_crate;
 use cargo_scan::effect::EffectInstance;
 use cargo_scan::scan_stats::{self, CrateStats};
 use cargo_scan::util;
@@ -15,7 +16,6 @@ use std::collections::HashMap;
 use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
-use std::process::Command;
 use std::sync::mpsc;
 use threadpool::ThreadPool;
 
@@ -38,7 +38,7 @@ const RESULTS_PATTERNS_SUFFIX: &str = "_patterns.csv";
 const RESULTS_METADATA_SUFFIX: &str = "_metadata.csv";
 
 // Whether to remove and re-download old downloaded packages
-const UPDATE_DOWNLOADS: bool = false;
+const UPDATE_DOWNLOADS: bool = true;
 
 /*
  CLI
@@ -83,21 +83,15 @@ fn crate_stats(
  let output_dir = download_loc.join(Path::new(crt));
 
  if !test_run {
- if UPDATE_DOWNLOADS {
+ if UPDATE_DOWNLOADS && output_dir.is_dir() {
  fs::remove_dir_all(&output_dir).expect("failed to remove old dir");
  }
 
  if !output_dir.is_dir() {
- info!("Downloading {} to: {:?}", crt, output_dir);
-
- let _output = Command::new("cargo")
- .arg("download")
- .arg("-x")
- .arg(crt)
- .arg("-o")
- .arg(&output_dir)
- .output()
- .expect("failed to run cargo download");
+ info!("Downloading {} to: {}", crt, output_dir.to_string_lossy());
+
+ download_crate::download_latest_crate_version(crt, CRATES_DIR)
+ .expect("failed to download crate");
  }
  }
 

src/download_crate.rs

@@ -1,13 +1,19 @@
 use std::fs::{create_dir_all, remove_file, write, File};
 use std::path::PathBuf;
+use std::process::Command;
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use cargo_lock::Package;
 use curl::easy::Easy;
 use flate2::read::GzDecoder;
 use log::info;
+use regex::Regex;
 use tar::Archive;
 
+// Regexes to match crate names and versions
+const CRATE_NAME_REGEX: &str = r"[a-zA-Z0-9_-]+";
+const SEMVER_REGEX: &str = r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?";
+
 fn get_crates_io_url(package_name: &str, package_version: &str) -> String {
  format!(
  "https://crates.io/api/v1/crates/{}/{}/download",
@@ -86,6 +92,54 @@ pub fn download_crate_from_info(
  download_crate(&url, package_name, package_version, download_dir)
 }
 
+/// Get the latest version of a crate from only the package name.
+pub fn get_latest_version(package_name: &str) -> Result<String> {
+ // Query `cargo search`.
+ let result = Command::new("cargo")
+ .arg("search")
+ .arg(package_name)
+ .arg("--limit")
+ .arg("1")
+ .output()?;
+
+ // Convert the output to a string.
+ let output = String::from_utf8(result.stdout)?;
+
+ // Parse the output. It should contain <crate name> = "<version>"
+ let re_str = format!("^({}) = \"({})\"", CRATE_NAME_REGEX, SEMVER_REGEX);
+ let re = Regex::new(&re_str).unwrap();
+ if let Some(caps) = re.captures(&output) {
+ // Debug
+ // println!("Match found: {:?}", caps);
+
+ let name = &caps[1];
+ let version = &caps[2];
+ if name == package_name {
+ // Debug
+ // println!("Found version: {} for package: {}", version, package_name);
+
+ return Ok(version.to_string());
+ }
+ }
+
+ Err(anyhow!("No match found for package name: {}", package_name))
+}
+
+/// Downloads the latest version of a crate from only the package name
+/// Also, moves the output directory to just use the package name in the final output folder.
+pub fn download_latest_crate_version(
+ package_name: &str,
+ download_dir: &str,
+) -> Result<PathBuf> {
+ let latest_version = get_latest_version(package_name)?;
+ let result = download_crate_from_info(package_name, &latest_version, download_dir)?;
+ let _output = Command::new("mv")
+ .arg(format!("{}/{}-{}", download_dir, package_name, latest_version))
+ .arg(format!("{}/{}", download_dir, package_name))
+ .output()?;
+ Ok(result)
+}
+
 /// Downloads the crate from the `cargo_lock::Package`
 pub fn download_crate_from_package(
  package: &Package,