From 36b506318549e52f3f8dd1144c9e24a457b0472d Mon Sep 17 00:00:00 2001 From: Casey Rodarmor Date: Mon, 2 Sep 2024 23:44:06 -0700 Subject: [PATCH] Pass `--mmap` to memory-map files for hashing (#6) --- Cargo.lock | 10 ++++++++++ Cargo.toml | 2 +- src/arguments.rs | 29 +++++++++++++++++++++++++++ src/main.rs | 10 ++++++---- src/options.rs | 7 +++++++ src/relative_path.rs | 6 ++++++ src/subcommand.rs | 6 +++--- src/subcommand/create.rs | 11 +++++++---- src/subcommand/verify.rs | 11 +++++++---- tests/create.rs | 25 ++++++++++++++++++++++++ tests/verify.rs | 42 ++++++++++++++++++++++++++++++++++++++++ 11 files changed, 143 insertions(+), 16 deletions(-) create mode 100644 src/arguments.rs create mode 100644 src/options.rs diff --git a/Cargo.lock b/Cargo.lock index 4a77102..f41e695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,6 +128,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", + "memmap2", "serde", ] @@ -378,6 +379,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322" +dependencies = [ + "libc", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index c698a46..4ee7efd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ license = "CC0-1.0" repository = "https://github.com/casey/filepack" [dependencies] -blake3 = { version = "1.5.4", features = ["serde"] } +blake3 = { version = "1.5.4", features = ["mmap", "serde"] } camino = { version = "1.1.9", features = ["serde1"] } clap = { version = "4.5.16", features = ["derive"] } serde = { version = "1.0.209", features = ["derive"] } diff --git a/src/arguments.rs b/src/arguments.rs new file mode 100644 index 0000000..db69d2a --- /dev/null +++ b/src/arguments.rs @@ -0,0 +1,29 @@ +use { + super::*, + clap::builder::{ + styling::{AnsiColor, Effects}, + Styles, + }, +}; + +#[derive(Parser)] +#[command( + version, + styles = Styles::styled() + .header(AnsiColor::Green.on_default() | Effects::BOLD) + .usage(AnsiColor::Green.on_default() | Effects::BOLD) + .literal(AnsiColor::Blue.on_default() | Effects::BOLD) + .placeholder(AnsiColor::Cyan.on_default())) +] +pub(crate) struct Arguments { + #[command(flatten)] + options: Options, + #[command(subcommand)] + subcommand: Subcommand, +} + +impl Arguments { + pub(crate) fn run(self) -> Result { + self.subcommand.run(self.options) + } +} diff --git a/src/main.rs b/src/main.rs index 1838e8b..024f26a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use { self::{ - error::Error, hash::Hash, lint::Lint, list::List, manifest::Manifest, - relative_path::RelativePath, subcommand::Subcommand, + arguments::Arguments, error::Error, hash::Hash, lint::Lint, list::List, manifest::Manifest, + options::Options, relative_path::RelativePath, subcommand::Subcommand, }, blake3::Hasher, camino::{Utf8Component, Utf8Path, Utf8PathBuf}, @@ -14,25 +14,27 @@ use { fmt::{self, Display, Formatter}, fs::{self, File}, io, - path::PathBuf, + path::{Path, PathBuf}, process, str::FromStr, }, walkdir::WalkDir, }; +mod arguments; mod error; mod hash; mod lint; mod list; mod manifest; +mod options; mod relative_path; mod subcommand; type Result = std::result::Result; fn main() { - if let Err(err) = Subcommand::parse().run() { + if let Err(err) = Arguments::parse().run() { eprintln!("error: {err}"); for (i, err) in err.iter_chain().skip(1).enumerate() { diff --git a/src/options.rs b/src/options.rs new file mode 100644 index 0000000..a0113e0 --- /dev/null +++ b/src/options.rs @@ -0,0 +1,7 @@ +use super::*; + +#[derive(Parser)] +pub(crate) struct Options { + #[arg(long, help = "Memory-map files for hashing")] + pub(crate) mmap: bool, +} diff --git a/src/relative_path.rs b/src/relative_path.rs index 5526c19..84ba757 100644 --- a/src/relative_path.rs +++ b/src/relative_path.rs @@ -86,6 +86,12 @@ impl AsRef for RelativePath { } } +impl AsRef for RelativePath { + fn as_ref(&self) -> &Path { + self.0.as_ref() + } +} + impl<'de> Deserialize<'de> for RelativePath { fn deserialize(deserializer: D) -> Result where diff --git a/src/subcommand.rs b/src/subcommand.rs index 6e5d556..c447a10 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -24,10 +24,10 @@ pub(crate) enum Subcommand { } impl Subcommand { - pub(crate) fn run(self) -> Result { + pub(crate) fn run(self, options: Options) -> Result { match self { - Self::Create { root } => create::run(&root), - Self::Verify { root } => verify::run(&root), + Self::Create { root } => create::run(options, &root), + Self::Verify { root } => verify::run(options, &root), } } } diff --git a/src/subcommand/create.rs b/src/subcommand/create.rs index 905c53f..8793459 100644 --- a/src/subcommand/create.rs +++ b/src/subcommand/create.rs @@ -1,6 +1,6 @@ use super::*; -pub(crate) fn run(root: &Utf8Path) -> Result { +pub(crate) fn run(options: Options, root: &Utf8Path) -> Result { let mut files = HashMap::new(); let mut dirs = Vec::new(); @@ -31,11 +31,14 @@ pub(crate) fn run(root: &Utf8Path) -> Result { return Err(error::Symlink { path }.build()); } - let file = File::open(path).context(error::Io { path })?; - let mut hasher = Hasher::new(); - hasher.update_reader(file).context(error::Io { path })?; + if options.mmap { + hasher.update_mmap(path).context(error::Io { path })?; + } else { + let file = File::open(root.join(path)).context(error::Io { path })?; + hasher.update_reader(file).context(error::Io { path })?; + } let relative = path.strip_prefix(root).unwrap(); diff --git a/src/subcommand/verify.rs b/src/subcommand/verify.rs index 98d0734..8b0a438 100644 --- a/src/subcommand/verify.rs +++ b/src/subcommand/verify.rs @@ -1,6 +1,6 @@ use super::*; -pub(crate) fn run(root: &Utf8Path) -> Result { +pub(crate) fn run(options: Options, root: &Utf8Path) -> Result { let source = root.join(Manifest::FILENAME); let json = fs::read_to_string(&source).context(error::Io { @@ -12,11 +12,14 @@ pub(crate) fn run(root: &Utf8Path) -> Result { })?; for (path, &expected) in &manifest.files { - let file = File::open(root.join(path)).context(error::Io { path })?; - let mut hasher = Hasher::new(); - hasher.update_reader(file).context(error::Io { path })?; + if options.mmap { + hasher.update_mmap(path).context(error::Io { path })?; + } else { + let file = File::open(root.join(path)).context(error::Io { path })?; + hasher.update_reader(file).context(error::Io { path })?; + } let actual = Hash::from(hasher.finalize()); diff --git a/tests/create.rs b/tests/create.rs index dca4aba..cedd821 100644 --- a/tests/create.rs +++ b/tests/create.rs @@ -46,6 +46,31 @@ fn single_file() { .success(); } +#[test] +fn single_file_with_mmap() { + let dir = TempDir::new().unwrap(); + + dir.child("foo").touch().unwrap(); + + Command::cargo_bin("filepack") + .unwrap() + .args(["--mmap", "create", "."]) + .current_dir(&dir) + .assert() + .success(); + + dir.child("filepack.json").assert( + r#"{"files":{"foo":"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"}}"#, + ); + + Command::cargo_bin("filepack") + .unwrap() + .args(["--mmap", "verify", "."]) + .current_dir(&dir) + .assert() + .success(); +} + #[test] fn file_in_subdirectory() { let dir = TempDir::new().unwrap(); diff --git a/tests/verify.rs b/tests/verify.rs index 320aa92..5a939f4 100644 --- a/tests/verify.rs +++ b/tests/verify.rs @@ -17,6 +17,48 @@ fn no_files() { .success(); } +#[test] +fn single_file() { + let dir = TempDir::new().unwrap(); + + dir.child("foo").touch().unwrap(); + + dir + .child("filepack.json") + .write_str( + r#"{"files":{"foo":"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"}}"#, + ) + .unwrap(); + + Command::cargo_bin("filepack") + .unwrap() + .args(["verify", "."]) + .current_dir(&dir) + .assert() + .success(); +} + +#[test] +fn single_file_with_mmap() { + let dir = TempDir::new().unwrap(); + + dir.child("foo").touch().unwrap(); + + dir + .child("filepack.json") + .write_str( + r#"{"files":{"foo":"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"}}"#, + ) + .unwrap(); + + Command::cargo_bin("filepack") + .unwrap() + .args(["--mmap", "verify", "."]) + .current_dir(&dir) + .assert() + .success(); +} + #[test] fn extra_fields_are_not_allowed() { let dir = TempDir::new().unwrap();