Add non-standard CompressionOptions support

* You ever wonder "what if a squashfs had custom compression options
  that I am forced to emit because the kernel is expecting this?". If this
  is you, then have fun with this addition!
  This allows one to implement compression_options for CompressionAction
  and emit.. whatever you want as the the compression option bytes.
* Change superblock struct in FilesystemWriter As Soon As Possible to make
  sure everything can just use those settings.
* Update tests to be un-padded for custom compression, easier to see
  the diff's with biodiff and such.
* Add --no-compression-options to `add` and `replace`

CHANGELOG.md

@@ -12,7 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Increase speed of internal `HashMap`s, by switching to `xxhash` and just using the `inode` as the key in other places.
 - Changed `SuperBlock::Flags` to be public.
 
+- Add non-standard CompressionOptions support ([#584](https://github.com/wcampbell0x2a/backhand/pull/584))
+ - Add `CompressionAction::compression_options` to override the default compression options emitted during writing.
+ - Add `FilesystemWriter::set_emit_compression_options`
+
 ### `backhand-cli`
+- Add `--no-compression-options` to `add` and `replace` to remove compression options from image after modification.
 - Add `--pad-len` to `replace` and `add` to control the length of end-of-image padding ([#604](https://github.com/wcampbell0x2a/backhand/pull/604))
 
 ### Dependencies

README.md

@@ -106,16 +106,17 @@ Arguments:
  <FILE_PATH_IN_IMAGE> Path of file once inserted into squashfs
 
 Options:
- -d, --dir Create empty directory
- -f, --file <FILE> Path of file to read, to write into squashfs
- -o, --out <OUT> Squashfs output image [default: added.squashfs]
- --mode <MODE> Override mode read from <FILE>
- --uid <UID> Override uid read from <FILE>
- --gid <GID> Override gid read from <FILE>
- --mtime <MTIME> Override mtime read from <FILE>
- --pad-len <PAD_LEN> Custom KiB padding length
- -h, --help Print help
- -V, --version Print version
+ -d, --dir Create empty directory
+ -f, --file <FILE> Path of file to read, to write into squashfs
+ -o, --out <OUT> Squashfs output image [default: added.squashfs]
+ --mode <MODE> Override mode read from <FILE>
+ --uid <UID> Override uid read from <FILE>
+ --gid <GID> Override gid read from <FILE>
+ --mtime <MTIME> Override mtime read from <FILE>
+ --pad-len <PAD_LEN> Custom KiB padding length
+ --no-compression-options Don't emit compression options
+ -h, --help Print help
+ -V, --version Print version
 ```
 
 ### replace-backhand
@@ -130,10 +131,11 @@ Arguments:
  <FILE_PATH_IN_IMAGE> Path of file replaced in image
 
 Options:
- -o, --out <OUT> Squashfs output image [default: replaced.squashfs]
- --pad-len <PAD_LEN> Custom KiB padding length
- -h, --help Print help
- -V, --version Print version
+ -o, --out <OUT> Squashfs output image [default: replaced.squashfs]
+ --pad-len <PAD_LEN> Custom KiB padding length
+ --no-compression-options Don't emit compression options
+ -h, --help Print help
+ -V, --version Print version
 ```
 
 ## Performance

backhand-cli/src/bin/add.rs

@@ -62,6 +62,10 @@ struct Args {
  /// Custom KiB padding length
  #[clap(long)]
  pad_len: Option<u32>,
+
+ /// Don't emit compression options
+ #[clap(long)]
+ no_compression_options: bool,
 }
 
 fn main() -> ExitCode {
@@ -111,6 +115,10 @@ fn main() -> ExitCode {
  filesystem.set_kib_padding(pad_len)
  }
 
+ if args.no_compression_options {
+ filesystem.set_emit_compression_options(false);
+ }
+
  // write new file
  let output = File::create(&args.out).unwrap();
  if let Err(e) = filesystem.write(output) {

backhand-cli/src/bin/replace.rs

@@ -39,6 +39,10 @@ struct Args {
  /// Custom KiB padding length
  #[clap(long)]
  pad_len: Option<u32>,
+
+ /// Don't emit compression options
+ #[clap(long)]
+ no_compression_options: bool,
 }
 
 fn main() -> ExitCode {
@@ -61,6 +65,9 @@ fn main() -> ExitCode {
  if let Some(pad_len) = args.pad_len {
  filesystem.set_kib_padding(pad_len)
  }
+ if args.no_compression_options {
+ filesystem.set_emit_compression_options(false);
+ }
 
  // write new file
  let output = File::create(&args.out).unwrap();

backhand-test/tests/add.rs

@@ -24,7 +24,7 @@ fn test_add() {
  hash: "6195e4d8d14c63dffa9691d36efa1eda2ee975b476bb95d4a0b59638fd9973cb".to_string(),
  url: format!("https://wcampbell.dev/squashfs/testing/test_05/{FILE_NAME}"),
  }];
- const TEST_PATH: &str = "test-assets/test_05";
+ const TEST_PATH: &str = "test-assets/test_01";
 
  test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap();
  let image_path = format!("{TEST_PATH}/{FILE_NAME}");
@@ -124,3 +124,75 @@ dr----x--t 2/4242 42 1970-01-01 00:01 squashfs-root/test
  assert_eq!(std::str::from_utf8(&output.stdout).unwrap(), expected);
  }
 }
+
+#[test]
+#[cfg(feature = "xz")]
+fn test_dont_emit_compression_options() {
+ use std::fs::File;
+ use std::io::Write;
+ use std::os::unix::prelude::PermissionsExt;
+
+ use backhand::DEFAULT_BLOCK_SIZE;
+ use nix::sys::stat::utimes;
+ use nix::sys::time::TimeVal;
+
+ const FILE_NAME: &str = "out.squashfs";
+ let asset_defs = [TestAssetDef {
+ filename: FILE_NAME.to_string(),
+ hash: "debe0986658b276be78c3836779d20464a03d9ba0a40903e6e8e947e434f4d67".to_string(),
+ url: format!("https://wcampbell.dev/squashfs/testing/test_08/{FILE_NAME}"),
+ }];
+ const TEST_PATH: &str = "test-assets/test_add_compression_options";
+
+ test_assets::download_test_files(&asset_defs, TEST_PATH, true).unwrap();
+ let image_path = format!("{TEST_PATH}/{FILE_NAME}");
+ let tmp_dir = tempdir().unwrap();
+
+ let mut file = File::create(tmp_dir.path().join("file").to_str().unwrap()).unwrap();
+ file.write_all(b"nice").unwrap();
+
+ // with compression option
+ let out_image = tmp_dir.path().join("out-comp-options").display().to_string();
+ let cmd = common::get_base_command("add-backhand")
+ .env("RUST_LOG", "none")
+ .args([
+ &image_path,
+ "/new",
+ "--file",
+ tmp_dir.path().join("file").to_str().unwrap(),
+ "-o",
+ &out_image,
+ "--no-compression-options",
+ ])
+ .unwrap();
+ cmd.assert().code(0);
+
+ let cmd = common::get_base_command("unsquashfs-backhand")
+ .env("RUST_LOG", "none")
+ .args(["-s", "--quiet", &out_image])
+ .unwrap();
+ let stdout = std::str::from_utf8(&cmd.stdout).unwrap();
+ stdout.contains("Compression Options: None");
+
+ // with no compression option
+ let out_image = tmp_dir.path().join("out-comp-options").display().to_string();
+ let cmd = common::get_base_command("add-backhand")
+ .env("RUST_LOG", "none")
+ .args([
+ &image_path,
+ "/new",
+ "--file",
+ tmp_dir.path().join("file").to_str().unwrap(),
+ "-o",
+ &out_image,
+ ])
+ .unwrap();
+ cmd.assert().code(0);
+
+ let cmd = common::get_base_command("unsquashfs-backhand")
+ .env("RUST_LOG", "none")
+ .args(["-s", "--quiet", &out_image])
+ .unwrap();
+ let stdout = std::str::from_utf8(&cmd.stdout).unwrap();
+ stdout.contains("Compression Options: Some");
+}

backhand-test/tests/non_standard.rs

@@ -22,6 +22,7 @@ fn full_test(
  test_path: &str,
  offset: u64,
  kind: &Kind,
+ pad: Option<u32>,
 ) {
  test_assets::download_test_files(assets_defs, test_path, true).unwrap();
  let og_path = format!("{test_path}/{filepath}");
@@ -36,6 +37,9 @@ fn full_test(
  )
  .unwrap();
  let mut new_filesystem = FilesystemWriter::from_fs_reader(&og_filesystem).unwrap();
+ if let Some(pad) = pad {
+ new_filesystem.set_kib_padding(pad);
+ }
 
  // Test Debug is impl'ed properly on FilesystemWriter
  let _ = format!("{new_filesystem:#02x?}");
@@ -72,7 +76,14 @@ fn test_non_standard_be_v4_0() {
  .to_string(),
  }];
  const TEST_PATH: &str = "test-assets/non_standard_be_v4_0";
- full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, &Kind::from_const(kind::BE_V4_0).unwrap());
+ full_test(
+ &asset_defs,
+ FILE_NAME,
+ TEST_PATH,
+ 0,
+ &Kind::from_const(kind::BE_V4_0).unwrap(),
+ None,
+ );
 
  // test custom kind "builder-lite"
  let _kind = Kind::new(&DefaultCompressor)
@@ -91,12 +102,21 @@ fn test_non_standard_be_v4_1() {
  .to_string(),
  }];
  const TEST_PATH: &str = "test-assets/non_standard_be_v4_1";
- full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, &Kind::from_const(kind::BE_V4_0).unwrap());
+ full_test(
+ &asset_defs,
+ FILE_NAME,
+ TEST_PATH,
+ 0,
+ &Kind::from_const(kind::BE_V4_0).unwrap(),
+ None,
+ );
 }
 
 #[test]
 #[cfg(feature = "gzip")]
 fn test_custom_compressor() {
+ use backhand::SuperBlock;
+
  const FILE_NAME: &str = "squashfs_v4.nopad.unblob.bin";
  let asset_defs = [TestAssetDef {
  filename: FILE_NAME.to_string(),
@@ -138,10 +158,19 @@ fn test_custom_compressor() {
  ) -> Result<Vec<u8>, BackhandError> {
  DefaultCompressor.compress(bytes, fc, block_size)
  }
+
+ fn compression_options(
+ &self,
+ _superblock: &mut SuperBlock,
+ _kind: &Kind,
+ _fs_compressor: FilesystemCompressor,
+ ) -> Result<Vec<u8>, BackhandError> {
+ DefaultCompressor.compression_options(_superblock, _kind, _fs_compressor)
+ }
  }
 
  let kind = Kind::new_with_const(&CustomCompressor, kind::BE_V4_0);
 
  const TEST_PATH: &str = "test-assets/custom_compressor";
- full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, &kind);
+ full_test(&asset_defs, FILE_NAME, TEST_PATH, 0, &kind, Some(0));
 }

backhand/src/compressor.rs

@@ -1,19 +1,24 @@
 //! Types of supported compression algorithms
 
-use std::io::{Cursor, Read};
+use std::io::{Cursor, Read, Write};
 
 use deku::prelude::*;
 #[cfg(feature = "gzip")]
 use flate2::read::ZlibEncoder;
 #[cfg(feature = "gzip")]
 use flate2::Compression;
+use tracing::trace;
 #[cfg(feature = "xz")]
 use xz2::read::{XzDecoder, XzEncoder};
 #[cfg(feature = "xz")]
 use xz2::stream::{Check, Filters, LzmaOptions, MtStreamBuilder};
 
 use crate::error::BackhandError;
 use crate::filesystem::writer::{CompressionExtra, FilesystemCompressor};
+use crate::kind::Kind;
+use crate::metadata::MetadataWriter;
+use crate::squashfs::Flags;
+use crate::SuperBlock;
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, DekuRead, DekuWrite, Default)]
 #[deku(endian = "endian", ctx = "endian: deku::ctx::Endian")]
@@ -175,6 +180,25 @@ pub trait CompressionAction {
  fc: FilesystemCompressor,
  block_size: u32,
  ) -> Result<Vec<u8>, BackhandError>;
+
+ /// Compression Options for non-default compression specific options
+ ///
+ /// This function is called when calling [FilesystemWriter::write](crate::FilesystemWriter::write), and the returned bytes are the
+ /// section right after the SuperBlock.
+ ///
+ /// # Arguments
+ /// * `superblock` - Mutatable squashfs superblock info that will be written to disk after
+ /// this function is called. The fields `inode_count`, `block_size`,
+ /// `block_log` and `mod_time` *will* be set to `FilesystemWriter` options and can be trusted
+ /// in this function.
+ /// * `kind` - Kind information
+ /// * `fs_compressor` - Compression Options
+ fn compression_options(
+ &self,
+ superblock: &mut SuperBlock,
+ kind: &Kind,
+ fs_compressor: FilesystemCompressor,
+ ) -> Result<Vec<u8>, BackhandError>;
 }
 
 /// Default compressor that handles the compression features that are enabled
@@ -228,6 +252,7 @@ impl CompressionAction for DefaultCompressor {
  Ok(())
  }
 
+ /// Using the current compressor from the superblock, compress bytes
  fn compress(
  &self,
  bytes: &[u8],
@@ -336,4 +361,47 @@ impl CompressionAction for DefaultCompressor {
  _ => Err(BackhandError::UnsupportedCompression(fc.id)),
  }
  }
+
+ /// Using the current compressor options, create compression options
+ fn compression_options(
+ &self,
+ superblock: &mut SuperBlock,
+ kind: &Kind,
+ fs_compressor: FilesystemCompressor,
+ ) -> Result<Vec<u8>, BackhandError> {
+ let mut w = Cursor::new(vec![]);
+
+ // Write compression options, if any
+ if let Some(options) = &fs_compressor.options {
+ trace!("writing compression options");
+ superblock.flags |= Flags::CompressorOptionsArePresent as u16;
+ let mut compression_opt_buf_out = vec![];
+ let mut writer = Writer::new(&mut compression_opt_buf_out);
+ match options {
+ CompressionOptions::Gzip(gzip) => {
+ gzip.to_writer(&mut writer, kind.inner.type_endian)?
+ }
+ CompressionOptions::Lz4(lz4) => {
+ lz4.to_writer(&mut writer, kind.inner.type_endian)?
+ }
+ CompressionOptions::Zstd(zstd) => {
+ zstd.to_writer(&mut writer, kind.inner.type_endian)?
+ }
+ CompressionOptions::Xz(xz) => xz.to_writer(&mut writer, kind.inner.type_endian)?,
+ CompressionOptions::Lzo(lzo) => {
+ lzo.to_writer(&mut writer, kind.inner.type_endian)?
+ }
+ CompressionOptions::Lzma => {}
+ }
+ let mut metadata = MetadataWriter::new(
+ fs_compressor,
+ superblock.block_size,
+ Kind { inner: kind.inner.clone() },
+ );
+ metadata.write_all(&compression_opt_buf_out)?;
+ metadata.finalize(&mut w)?;
+ }
+
+ Ok(w.into_inner())
+ }
 }