feat: add datatest_stable to mutation-test

Added the first integration tests that are executed on these projects:
- `move-mutator/tests/move-assets/breakcontinue`
- `move-mutator/tests/move-assets/simple`

The expectation for the integration test is that the generated report by
the `move-mutation-test` will match the pre-generated reports in those
projects that end with `the .exp` suffix.

.config/nextest.toml

@@ -0,0 +1,13 @@
+[test-groups]
+serial-integration = { max-threads = 1 }
+
+[[profile.default.overrides]]
+filter = 'test(test_run_mutation_test::)'
+test-group = 'serial-integration'
+slow-timeout = { period = "120s" }
+threads-required = "num-cpus"
+
+[profile.ci]
+# Do not cancel the test run on the first failure.
+fail-fast = false
+retries = 2

.github/workflows/check-and-lint.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: self-hosted
     name: Basic ci-check for fmt/clippy/check
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install required deps
         run: sudo apt-get install libudev-dev libdw-dev lld
@@ -47,6 +47,3 @@ jobs:
 
       - name: Run clippy
         run: cargo clippy --all-targets -- -D warnings
-
-      - name: Run tests
-        run: cargo test

.github/workflows/run-tests.yml

@@ -0,0 +1,50 @@
+name: Basic test run
+
+on:
+  push:
+    branches: [ main, develop-m3, develop-m4 ]
+    paths:
+      - Cargo.toml
+      - Cargo.lock
+      - move-mutator/**
+      - move-spec-test/**
+      - move-mutation-test/**
+      - .github/workflows/run-tests.yml
+  pull_request:
+    branches: [ main, develop-m3, develop-m4 ]
+    paths:
+      - Cargo.toml
+      - Cargo.lock
+      - move-mutator/**
+      - move-spec-test/**
+      - move-mutation-test/**
+      - .github/workflows/run-tests.yml
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  basic-lint-and-check:
+    runs-on: self-hosted
+    name: Basic ci-check for tests
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install required deps
+        run: sudo apt-get install libudev-dev libdw-dev lld
+
+      - name: Setup Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: stable
+
+      - name: Install nextest
+        uses: taiki-e/install-action@nextest
+
+      - name: Run integration tests in the release mode (due to test duration)
+        run: cargo nextest run -r --profile ci test_run_mutation_test
+
+      - name: Run normal tests in the debug mode
+        run: cargo nextest run --profile ci -E 'not test(test_run_mutation_test)'
+

Cargo.toml

@@ -30,6 +30,7 @@ clap = { version = "4.5", features = ["derive"] }
 codespan = "0.11"
 codespan-reporting = "0.11"
 diffy = "0.3"
+datatest-stable = "0.2"
 either = "1.9"
 fixed = "= 1.25.1" # required by aptos deps
 fs_extra = "1.3"
@@ -56,6 +57,7 @@ rand = "0.8"
 rayon = "1.10"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
+stacker = "0.1"
 tabled = "0.16"
 tempfile = "3.12"
 termcolor = "1.1" # aptos deps require 1.1 here

move-mutation-test/Cargo.toml

@@ -37,3 +37,11 @@ pretty_env_logger = { workspace = true }
 rayon = { workspace = true }
 serde = { workspace = true }
 termcolor = { workspace = true }
+
+[dev-dependencies]
+datatest-stable = { workspace = true }
+stacker = { workspace = true }
+
+[[test]]
+name = "testsuite"
+harness = false

move-mutation-test/src/lib.rs

@@ -190,16 +190,19 @@ pub fn run_mutation_test(
         }
     }
 
-    if let Some(outfile) = &options.output {
-        test_report.save_to_json_file(outfile)?;
-    }
     println!("\nTotal mutants tested: {}", test_report.mutants_tested());
     println!("Total mutants killed: {}\n", test_report.mutants_killed());
     test_report.print_table();
 
     benchmarks.total_tool_duration.stop();
     benchmarks.display();
 
+    if let Some(outfile) = &options.output {
+        let out = std::env::current_dir()?.join(outfile);
+        test_report.save_to_json_file(&out)?;
+        println!("Report saved to: {}", out.display());
+    }
+
     Ok(())
 }
 

move-mutation-test/tests/testsuite.rs

@@ -0,0 +1,75 @@
+use aptos::common::types::MovePackageDir;
+use log::info;
+use move_mutation_test::cli::CLIOptions;
+use move_mutation_test::cli::TestBuildConfig;
+use move_mutation_test::run_mutation_test;
+use mutator_common::report::Report;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+const RED_ZONE: usize = 128 * 1024; // 128 KiB
+const STACK_SIZE: usize = 32 * RED_ZONE; // 4 MiB
+
+fn test_run_mutation_test(path: &Path, expected_report: String) -> datatest_stable::Result<()> {
+    let expected_report =
+        Report::load_from_str(expected_report).expect("failed to load the report");
+    let package_path = path.parent().expect("package path not found");
+
+    let mut move_pkg = MovePackageDir::new();
+    move_pkg.package_dir = Some(PathBuf::from(package_path));
+    let test_build_cfg = TestBuildConfig {
+        move_pkg,
+        dump_state: false,
+        filter: None,
+        ignore_compile_warnings: false,
+        apply_coverage: true,
+        gas_limit: 1_000,
+    };
+
+    let report_file = PathBuf::from("report.txt");
+    let cli_opts = CLIOptions {
+        output: Some(report_file.clone()),
+        ..Default::default()
+    };
+
+    stacker::maybe_grow(RED_ZONE, STACK_SIZE, || {
+        run_mutation_test(&cli_opts, &test_build_cfg).expect("running the mutation test failed");
+        info!(
+            "remaining stack size is {}",
+            stacker::remaining_stack().expect("failed to get the remaining stack size")
+        );
+    });
+
+    let generated_report = Report::load_from_json_file(&report_file).expect("report not found");
+
+    // Let's make sure the reports are equal.
+    let Report {
+        files: mut expected_entries,
+        ..
+    } = expected_report;
+    let Report {
+        files: generated_report_files,
+        ..
+    } = generated_report;
+
+    // Unfortunately, we cannot compare the files directly since the `package_path` is an absolute
+    // path and would differ on different machines depending on the package location.
+    for (file, mutant_stats) in generated_report_files {
+        let (expected_file, expected_mutant_stats) = expected_entries
+            .pop_first()
+            .expect("reports are not the same");
+        assert_eq!(file, expected_file);
+        assert_eq!(mutant_stats, expected_mutant_stats);
+    }
+    assert!(expected_entries.is_empty());
+
+    // Make sure we remove the file since these tests are executed serially - it makes no sense to
+    // run these tests in parallel since every test spawns the maximum number of threads.
+    fs::remove_file(report_file).unwrap();
+
+    Ok(())
+}
+
+const MOVE_ASSETS: &str = "../move-mutator/tests/move-assets";
+
+datatest_stable::harness!(test_run_mutation_test, MOVE_ASSETS, r".*\.exp",);