Add a test that fuzzers generate wasm proposals

This commit adds a test to the `wasmtime-fuzzing` crate as a
sanity-check that eventually a module is generated requiring all of the
features that wasmtime supports. This is intended to be another
double-check in the process of enabling a proposal in Wasmtime by
ensuring that the feature is added to this list which then transitively
requires that fuzzing eventually generates a module needing that feature.

cc bytecodealliance#9449

crates/fuzzing/src/oracles.rs

@@ -1166,36 +1166,110 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32
 #[cfg(test)]
 mod tests {
  use super::*;
+ use arbitrary::Unstructured;
+ use rand::prelude::*;
+ use wasmparser::{Validator, WasmFeatures};
+
+ fn gen_until_pass<T: for<'a> Arbitrary<'a>>(
+ mut f: impl FnMut(T, &mut Unstructured<'_>) -> Result<bool>,
+ ) -> bool {
+ let mut rng = SmallRng::seed_from_u64(0);
+ let mut buf = vec![0; 2048];
+ let n = 2000;
+ for _ in 0..n {
+ rng.fill_bytes(&mut buf);
+ let mut u = Unstructured::new(&buf);
+
+ if let Ok(config) = u.arbitrary() {
+ if f(config, &mut u).unwrap() {
+ return true;
+ }
+ }
+ }
+ false
+ }
 
  // Test that the `table_ops` fuzzer eventually runs the gc function in the host.
  // We've historically had issues where this fuzzer accidentally wasn't fuzzing
  // anything for a long time so this is an attempt to prevent that from happening
  // again.
  #[test]
  fn table_ops_eventually_gcs() {
- use arbitrary::Unstructured;
- use rand::prelude::*;
-
  // Skip if we're under emulation because some fuzz configurations will do
  // large address space reservations that QEMU doesn't handle well.
  if std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok() {
  return;
  }
 
- let mut rng = SmallRng::seed_from_u64(0);
- let mut buf = vec![0; 2048];
- let n = 100;
- for _ in 0..n {
- rng.fill_bytes(&mut buf);
- let u = Unstructured::new(&buf);
+ let ok = gen_until_pass(|(config, test), _| {
+ let result = table_ops(config, test)?;
+ Ok(result > 0)
+ });
 
- if let Ok((config, test)) = Arbitrary::arbitrary_take_rest(u) {
- if table_ops(config, test).unwrap() > 0 {
- return;
+ if !ok {
+ panic!("gc was never found");
+ }
+ }
+
+ #[test]
+ fn module_generation_uses_expected_proposals() {
+ // Proposals that Wasmtime supports. Eventually a module should be
+ // generated that needs these proposals.
+ let mut expected = WasmFeatures::MUTABLE_GLOBAL
+ | WasmFeatures::FLOATS
+ | WasmFeatures::SIGN_EXTENSION
+ | WasmFeatures::SATURATING_FLOAT_TO_INT
+ | WasmFeatures::MULTI_VALUE
+ | WasmFeatures::BULK_MEMORY
+ | WasmFeatures::REFERENCE_TYPES
+ | WasmFeatures::SIMD
+ | WasmFeatures::MULTI_MEMORY
+ | WasmFeatures::RELAXED_SIMD
+ | WasmFeatures::THREADS
+ | WasmFeatures::TAIL_CALL
+ | WasmFeatures::WIDE_ARITHMETIC
+ | WasmFeatures::MEMORY64
+ | WasmFeatures::GC_TYPES;
+
+ // All other features that wasmparser supports, which is presumably a
+ // superset of the features that wasm-smith supports, are listed here as
+ // unexpected. This means, for example, that if wasm-smith updates to
+ // include a new proposal by default that wasmtime implements then it
+ // will be required to be listed above.
+ let unexpected = WasmFeatures::all() ^ expected;
+
+ let ok = gen_until_pass(|config: generators::Config, u| {
+ let wasm = config.generate(u, None)?.to_bytes();
+
+ // Double-check the module is valid
+ Validator::new_with_features(WasmFeatures::all()).validate_all(&wasm)?;
+
+ // If any of the unexpected features are removed then this module
+ // should always be valid, otherwise something went wrong.
+ for feature in unexpected.iter() {
+ let ok =
+ Validator::new_with_features(WasmFeatures::all() ^ feature).validate_all(&wasm);
+ if ok.is_err() {
+ anyhow::bail!("generated a module with {feature:?} but that wasn't expected");
+ }
+ }
+
+ // If any of `expected` is removed and the module fails to validate,
+ // then that means the module requires that feature. Remove that
+ // from the set of features we're then expecting.
+ for feature in expected.iter() {
+ let ok =
+ Validator::new_with_features(WasmFeatures::all() ^ feature).validate_all(&wasm);
+ if ok.is_err() {
+ expected ^= feature;
  }
  }
- }
 
- panic!("after {n} runs nothing ever gc'd, something is probably wrong");
+ Ok(expected.is_empty())
+ });
+
+ if !ok {
+ panic!("never generated wasm module using {expected:?}");
+ }
  }
 }

docs/stability-wasm-proposals.md

@@ -103,6 +103,8 @@ For each column in the above tables, this is a further explanation of its meanin
 
 * **Fuzzed** - Has been fuzzed for at least a week minimum. We are also
  confident that the fuzzers are fully exercising the proposal's functionality.
+ The `module_generation_uses_expected_proposals` test in the `wasmtime-fuzzing`
+ crate must be updated to include this proposal.
 
  > For example, it would *not* have been enough to simply enable reference
  > types in the `compile` fuzz target to enable that proposal by