add ExpectedFailurePayload

crates/sui-benchmark/src/drivers/bench_driver.rs

@@ -18,6 +18,7 @@ use prometheus::{register_int_counter_vec_with_registry, CounterVec};
 use prometheus::{register_int_gauge_with_registry, GaugeVec};
 use prometheus::{HistogramVec, IntGauge};
 use rand::seq::SliceRandom;
+use sui_types::crypto::Signature;
 use tokio::sync::mpsc::{channel, Sender};
 use tokio::sync::OnceCell;
 use tokio_util::sync::CancellationToken;

crates/sui-benchmark/src/options.rs

@@ -181,6 +181,9 @@ pub enum RunSpec {
         // relative weight of randomness transactions in the benchmark workload
         #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])]
         randomness: Vec<u32>,
+        // relative weight of expected failure transactions in the benchmark workload
+        #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])]
+        expected_failure: Vec<u32>,
 
         // --- workload-specific options --- (TODO: use subcommands or similar)
         // 100 for max hotness i.e all requests target

crates/sui-benchmark/src/workloads/adversarial.rs

@@ -10,7 +10,7 @@ use crate::in_memory_wallet::move_call_pt_impl;
 use crate::in_memory_wallet::InMemoryWallet;
 use crate::system_state_observer::{SystemState, SystemStateObserver};
 use crate::workloads::payload::Payload;
-use crate::workloads::{Gas, GasCoinConfig};
+use crate::workloads::{workload::FailureType, Gas, GasCoinConfig};
 use crate::ProgrammableTransactionBuilder;
 use crate::{convert_move_call_args, BenchMoveCallArg, ExecutionEffects, ValidatorProxy};
 use anyhow::anyhow;
@@ -189,6 +189,10 @@ impl Payload for AdversarialTestPayload {
                 .expect("Protocol config not in system state"),
         )
     }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 impl AdversarialTestPayload {

crates/sui-benchmark/src/workloads/batch_payment.rs

@@ -5,7 +5,7 @@ use crate::drivers::Interval;
 use crate::in_memory_wallet::InMemoryWallet;
 use crate::system_state_observer::SystemStateObserver;
 use crate::workloads::payload::Payload;
-use crate::workloads::workload::{Workload, STORAGE_COST_PER_COIN};
+use crate::workloads::workload::{FailureType, Workload, STORAGE_COST_PER_COIN};
 use crate::workloads::workload::{WorkloadBuilder, ESTIMATED_COMPUTATION_COST};
 use crate::workloads::{Gas, GasCoinConfig, WorkloadBuilderInfo, WorkloadParams};
 use crate::{ExecutionEffects, ValidatorProxy};
@@ -116,6 +116,10 @@ impl Payload for BatchPaymentTestPayload {
             gas_budget,
         )
     }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]

crates/sui-benchmark/src/workloads/delegation.rs

@@ -4,7 +4,7 @@
 use crate::drivers::Interval;
 use crate::system_state_observer::SystemStateObserver;
 use crate::workloads::payload::Payload;
-use crate::workloads::workload::{Workload, WorkloadBuilder};
+use crate::workloads::workload::{FailureType, Workload, WorkloadBuilder};
 use crate::workloads::workload::{
     ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING, STORAGE_COST_PER_COIN,
 };
@@ -80,6 +80,10 @@ impl Payload for DelegationTestPayload {
             ),
         }
     }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]

crates/sui-benchmark/src/workloads/expected_failure.rs

@@ -0,0 +1,79 @@
+use crate::workloads::{FailureType, Payload};
+use crate::ExecutionEffects;
+use std::fmt;
+use sui_types::base_types::{ObjectID, SuiAddress};
+use sui_types::transaction::{Transaction, TransactionData};
+
+#[derive(Debug, Clone)]
+pub struct ExpectedFailurePayload {
+    failure_type: FailureType,
+    sender: SuiAddress,
+}
+
+impl ExpectedFailurePayload {
+    pub fn new(failure_type: FailureType, sender: SuiAddress) -> Self {
+        Self {
+            failure_type,
+            sender,
+        }
+    }
+
+    fn create_failing_transaction(&self) -> Transaction {
+        let mut rng = rand::thread_rng();
+        let random_object_id = ObjectID::random();
+
+        match self.failure_type {
+            FailureType::LowGasBudget => {
+                // Create a transaction with insufficient gas
+                let data = TransactionData::new_move_call(
+                    self.sender,
+                    random_object_id,
+                    "dummy_package",
+                    "dummy_module",
+                    "dummy_function",
+                    vec![],
+                    vec![],
+                    1, // Set gas budget to 1, which is insufficient
+                    1000,
+                )
+                .unwrap();
+                Transaction::new(data)
+            }
+            FailureType::InvalidSignature => {
+                // Create a transaction with an invalid signature
+                let data = TransactionData::new_transfer(
+                    random_object_id,
+                    self.sender,
+                    self.sender,
+                    random_object_id,
+                    1000,
+                    1000,
+                );
+                let mut tx = Transaction::new(data);
+                tx.signatures_mut().push(vec![0; 32]); // Invalid signature
+                tx
+            }
+            FailureType::Random => unreachable!(),
+        }
+    }
+}
+
+impl Payload for ExpectedFailurePayload {
+    fn make_new_payload(&mut self, _effects: &ExecutionEffects) {
+        // For this payload, we don't need to update anything based on effects
+    }
+
+    fn make_transaction(&mut self) -> Transaction {
+        self.create_failing_transaction()
+    }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        Some(self.failure_type)
+    }
+}
+
+impl fmt::Display for ExpectedFailurePayload {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "ExpectedFailurePayload({:?})", self.failure_type)
+    }
+}

crates/sui-benchmark/src/workloads/mod.rs

@@ -4,6 +4,7 @@
 pub mod adversarial;
 pub mod batch_payment;
 pub mod delegation;
+pub mod expected_failure;
 pub mod payload;
 pub mod randomness;
 pub mod shared_counter;

crates/sui-benchmark/src/workloads/randomness.rs

@@ -6,7 +6,7 @@ use crate::system_state_observer::SystemStateObserver;
 use crate::util::publish_basics_package;
 use crate::workloads::payload::Payload;
 use crate::workloads::workload::{
-    Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING,
+    FailureType, Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING,
 };
 use crate::workloads::GasCoinConfig;
 use crate::workloads::{Gas, WorkloadBuilderInfo, WorkloadParams};
@@ -58,6 +58,9 @@ impl Payload for RandomnessTestPayload {
             .call_emit_random(self.package_id, self.randomness_initial_shared_version)
             .build_and_sign(self.gas.2.as_ref())
     }
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]

crates/sui-benchmark/src/workloads/shared_counter.rs

@@ -6,7 +6,7 @@ use crate::system_state_observer::SystemStateObserver;
 use crate::util::publish_basics_package;
 use crate::workloads::payload::Payload;
 use crate::workloads::workload::{
-    Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING,
+    FailureType, Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING,
     STORAGE_COST_PER_COUNTER,
 };
 use crate::workloads::GasCoinConfig;
@@ -72,6 +72,9 @@ impl Payload for SharedCounterTestPayload {
             )
             .build_and_sign(self.gas.2.as_ref())
     }
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]

crates/sui-benchmark/src/workloads/shared_object_deletion.rs

@@ -6,7 +6,7 @@ use crate::system_state_observer::SystemStateObserver;
 use crate::util::publish_basics_package;
 use crate::workloads::payload::Payload;
 use crate::workloads::workload::{
-    Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING,
+    FailureType, Workload, WorkloadBuilder, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING,
     STORAGE_COST_PER_COUNTER,
 };
 use crate::workloads::GasCoinConfig;
@@ -118,6 +118,9 @@ impl Payload for SharedCounterDeletionTestPayload {
         }
         .build_and_sign(self.gas.2.as_ref())
     }
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]

crates/sui-benchmark/src/workloads/transfer_object.rs

@@ -13,7 +13,7 @@ use crate::system_state_observer::SystemStateObserver;
 use crate::workloads::payload::Payload;
 use crate::workloads::workload::WorkloadBuilder;
 use crate::workloads::workload::{
-    Workload, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING, STORAGE_COST_PER_COIN,
+    FailureType, Workload, ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING, STORAGE_COST_PER_COIN,
 };
 use crate::workloads::{Gas, GasCoinConfig, WorkloadBuilderInfo, WorkloadParams};
 use crate::{ExecutionEffects, ValidatorProxy};
@@ -80,6 +80,9 @@ impl Payload for TransferObjectTestPayload {
                 .reference_gas_price,
         )
     }
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 impl std::fmt::Display for TransferObjectTestPayload {

crates/sui-benchmark/src/workloads/workload.rs

@@ -5,8 +5,14 @@ use crate::system_state_observer::SystemStateObserver;
 use crate::workloads::payload::Payload;
 use crate::workloads::{Gas, GasCoinConfig};
 use crate::ValidatorProxy;
+use anyhow::anyhow;
 use async_trait::async_trait;
+use rand::distributions::{Distribution, Standard};
+use rand::Rng;
+use std::str::FromStr;
 use std::sync::Arc;
+use strum::{EnumCount, IntoEnumIterator};
+use strum_macros::{EnumCount as EnumCountMacro, EnumIter};
 use sui_types::gas_coin::MIST_PER_SUI;
 
 // This is the maximum gas we will transfer from primary coin into any gas coin
@@ -23,13 +29,56 @@ pub const STORAGE_COST_PER_COUNTER: u64 = 341 * 76 * 100;
 /// Used to estimate the budget required for each transaction.
 pub const ESTIMATED_COMPUTATION_COST: u64 = 1_000_000;
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, EnumCountMacro, EnumIter, Clone, Copy)]
 pub enum FailureType {
+    Random = 0,
     InvalidSignature,
     LowGasBudget,
     // Add other failure types as needed
 }
 
+impl TryFrom<u32> for FailureType {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u32) -> Result<Self, Self::Error> {
+        match value {
+            0 => Ok(rand::random()),
+            _ => FailureType::iter().nth(value as usize).ok_or_else(|| {
+                anyhow!(
+                    "Invalid failure type specifier. Valid options are {} to {}",
+                    0,
+                    FailureType::COUNT
+                )
+            }),
+        }
+    }
+}
+
+impl FromStr for FailureType {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let v = u32::from_str(s).map(FailureType::try_from);
+
+        if let Ok(Ok(q)) = v {
+            return Ok(q);
+        }
+
+        Err(anyhow!(
+            "Invalid input string. Valid values are 0 to {}",
+            FailureType::COUNT
+        ))
+    }
+}
+
+impl Distribution<FailureType> for Standard {
+    fn sample<R: Rng + ?Sized>(&self, rng: &mut R) -> FailureType {
+        // Exclude the "Random" variant
+        let n = rng.gen_range(1..FailureType::COUNT);
+        FailureType::iter().nth(n).unwrap()
+    }
+}
+
 #[async_trait]
 pub trait WorkloadBuilder<T: Payload + ?Sized>: Send + Sync + std::fmt::Debug {
     async fn generate_coin_config_for_init(&self) -> Vec<GasCoinConfig>;