add ExpectedFailurePayload

crates/sui-benchmark/src/drivers/bench_driver.rs

@@ -43,9 +43,6 @@ use tokio::task::{JoinHandle, JoinSet};
 use tokio::{time, time::Instant};
 use tracing::{debug, error, info, warn};
 
-use crate::workloads::workload::FailureType;
-use sui_types::utils::to_sender_signed_transaction;
-
 use super::Interval;
 use super::{BenchmarkStats, StressStats};
 pub struct BenchMetrics {
@@ -928,11 +925,6 @@ async fn run_bench_worker(
                     metrics_cloned.num_in_flight.with_label_values(&[&payload.to_string()]).inc();
                     metrics_cloned.num_submitted.with_label_values(&[&payload.to_string()]).inc();
                     let tx = payload.make_transaction();
-                    let tx = if let Some(failure_type) = payload.get_failure_type() {
-                        apply_failure(tx, failure_type)
-                    } else {
-                        tx
-                    };
                     let start = Arc::new(Instant::now());
                     // TODO: clone committee for each request is not ideal.
                     let committee = worker.proxy.clone_committee();
@@ -1116,37 +1108,3 @@ fn stress_stats_collector(
         }
     })
 }
-
-pub fn apply_failure(mut transaction: Transaction, failure_type: FailureType) -> Transaction {
-    let data = transaction.data().clone();
-
-    // TODO(william) make this work
-    match failure_type {
-        FailureType::InvalidSignature => {
-            let data = transaction.data().clone();
-            let invalid_keypair = SuiKeyPair::Ed25519(Ed25519KeyPair::generate(&mut OsRng));
-            let invalid_signature =
-                Signature::new_secure(data.intent_message().value.digest(), &invalid_keypair);
-            let new_data = SenderSignedData::new(
-                data.intent_message().clone(),
-                vec![invalid_signature],
-                data.gas().clone(),
-            );
-            to_sender_signed_transaction(new_data, transaction.tx_signatures()[0].as_ref())
-        }
-        FailureType::LowGasBudget => {
-            let data = transaction.data().clone();
-            let new_data = SenderSignedData::new(
-                data.intent_message().clone(),
-                data.tx_signatures().to_vec(),
-                GasData {
-                    payment: data.gas().payment.clone(),
-                    owner: data.gas().owner,
-                    price: data.gas().price,
-                    budget: 1, // Set to minimum possible value to ensure failure
-                },
-            );
-            to_sender_signed_transaction(new_data, transaction.tx_signatures()[0].as_ref())
-        }
-    }
-}

crates/sui-benchmark/src/options.rs

@@ -181,6 +181,9 @@ pub enum RunSpec {
         // relative weight of randomness transactions in the benchmark workload
         #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])]
         randomness: Vec<u32>,
+        // relative weight of expected failure transactions in the benchmark workload
+        #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = [0])]
+        expected_failure: Vec<u32>,
 
         // --- workload-specific options --- (TODO: use subcommands or similar)
         // 100 for max hotness i.e all requests target
@@ -210,6 +213,12 @@ pub enum RunSpec {
         // Default is (0-0.5) implying random load at 50% load. See `AdversarialPayloadType` enum for `adversarial_type`
         #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = ["0-1.0".to_string()])]
         adversarial_cfg: Vec<String>,
+        // type and load % of expected failure transactions in the benchmark workload.
+        // Format is "{expected_failure_type}-{load_factor}".
+        // `load_factor` is a number between 0.0 and 1.0 which dictates how much load per tx
+        // Default is (0-0.5) implying random load at 50% load. See `FailureType` enum for `expected_failure_type`
+        #[clap(long, num_args(1..), value_delimiter = ',', default_values_t = ["0-1.0".to_string()])]
+        expected_failure_cfg: Vec<String>,
 
         // --- generic options ---
         // Target qps

crates/sui-benchmark/src/workloads/adversarial.rs

@@ -10,7 +10,7 @@ use crate::in_memory_wallet::move_call_pt_impl;
 use crate::in_memory_wallet::InMemoryWallet;
 use crate::system_state_observer::{SystemState, SystemStateObserver};
 use crate::workloads::payload::Payload;
-use crate::workloads::{Gas, GasCoinConfig};
+use crate::workloads::{workload::FailureType, Gas, GasCoinConfig};
 use crate::ProgrammableTransactionBuilder;
 use crate::{convert_move_call_args, BenchMoveCallArg, ExecutionEffects, ValidatorProxy};
 use anyhow::anyhow;
@@ -189,6 +189,10 @@ impl Payload for AdversarialTestPayload {
                 .expect("Protocol config not in system state"),
         )
     }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 impl AdversarialTestPayload {

crates/sui-benchmark/src/workloads/batch_payment.rs

@@ -5,7 +5,7 @@ use crate::drivers::Interval;
 use crate::in_memory_wallet::InMemoryWallet;
 use crate::system_state_observer::SystemStateObserver;
 use crate::workloads::payload::Payload;
-use crate::workloads::workload::{Workload, STORAGE_COST_PER_COIN};
+use crate::workloads::workload::{FailureType, Workload, STORAGE_COST_PER_COIN};
 use crate::workloads::workload::{WorkloadBuilder, ESTIMATED_COMPUTATION_COST};
 use crate::workloads::{Gas, GasCoinConfig, WorkloadBuilderInfo, WorkloadParams};
 use crate::{ExecutionEffects, ValidatorProxy};
@@ -116,6 +116,10 @@ impl Payload for BatchPaymentTestPayload {
             gas_budget,
         )
     }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]

crates/sui-benchmark/src/workloads/delegation.rs

@@ -4,7 +4,7 @@
 use crate::drivers::Interval;
 use crate::system_state_observer::SystemStateObserver;
 use crate::workloads::payload::Payload;
-use crate::workloads::workload::{Workload, WorkloadBuilder};
+use crate::workloads::workload::{FailureType, Workload, WorkloadBuilder};
 use crate::workloads::workload::{
     ESTIMATED_COMPUTATION_COST, MAX_GAS_FOR_TESTING, STORAGE_COST_PER_COIN,
 };
@@ -80,6 +80,10 @@ impl Payload for DelegationTestPayload {
             ),
         }
     }
+
+    fn get_failure_type(&self) -> Option<FailureType> {
+        None
+    }
 }
 
 #[derive(Debug)]