Add changing SLO experiment scripts and definitions

experiments/15-e2e-scenarios-v2/slo_change/COND

@@ -0,0 +1,31 @@
+include("../common.cond")
+
+QUERIES = [99, 56, 32, 92, 91, 49, 30, 83, 94, 38, 87, 86, 76, 37, 31, 46, 58, 61, 62, 64, 69, 73, 74, 51, 57, 60]
+
+COMMON_CONFIGS = {
+ "physical-config-file": "config/physical_config_100gb.yml",
+ "schema-name": "imdb_extended_100g",
+ "ra-query-bank-file": IMDB_100GB_REGULAR_QUERY_BANK,
+ "txn-scale-factor": IMDB_100GB_SF,
+ "num-front-ends": 24,
+ "dataset-type": "100gb",
+ "ra-query-indexes": ",".join(map(str, QUERIES))
+}
+
+run_experiment(
+ name="brad_100g",
+ run="./run_workload.sh",
+ options={
+ "system-config-file": "slo_change_config.yml",
+ **COMMON_CONFIGS,
+ },
+)
+
+run_command(
+ name="brad_100g_debug",
+ run="./run_workload_debug.sh",
+ options={
+ "system-config-file": "slo_change_config.yml",
+ **COMMON_CONFIGS,
+ },
+)

experiments/15-e2e-scenarios-v2/slo_change/run_workload.sh

@@ -0,0 +1,48 @@
+#! /bin/bash
+
+script_loc=$(cd $(dirname $0) && pwd -P)
+cd $script_loc
+source ../common.sh
+
+# Arguments:
+# --system-config-file
+# --physical-config-file
+# --query-indexes
+extract_named_arguments $@
+
+start_brad $system_config_file $physical_config_file
+log_workload_point "brad_start_initiated"
+sleep 30
+
+log_workload_point "clients_starting"
+# 8 clients, offset 16 (for the transactional clients)
+start_repeating_olap_runner 8 15 5 $ra_query_indexes "ra_8" 16
+rana_pid=$runner_pid
+
+start_txn_runner_serial 16 # Implicit: --dataset-type
+txn_pid=$runner_pid
+
+log_workload_point "clients_started"
+
+function inner_cancel_experiment() {
+ cancel_experiment $rana_pid $txn_pid
+}
+
+trap "inner_cancel_experiment" INT
+trap "inner_cancel_experiment" TERM
+
+# Sleep for 10 minutes and then change the SLOs.
+sleep $(( 10 * 60 ))
+
+log_workload_point "changing_slo"
+brad cli --command "BRAD_CHANGE_SLO 30.0 0.030"
+log_workload_point "changed_slo"
+
+# Wait another hour before stopping.
+sleep $(( 60 * 60 ))
+
+# Shut down everything now.
+log_workload_point "experiment_workload_done"
+>&2 echo "Experiment done. Shutting down runners..."
+graceful_shutdown $rana_pid $txn_pid
+log_workload_point "shutdown_complete"

experiments/15-e2e-scenarios-v2/slo_change/run_workload_debug.sh

@@ -0,0 +1,49 @@
+#! /bin/bash
+
+script_loc=$(cd $(dirname $0) && pwd -P)
+cd $script_loc
+source ../common.sh
+
+# Arguments:
+# --system-config-file
+# --physical-config-file
+# --query-indexes
+extract_named_arguments $@
+
+export BRAD_IGNORE_BLUEPRINT=1
+start_brad_debug $system_config_file $physical_config_file
+log_workload_point "brad_start_initiated"
+sleep 30
+
+log_workload_point "clients_starting"
+# 8 clients, offset 16 (for the transactional clients)
+start_repeating_olap_runner 8 15 5 $ra_query_indexes "ra_8" 16
+rana_pid=$runner_pid
+
+start_txn_runner_serial 16 # Implicit: --dataset-type
+txn_pid=$runner_pid
+
+log_workload_point "clients_started"
+
+function inner_cancel_experiment() {
+ cancel_experiment $rana_pid $txn_pid
+}
+
+trap "inner_cancel_experiment" INT
+trap "inner_cancel_experiment" TERM
+
+# Sleep for 2 minutes and then change the SLOs.
+sleep $(( 2 * 60 ))
+
+log_workload_point "changing_slo"
+brad cli --command "BRAD_CHANGE_SLO 30.0 0.030"
+log_workload_point "changed_slo"
+
+# Wait another 10 mins before stopping.
+sleep $(( 10 * 60 ))
+
+# Shut down everything now.
+log_workload_point "experiment_workload_done"
+>&2 echo "Experiment done. Shutting down runners..."
+graceful_shutdown $rana_pid $txn_pid
+log_workload_point "shutdown_complete"

experiments/15-e2e-scenarios-v2/slo_change/set_up_starting_blueprint.sh

@@ -0,0 +1,20 @@
+#! /bin/bash
+
+if [ -z $1 ]; then
+ >&2 echo "Usage: $0 path/to/physical/config.yml"
+ exit 1
+fi
+
+script_loc=$(cd $(dirname $0) && pwd -P)
+cd $script_loc
+source ../common.sh
+
+python3 ../../../workloads/IMDB_extended/set_up_starting_blueprint.py \
+ --schema-name imdb_extended_100g \
+ --query-bank-file ../../../workloads/IMDB_100GB/regular_test/queries.sql \
+ --aurora-queries "99,56,32,92,91" \
+ --redshift-queries "49,30,83,94,38,87,86,76,37,31,46,58,61,62,64,69,73,74,51,57,60" \
+ --redshift-provisioning "dc2.large:2" \
+ --aurora-provisioning "db.t4g.medium:2" \
+ --system-config-file slo_change_config.yml \
+ --physical-config-file $1

experiments/15-e2e-scenarios-v2/slo_change/slo_change_config.yml

@@ -0,0 +1,168 @@
+# This file contains configurations that are used by BRAD. These are default
+# values and should be customized for specific situations.
+
+# BRAD's front end servers will listen for client connections on this interface
+# and port. If `num_front_ends` is greater than one, subsequent front ends will
+# listen on successive ports (e.g., 6584, 6585, etc.).
+front_end_interface: "0.0.0.0"
+front_end_port: 6583
+num_front_ends: 24
+
+# Logging paths. If the value is in ALL_CAPS (with underscores), it is
+# interpreted as an environment variable (BRAD will log to the path stored in
+# the environment variable).
+
+# Where BRAD's daemon process will write its logs.
+daemon_log_file: COND_OUT
+
+# Where BRAD's front end processes will write their logs.
+front_end_log_path: COND_OUT
+
+# Where BRAD's blueprint planner will write debug logs.
+planner_log_path: COND_OUT
+
+# Where BRAD's metrics loggers will write their logs.
+metrics_log_path: COND_OUT
+
+# Probability that each transactional query will be logged.
+txn_log_prob: 0.01
+
+# Set to a non-zero value enable automatic data syncing. When this is set to 0,
+# automatic syncing is disabled.
+data_sync_period_seconds: 0
+
+# BRAD's front end servers will report their metrics at regular intervals.
+front_end_metrics_reporting_period_seconds: 30
+front_end_query_latency_buffer_size: 100
+
+# `default` means to use the policy encoded in the blueprint. Other values will
+# override the blueprint.
+routing_policy: default
+
+# Whether to disable table movement for benchmark purposes (i.e., keep all
+# tables on all engines.)
+disable_table_movement: true
+
+# Epoch length for metrics and forecasting. This is the granularity at which
+# metrics/forecasting will be performed.
+epoch_length:
+ weeks: 0
+ days: 0
+ hours: 0
+ minutes: 1
+
+# Blueprint planning strategy.
+strategy: fp_query_based_beam
+
+# Used to specify the period of time over which to use data for planning.
+# Currrently, this is a "look behind" window for the workload.
+planning_window:
+ weeks: 0
+ days: 0
+ hours: 1
+ minutes: 0
+
+# Used to aggregate metrics collected in the planning window.
+metrics_agg:
+ method: ewm # 'mean' is another option
+ alpha: 0.86466472 # 1 - 1 / e^2
+
+# Used during planning.
+reinterpret_second_as: 1
+
+# The query distribution must change by at least this much for a new blueprint
+# to be accepted.
+query_dist_change_frac: 0.1
+
+# The search bound for the provisioning.
+max_provisioning_multiplier: 2.5
+
+# Flag options for blueprint planning.
+use_io_optimized_aurora: true
+use_recorded_routing_if_available: true
+ensure_tables_together_on_one_engine: true
+
+# Loads used to prime the system when no information is available.
+aurora_initialize_load_fraction: 0.25
+redshift_initialize_load_fraction: 0.25
+
+# BRAD will not reduce predicted load lower than these values. Raise these
+# values to be more conservative against mispredictions.
+aurora_min_load_removal_fraction: 0.8
+redshift_min_load_removal_fraction: 0.9
+
+# Blueprint planning performance ceilings.
+# These will change to 30 s and 30 ms during the experiment.
+query_latency_p90_ceiling_s: 60.0
+txn_latency_p90_ceiling_s: 0.060
+
+# If set to true, BRAD will attempt to use the specified preset Redshift
+# clusters instead of resizing the main Redshift cluster.
+use_preset_redshift_clusters: true
+
+# Used for ordering blueprints during planning.
+comparator:
+ type: benefit_perf_ceiling # or `perf_ceiling`
+
+ benefit_horizon: # Only used by the `benefit_perf_ceiling` comparator
+ weeks: 0
+ days: 0
+ hours: 3
+ minutes: 0
+
+ penalty_threshold: 0.8 # Only used by the `benefit_perf_ceiling` comparator
+ penalty_power: 8 # Only used by the `benefit_perf_ceiling` comparator
+
+# Used for precomputed predictions.
+std_datasets:
+ - name: regular
+ path: workloads/IMDB_100GB/regular_test/
+ - name: adhoc
+ path: workloads/IMDB_100GB/adhoc_test/
+
+aurora_max_query_factor: 4.0
+aurora_max_query_factor_replace: 10000.0
+
+redshift_peak_load_threshold: 95.0
+redshift_peak_load_multiplier: 2.0
+
+planner_max_workers: 16
+aurora_provisioning_search_distance: 900.0
+redshift_provisioning_search_distance: 900.0
+
+# Blueprint planning trigger configs.
+
+triggers:
+ enabled: true
+ check_period_s: 90 # Triggers are checked every X seconds.
+ check_period_offset_s: 600 # Wait 10 mins before starting.
+ observe_new_blueprint_mins: 10
+
+ elapsed_time:
+ disabled: true
+ multiplier: 60 # Multiplier over `planning_window`.
+
+ redshift_cpu:
+ lo: 15
+ hi: 85
+ sustained_epochs: 3
+
+ aurora_cpu:
+ lo: 15
+ hi: 85
+ sustained_epochs: 3
+
+ variable_costs:
+ disabled: true
+ threshold: 1.0
+
+ query_latency_ceiling:
+ ceiling_s: 60.0
+ sustained_epochs: 3
+
+ txn_latency_ceiling:
+ ceiling_s: 0.060
+ sustained_epochs: 3
+
+ recent_change:
+ delay_epochs: 5

src/brad/config/system_event.py

@@ -33,3 +33,7 @@ class SystemEvent(enum.Enum):
 
  # Use this for long running experiments.
  ReachedExpectedState = "reached_expected_state"
+
+ # Used when a service level objective is changed while BRAD is running (used
+ # for experiments).
+ ChangedSlos = "changed_slos"

src/brad/daemon/daemon.py

@@ -733,7 +733,7 @@ async def _handle_internal_command(self, command: str) -> RowList:
  parts = command.split(" ")
  if self._temp_config is None:
  return [("Cannot change SLOs because TempConfig is missing.",)]
- if len(parts) <= 3:
+ if len(parts) < 3:
  return [("Need to specify query and txn p90 SLOs",)]
 
  query_p90_s = float(parts[1])
@@ -757,6 +757,12 @@ async def _handle_internal_command(self, command: str) -> RowList:
  elif isinstance(t, TransactionLatencyCeiling):
  t.set_latency_ceiling(txn_p90_s)
 
+ if self._system_event_logger is not None:
+ self._system_event_logger.log(
+ SystemEvent.ChangedSlos,
+ f"query_p90_s={query_p90_s}; txn_p90_s={txn_p90_s}",
+ )
+
  return [
  (
  f"p90 SLOs changed to (query {query_p90_s:.3f} s), (txn {txn_p90_s:.3f} s)",