diff --git a/cluster-density.go b/cluster-density.go index 831a3c88..8577dfc1 100644 --- a/cluster-density.go +++ b/cluster-density.go @@ -33,6 +33,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm var churnDelay, churnDuration time.Duration var churnDeletionStrategy string var podReadyThreshold time.Duration + var metricsProfiles []string cmd := &cobra.Command{ Use: variant, Short: fmt.Sprintf("Runs %v workload", variant), @@ -60,7 +61,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm log.Errorf("image-registry deployment is not deployed") } } - setMetrics(cmd, "metrics-aggregated.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -73,6 +74,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm cmd.Flags().IntVar(&churnPercent, "churn-percent", 10, "Percentage of job iterations that kube-burner will churn each round") cmd.Flags().StringVar(&churnDeletionStrategy, "churn-deletion-strategy", "default", "Churn deletion strategy to use") cmd.Flags().BoolVar(&svcLatency, "service-latency", false, "Enable service latency measurement") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-aggregated.yml"}, "Comma separated list of metrics profiles to use") cmd.MarkFlagRequired("iterations") return cmd } diff --git a/common.go b/common.go index 543ac99d..6ed51322 100644 --- a/common.go +++ b/common.go @@ -28,16 +28,13 @@ import ( var clusterMetadata ocpmetadata.ClusterMetadata -func setMetrics(cmd *cobra.Command, metricsProfile string) { - var metricsProfiles []string +func setMetrics(cmd *cobra.Command, metricsProfiles []string) { profileType, _ := cmd.Root().PersistentFlags().GetString("profile-type") switch ProfileType(profileType) { case Reporting: metricsProfiles = []string{"metrics-report.yml"} - case Regular: - metricsProfiles = []string{metricsProfile} case Both: - metricsProfiles = []string{"metrics-report.yml", metricsProfile} + metricsProfiles = append(metricsProfiles, "metrics-report.yml") } os.Setenv("METRICS", strings.Join(metricsProfiles, ",")) } diff --git a/crd-scale.go b/crd-scale.go index 1ebb2636..31adcb3c 100644 --- a/crd-scale.go +++ b/crd-scale.go @@ -25,6 +25,7 @@ import ( // NewCrdScale holds the crd-scale workload func NewCrdScale(wh *workloads.WorkloadHelper) *cobra.Command { var iterations int + var metricsProfiles []string cmd := &cobra.Command{ Use: "crd-scale", Short: "Runs crd-scale workload", @@ -33,11 +34,12 @@ func NewCrdScale(wh *workloads.WorkloadHelper) *cobra.Command { os.Setenv("JOB_ITERATIONS", fmt.Sprint(iterations)) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics-aggregated.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } cmd.Flags().IntVar(&iterations, "iterations", 0, "Number of CRDs to create") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-aggregated.yml"}, "Comma separated list of metrics profiles to use") cmd.MarkFlagRequired("iterations") return cmd } diff --git a/egressip.go b/egressip.go index feb38e58..2011b2d5 100644 --- a/egressip.go +++ b/egressip.go @@ -149,6 +149,7 @@ func NewEgressIP(wh *workloads.WorkloadHelper, variant string) *cobra.Command { var iterations, addressesPerIteration int var externalServerIP string var podReadyThreshold time.Duration + var metricsProfiles []string cmd := &cobra.Command{ Use: variant, Short: fmt.Sprintf("Runs %v workload", variant), @@ -160,7 +161,7 @@ func NewEgressIP(wh *workloads.WorkloadHelper, variant string) *cobra.Command { generateEgressIPs(iterations, addressesPerIteration, externalServerIP) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics-egressip.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -168,6 +169,7 @@ func NewEgressIP(wh *workloads.WorkloadHelper, variant string) *cobra.Command { cmd.Flags().IntVar(&iterations, "iterations", 0, fmt.Sprintf("%v iterations", variant)) cmd.Flags().StringVar(&externalServerIP, "external-server-ip", "", "External server IP address") cmd.Flags().IntVar(&addressesPerIteration, "addresses-per-iteration", 1, fmt.Sprintf("%v iterations", variant)) + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-egressip.yml"}, "Comma separated list of metrics profiles to use") cmd.MarkFlagRequired("iterations") cmd.MarkFlagRequired("external-server-ip") return cmd diff --git a/index.go b/index.go index 1f7e0831..6b2b76bf 100644 --- a/index.go +++ b/index.go @@ -18,7 +18,6 @@ import ( "encoding/json" "fmt" "os" - "strings" "time" "github.com/cloud-bulldozer/go-commons/indexers" @@ -35,7 +34,8 @@ import ( // NewIndex orchestrates indexing for ocp wrapper func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobra.Command { - var metricsProfile, jobName string + var jobName string + var metricsProfiles []string var start, end int64 var userMetadata, metricsDirectory string var prometheusStep time.Duration @@ -70,9 +70,6 @@ func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobr log.Fatal("Error obtaining prometheus information from cluster: ", err.Error()) } } - metricsProfiles := strings.FieldsFunc(metricsProfile, func(r rune) bool { - return r == ',' || r == ' ' - }) indexer = config.MetricsEndpoint{ Endpoint: prometheusURL, Token: prometheusToken, @@ -148,7 +145,7 @@ func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobr burner.IndexJobSummary([]burner.JobSummary{jobSummary}, indexerValue) }, } - cmd.Flags().StringVarP(&metricsProfile, "metrics-profile", "m", "metrics.yml", "comma-separated list of metric profiles") + cmd.Flags().StringSliceVarP(&metricsProfiles, "metrics-profile", "m", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") cmd.Flags().StringVar(&metricsDirectory, "metrics-directory", "collected-metrics", "Directory to dump the metrics files in, when using default local indexing") cmd.Flags().DurationVar(&prometheusStep, "step", 30*time.Second, "Prometheus step size") cmd.Flags().Int64Var(&start, "start", time.Now().Unix()-3600, "Epoch start time") diff --git a/networkpolicy.go b/networkpolicy.go index 8c72d164..d4a04886 100644 --- a/networkpolicy.go +++ b/networkpolicy.go @@ -29,6 +29,7 @@ func NewNetworkPolicy(wh *workloads.WorkloadHelper, variant string) *cobra.Comma var churn bool var churnDelay, churnDuration time.Duration var churnDeletionStrategy string + var metricsProfiles []string cmd := &cobra.Command{ Use: variant, Short: fmt.Sprintf("Runs %v workload", variant), @@ -42,7 +43,7 @@ func NewNetworkPolicy(wh *workloads.WorkloadHelper, variant string) *cobra.Comma os.Setenv("CHURN_DELETION_STRATEGY", churnDeletionStrategy) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -53,6 +54,7 @@ func NewNetworkPolicy(wh *workloads.WorkloadHelper, variant string) *cobra.Comma cmd.Flags().DurationVar(&churnDelay, "churn-delay", 2*time.Minute, "Time to wait between each churn") cmd.Flags().IntVar(&churnPercent, "churn-percent", 10, "Percentage of job iterations that kube-burner will churn each round") cmd.Flags().StringVar(&churnDeletionStrategy, "churn-deletion-strategy", "default", "Churn deletion strategy to use") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") cmd.MarkFlagRequired("iterations") return cmd } diff --git a/node-density-cni.go b/node-density-cni.go index e6f148c5..e1b3d057 100644 --- a/node-density-cni.go +++ b/node-density-cni.go @@ -32,6 +32,7 @@ func NewNodeDensityCNI(wh *workloads.WorkloadHelper) *cobra.Command { var namespacedIterations, svcLatency bool var podReadyThreshold time.Duration var iterationsPerNamespace int + var metricsProfiles []string cmd := &cobra.Command{ Use: "node-density-cni", Short: "Runs node-density-cni workload", @@ -49,7 +50,7 @@ func NewNodeDensityCNI(wh *workloads.WorkloadHelper) *cobra.Command { os.Setenv("SVC_LATENCY", strconv.FormatBool(svcLatency)) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -58,5 +59,6 @@ func NewNodeDensityCNI(wh *workloads.WorkloadHelper) *cobra.Command { cmd.Flags().BoolVar(&namespacedIterations, "namespaced-iterations", true, "Namespaced iterations") cmd.Flags().IntVar(&iterationsPerNamespace, "iterations-per-namespace", 1000, "Iterations per namespace") cmd.Flags().BoolVar(&svcLatency, "service-latency", false, "Enable service latency measurement") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") return cmd } diff --git a/node-density-heavy.go b/node-density-heavy.go index 7e2ea9c4..67d00139 100644 --- a/node-density-heavy.go +++ b/node-density-heavy.go @@ -31,6 +31,7 @@ func NewNodeDensityHeavy(wh *workloads.WorkloadHelper) *cobra.Command { var podReadyThreshold, probesPeriod time.Duration var namespacedIterations bool var iterationsPerNamespace int + var metricsProfiles []string cmd := &cobra.Command{ Use: "node-density-heavy", Short: "Runs node-density-heavy workload", @@ -49,7 +50,7 @@ func NewNodeDensityHeavy(wh *workloads.WorkloadHelper) *cobra.Command { os.Setenv("ITERATIONS_PER_NAMESPACE", fmt.Sprint(iterationsPerNamespace)) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -58,5 +59,6 @@ func NewNodeDensityHeavy(wh *workloads.WorkloadHelper) *cobra.Command { cmd.Flags().IntVar(&podsPerNode, "pods-per-node", 245, "Pods per node") cmd.Flags().BoolVar(&namespacedIterations, "namespaced-iterations", true, "Namespaced iterations") cmd.Flags().IntVar(&iterationsPerNamespace, "iterations-per-namespace", 1000, "Iterations per namespace") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") return cmd } diff --git a/node-density.go b/node-density.go index e008572f..43a8112d 100644 --- a/node-density.go +++ b/node-density.go @@ -30,6 +30,7 @@ func NewNodeDensity(wh *workloads.WorkloadHelper) *cobra.Command { var podsPerNode int var podReadyThreshold time.Duration var containerImage string + var metricsProfiles []string cmd := &cobra.Command{ Use: "node-density", Short: "Runs node-density workload", @@ -45,12 +46,13 @@ func NewNodeDensity(wh *workloads.WorkloadHelper) *cobra.Command { os.Setenv("CONTAINER_IMAGE", containerImage) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } cmd.Flags().IntVar(&podsPerNode, "pods-per-node", 245, "Pods per node") cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 15*time.Second, "Pod ready timeout threshold") cmd.Flags().StringVar(&containerImage, "container-image", "gcr.io/google_containers/pause:3.1", "Container image") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") return cmd } diff --git a/pvc-density.go b/pvc-density.go index 9ff92c41..88884030 100644 --- a/pvc-density.go +++ b/pvc-density.go @@ -39,7 +39,7 @@ var dynamicStorageProvisioners = map[string]string{ func NewPVCDensity(wh *workloads.WorkloadHelper) *cobra.Command { var iterations int - var storageProvisioners []string + var storageProvisioners, metricsProfiles []string var claimSize string var containerImage string provisioner := "aws" @@ -64,7 +64,7 @@ func NewPVCDensity(wh *workloads.WorkloadHelper) *cobra.Command { os.Setenv("STORAGE_PROVISIONER", fmt.Sprint(dynamicStorageProvisioners[provisioner])) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -73,6 +73,6 @@ func NewPVCDensity(wh *workloads.WorkloadHelper) *cobra.Command { cmd.Flags().StringVar(&provisioner, "provisioner", provisioner, fmt.Sprintf("[%s]", strings.Join(storageProvisioners, " "))) cmd.Flags().StringVar(&claimSize, "claim-size", "256Mi", "claim-size=256Mi") cmd.Flags().StringVar(&containerImage, "container-image", "gcr.io/google_containers/pause:3.1", "Container image") - + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") return cmd } diff --git a/test/ocp/custom-metrics.yml b/test/ocp/custom-metrics.yml new file mode 100644 index 00000000..d87dd80b --- /dev/null +++ b/test/ocp/custom-metrics.yml @@ -0,0 +1,9 @@ +- query: process_resident_memory_bytes{job="prometheus-k8s"} + metricName: prometheusRSS + +- query: irate(process_cpu_seconds_total{job="prometheus-k8s"}[2m]) and on (job) topk(2,avg_over_time(process_cpu_seconds_total{job="prometheus-k8s"}[{{.elapsed}}:])) + metricName: top2PrometheusCPU + +- query: prometheus_build_info + metricName: prometheusBuildInfo + instant: true diff --git a/test/ocp/metrics-endpoints.yaml b/test/ocp/metrics-endpoints.yaml index 662c0f3b..c2cbce39 100644 --- a/test/ocp/metrics-endpoints.yaml +++ b/test/ocp/metrics-endpoints.yaml @@ -1,6 +1,6 @@ - endpoint: http://localhost:9090 metrics: - - metrics.yml + - custom-metrics.yml alerts: - alerts.yml indexer: @@ -10,7 +10,7 @@ type: opensearch - endpoint: http://localhost:9090 metrics: - - metrics.yml + - custom-metrics.yml indexer: esServers: ["{{.ES_SERVER}}"] insecureSkipVerify: true diff --git a/test/ocp/metrics.yml b/test/ocp/metrics.yml deleted file mode 100644 index c9b887c2..00000000 --- a/test/ocp/metrics.yml +++ /dev/null @@ -1,9 +0,0 @@ -- query: process_resident_memory_bytes{job="prometheus"} - metricName: prometheusRSS - -- query: irate(process_cpu_seconds_total{job="prometheus"}[2m]) and on (job) topk(2,avg_over_time(process_cpu_seconds_total{job="prometheus"}[{{.elapsed}}:])) - metricName: top2PrometheusCPU - -- query: prometheus_build_info - metricName: prometheusBuildInfo - instant: true diff --git a/test/test-ocp.bats b/test/test-ocp.bats index 453a0d91..eca8adac 100755 --- a/test/test-ocp.bats +++ b/test/test-ocp.bats @@ -15,7 +15,7 @@ setup_file() { setup() { export UUID; UUID=$(uuidgen) - export COMMON_FLAGS="--es-server=${ES_SERVER} --es-index=${ES_INDEX} --alerting=true --uuid=${UUID} --qps=5 --burst=5" + export COMMON_FLAGS="--es-server=${ES_SERVER} --es-index=${ES_INDEX} --alerting=true --qps=5 --burst=5" } teardown() { @@ -30,12 +30,12 @@ teardown_file() { } @test "custom-workload as node-density" { - run_cmd kube-burner-ocp init --config=custom-workload.yml ${COMMON_FLAGS} --metrics-endpoint metrics-endpoints.yaml + run_cmd kube-burner-ocp init --config=custom-workload.yml --metrics-endpoint metrics-endpoints.yaml --uuid=${UUID} check_metric_value jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement } @test "node-density: es-indexing=true" { - run_cmd kube-burner-ocp node-density --pods-per-node=75 --pod-ready-threshold=10s ${COMMON_FLAGS} + run_cmd kube-burner-ocp node-density --pods-per-node=75 --pod-ready-threshold=10s --uuid=${UUID} ${COMMON_FLAGS} check_metric_value etcdVersion jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement } @@ -45,18 +45,18 @@ teardown_file() { } @test "cluster-density-ms: metrics-endpoint=true; es-indexing=true" { - run_cmd kube-burner-ocp cluster-density-ms --iterations=1 --churn=false --metrics-endpoint metrics-endpoints.yaml ${COMMON_FLAGS} + run_cmd kube-burner-ocp cluster-density-ms --iterations=1 --churn=false --metrics-endpoint metrics-endpoints.yaml --uuid=${UUID} check_metric_value jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement } @test "cluster-density-v2: profile-type=both; user-metadata=true; es-indexing=true; churning=true; svcLatency=true" { - run_cmd kube-burner-ocp cluster-density-v2 --iterations=5 --churn-duration=1m --churn-delay=5s --profile-type=both ${COMMON_FLAGS} --user-metadata=user-metadata.yml --service-latency + run_cmd kube-burner-ocp cluster-density-v2 --iterations=2 --churn-duration=1m --churn-delay=5s --profile-type=both ${COMMON_FLAGS} --user-metadata=user-metadata.yml --service-latency --uuid=${UUID} check_metric_value cpu-kubelet jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement svcLatencyMeasurement svcLatencyQuantilesMeasurement etcdVersion } -@test "cluster-density-v2: churn-deletion-strategy=gvr" { - run_cmd kube-burner-ocp cluster-density-v2 --iterations=2 --churn=true --churn-duration=1m --churn-delay=10s --churn-deletion-strategy=gvr ${COMMON_FLAGS} - check_metric_value etcdVersion jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement +@test "cluster-density-v2: churn-deletion-strategy=gvr; custom-metrics=true" { + run_cmd kube-burner-ocp cluster-density-v2 --iterations=2 --churn=true --churn-duration=1m --churn-delay=5s --churn-deletion-strategy=gvr --metrics-profile=custom-metrics.yml ${COMMON_FLAGS} --uuid=${UUID} + check_metric_value prometheusRSS jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement } @test "cluster-density-v2: indexing=false; churning=false" { @@ -76,7 +76,7 @@ teardown_file() { } @test "index: local-indexing=true" { - run_cmd kube-burner-ocp index --uuid="${UUID}" --metrics-profile "metrics.yml,metrics.yml" + run_cmd kube-burner-ocp index --uuid=${UUID} --metrics-profile "custom-metrics.yml" } @test "index: metrics-endpoints=true; es-indexing=true" { @@ -84,12 +84,12 @@ teardown_file() { } @test "networkpolicy-multitenant" { - run_cmd kube-burner-ocp networkpolicy-multitenant --iterations 5 ${COMMON_FLAGS} + run_cmd kube-burner-ocp networkpolicy-multitenant --iterations 5 ${COMMON_FLAGS} --uuid=${UUID} } @test "pvc-density" { # Since 'aws' is the chosen storage provisioner, this will only execute successfully if the ocp environment is aws - run_cmd kube-burner-ocp pvc-density --iterations=2 --provisioner=aws ${COMMON_FLAGS} + run_cmd kube-burner-ocp pvc-density --iterations=2 --provisioner=aws ${COMMON_FLAGS} --uuid=${UUID} check_metric_value jobSummary podLatencyMeasurement podLatencyQuantilesMeasurement } diff --git a/udn-density-l3-pods.go b/udn-density-l3-pods.go index fd40eb6c..5b2a28d3 100644 --- a/udn-density-l3-pods.go +++ b/udn-density-l3-pods.go @@ -30,6 +30,7 @@ func NewUDNDensityL3Pods(wh *workloads.WorkloadHelper) *cobra.Command { var churn bool var churnDelay, churnDuration, podReadyThreshold time.Duration var churnDeletionStrategy string + var metricsProfiles []string cmd := &cobra.Command{ Use: "udn-density-l3-pods", Short: "Runs node-density-udn workload", @@ -45,7 +46,7 @@ func NewUDNDensityL3Pods(wh *workloads.WorkloadHelper) *cobra.Command { os.Setenv("POD_READY_THRESHOLD", fmt.Sprintf("%v", podReadyThreshold)) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -57,5 +58,6 @@ func NewUDNDensityL3Pods(wh *workloads.WorkloadHelper) *cobra.Command { cmd.Flags().StringVar(&churnDeletionStrategy, "churn-deletion-strategy", "default", "Churn deletion strategy to use") cmd.Flags().IntVar(&iterations, "iterations", 0, "Iterations") cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 1*time.Minute, "Pod ready timeout threshold") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") return cmd } diff --git a/web-burner.go b/web-burner.go index 68d42c38..6c136d29 100644 --- a/web-burner.go +++ b/web-burner.go @@ -29,6 +29,7 @@ func NewWebBurner(wh *workloads.WorkloadHelper, variant string) *cobra.Command { var bfd, crd, icni, probe, sriov bool var bridge string var podReadyThreshold time.Duration + var metricsProfiles []string cmd := &cobra.Command{ Use: variant, Short: fmt.Sprintf("Runs %v workload", variant), @@ -44,7 +45,7 @@ func NewWebBurner(wh *workloads.WorkloadHelper, variant string) *cobra.Command { os.Setenv("SRIOV", fmt.Sprint(sriov)) }, Run: func(cmd *cobra.Command, args []string) { - setMetrics(cmd, "metrics.yml") + setMetrics(cmd, metricsProfiles) wh.Run(cmd.Name()) }, } @@ -57,5 +58,6 @@ func NewWebBurner(wh *workloads.WorkloadHelper, variant string) *cobra.Command { cmd.Flags().BoolVar(&probe, "probe", false, "Enable readiness probes") cmd.Flags().BoolVar(&sriov, "sriov", true, "Enable SRIOV") cmd.Flags().StringVar(&bridge, "bridge", "br-ex", "Data-plane bridge") + cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use") return cmd }