Merge branch 'main' into dev-es-removal

cluster-density.go

@@ -33,6 +33,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm
 	var churnDelay, churnDuration time.Duration
 	var churnDeletionStrategy string
 	var podReadyThreshold time.Duration
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:   variant,
 		Short: fmt.Sprintf("Runs %v workload", variant),
@@ -60,7 +61,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm
 					log.Errorf("image-registry deployment is not deployed")
 				}
 			}
-			setMetrics(cmd, "metrics-aggregated.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
@@ -73,6 +74,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm
 	cmd.Flags().IntVar(&churnPercent, "churn-percent", 10, "Percentage of job iterations that kube-burner will churn each round")
 	cmd.Flags().StringVar(&churnDeletionStrategy, "churn-deletion-strategy", "default", "Churn deletion strategy to use")
 	cmd.Flags().BoolVar(&svcLatency, "service-latency", false, "Enable service latency measurement")
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-aggregated.yml"}, "Comma separated list of metrics profiles to use")
 	cmd.MarkFlagRequired("iterations")
 	return cmd
 }

cmd/config/metrics-aggregated.yml

@@ -12,14 +12,6 @@
 - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH"}[2m])) by (verb,resource,code) > 0
   metricName: APIRequestRate
 
-# Kubeproxy and OVN service sync latency
-
-- query: histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[2m])) by (le)) > 0
-  metricName: serviceSyncLatency
-
-- query: histogram_quantile(0.99, sum(rate(ovnkube_master_network_programming_duration_seconds_bucket{kind="service"}[2m])) by (le))
-  metricName: serviceSyncLatency
-
 # Containers & pod metrics
 
 - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|network-node-identity|multus|.*apiserver|authentication|.*controller-manager|.*scheduler|image-registry|operator-lifecycle-manager)|cilium|stackrox|calico.*|tigera.*"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0

cmd/config/metrics-egressip.yml

@@ -24,14 +24,6 @@
 - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH"}[2m])) by (verb,resource,code) > 0
   metricName: APIRequestRate
 
-# Kubeproxy and OVN service sync latency
-
-- query: histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[2m])) by (le)) > 0
-  metricName: serviceSyncLatency
-
-- query: histogram_quantile(0.99, sum(rate(ovnkube_master_network_programming_duration_seconds_bucket{kind="service"}[2m])) by (le))
-  metricName: serviceSyncLatency
-
 # Containers & pod metrics
 
 - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|network-node-identity|multus|.*apiserver|authentication|.*controller-manager|.*scheduler|image-registry|operator-lifecycle-manager)|cilium|stackrox|calico.*|tigera.*"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0

cmd/config/metrics.yml

@@ -12,14 +12,6 @@
 - query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH"}[2m])) by (verb,resource,code) > 0
   metricName: APIRequestRate
 
-# Kubeproxy and OVN service sync latency
-
-- query: histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket[2m])) by (le)) > 0
-  metricName: serviceSyncLatency
-
-- query: histogram_quantile(0.99, sum(rate(ovnkube_master_network_programming_duration_seconds_bucket{kind="service"}[2m])) by (le))
-  metricName: serviceSyncLatency
-
 # Containers & pod metrics
 
 - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!~"POD|",namespace=~"openshift-(etcd|.*apiserver|monitoring|user-workload-monitoring|ovn-kubernetes|network-node-identity|multus|sdn|ingress|.*controller-manager|.*scheduler)|cilium|stackrox|calico.*|tigera.*"}[2m]) * 100) by (container, pod, namespace, node)) > 0

common.go

@@ -28,16 +28,13 @@ import (
 
 var clusterMetadata ocpmetadata.ClusterMetadata
 
-func setMetrics(cmd *cobra.Command, metricsProfile string) {
-	var metricsProfiles []string
+func setMetrics(cmd *cobra.Command, metricsProfiles []string) {
 	profileType, _ := cmd.Root().PersistentFlags().GetString("profile-type")
 	switch ProfileType(profileType) {
 	case Reporting:
 		metricsProfiles = []string{"metrics-report.yml"}
-	case Regular:
-		metricsProfiles = []string{metricsProfile}
 	case Both:
-		metricsProfiles = []string{"metrics-report.yml", metricsProfile}
+		metricsProfiles = append(metricsProfiles, "metrics-report.yml")
 	}
 	os.Setenv("METRICS", strings.Join(metricsProfiles, ","))
 }

crd-scale.go

@@ -25,6 +25,7 @@ import (
 // NewCrdScale holds the crd-scale workload
 func NewCrdScale(wh *workloads.WorkloadHelper) *cobra.Command {
 	var iterations int
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:          "crd-scale",
 		Short:        "Runs crd-scale workload",
@@ -33,11 +34,12 @@ func NewCrdScale(wh *workloads.WorkloadHelper) *cobra.Command {
 			os.Setenv("JOB_ITERATIONS", fmt.Sprint(iterations))
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics-aggregated.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
 	cmd.Flags().IntVar(&iterations, "iterations", 0, "Number of CRDs to create")
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-aggregated.yml"}, "Comma separated list of metrics profiles to use")
 	cmd.MarkFlagRequired("iterations")
 	return cmd
 }

egressip.go

@@ -149,6 +149,7 @@ func NewEgressIP(wh *workloads.WorkloadHelper, variant string) *cobra.Command {
 	var iterations, addressesPerIteration int
 	var externalServerIP string
 	var podReadyThreshold time.Duration
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:   variant,
 		Short: fmt.Sprintf("Runs %v workload", variant),
@@ -160,14 +161,15 @@ func NewEgressIP(wh *workloads.WorkloadHelper, variant string) *cobra.Command {
 			generateEgressIPs(iterations, addressesPerIteration, externalServerIP)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics-egressip.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
 	cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 2*time.Minute, "Pod ready timeout threshold")
 	cmd.Flags().IntVar(&iterations, "iterations", 0, fmt.Sprintf("%v iterations", variant))
 	cmd.Flags().StringVar(&externalServerIP, "external-server-ip", "", "External server IP address")
 	cmd.Flags().IntVar(&addressesPerIteration, "addresses-per-iteration", 1, fmt.Sprintf("%v iterations", variant))
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics-egressip.yml"}, "Comma separated list of metrics profiles to use")
 	cmd.MarkFlagRequired("iterations")
 	cmd.MarkFlagRequired("external-server-ip")
 	return cmd

index.go

@@ -18,7 +18,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/cloud-bulldozer/go-commons/indexers"
@@ -35,7 +34,8 @@ import (
 
 // NewIndex orchestrates indexing for ocp wrapper
 func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobra.Command {
-	var metricsProfile, jobName string
+	var jobName string
+	var metricsProfiles []string
 	var start, end int64
 	var userMetadata, metricsDirectory string
 	var prometheusStep time.Duration
@@ -70,9 +70,6 @@ func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobr
 					log.Fatal("Error obtaining prometheus information from cluster: ", err.Error())
 				}
 			}
-			metricsProfiles := strings.FieldsFunc(metricsProfile, func(r rune) bool {
-				return r == ',' || r == ' '
-			})
 			indexer = config.MetricsEndpoint{
 				Endpoint:      prometheusURL,
 				Token:         prometheusToken,
@@ -148,7 +145,7 @@ func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobr
 			burner.IndexJobSummary([]burner.JobSummary{jobSummary}, indexerValue)
 		},
 	}
-	cmd.Flags().StringVarP(&metricsProfile, "metrics-profile", "m", "metrics.yml", "comma-separated list of metric profiles")
+	cmd.Flags().StringSliceVarP(&metricsProfiles, "metrics-profile", "m", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use")
 	cmd.Flags().StringVar(&metricsDirectory, "metrics-directory", "collected-metrics", "Directory to dump the metrics files in, when using default local indexing")
 	cmd.Flags().DurationVar(&prometheusStep, "step", 30*time.Second, "Prometheus step size")
 	cmd.Flags().Int64Var(&start, "start", time.Now().Unix()-3600, "Epoch start time")

networkpolicy.go

@@ -29,6 +29,7 @@ func NewNetworkPolicy(wh *workloads.WorkloadHelper, variant string) *cobra.Comma
 	var churn bool
 	var churnDelay, churnDuration time.Duration
 	var churnDeletionStrategy string
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:   variant,
 		Short: fmt.Sprintf("Runs %v workload", variant),
@@ -42,7 +43,7 @@ func NewNetworkPolicy(wh *workloads.WorkloadHelper, variant string) *cobra.Comma
 			os.Setenv("CHURN_DELETION_STRATEGY", churnDeletionStrategy)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
@@ -53,6 +54,7 @@ func NewNetworkPolicy(wh *workloads.WorkloadHelper, variant string) *cobra.Comma
 	cmd.Flags().DurationVar(&churnDelay, "churn-delay", 2*time.Minute, "Time to wait between each churn")
 	cmd.Flags().IntVar(&churnPercent, "churn-percent", 10, "Percentage of job iterations that kube-burner will churn each round")
 	cmd.Flags().StringVar(&churnDeletionStrategy, "churn-deletion-strategy", "default", "Churn deletion strategy to use")
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use")
 	cmd.MarkFlagRequired("iterations")
 	return cmd
 }

node-density-cni.go

@@ -32,6 +32,7 @@ func NewNodeDensityCNI(wh *workloads.WorkloadHelper) *cobra.Command {
 	var namespacedIterations, svcLatency bool
 	var podReadyThreshold time.Duration
 	var iterationsPerNamespace int
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:          "node-density-cni",
 		Short:        "Runs node-density-cni workload",
@@ -49,7 +50,7 @@ func NewNodeDensityCNI(wh *workloads.WorkloadHelper) *cobra.Command {
 			os.Setenv("SVC_LATENCY", strconv.FormatBool(svcLatency))
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
@@ -58,5 +59,6 @@ func NewNodeDensityCNI(wh *workloads.WorkloadHelper) *cobra.Command {
 	cmd.Flags().BoolVar(&namespacedIterations, "namespaced-iterations", true, "Namespaced iterations")
 	cmd.Flags().IntVar(&iterationsPerNamespace, "iterations-per-namespace", 1000, "Iterations per namespace")
 	cmd.Flags().BoolVar(&svcLatency, "service-latency", false, "Enable service latency measurement")
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use")
 	return cmd
 }

node-density-heavy.go

@@ -31,6 +31,7 @@ func NewNodeDensityHeavy(wh *workloads.WorkloadHelper) *cobra.Command {
 	var podReadyThreshold, probesPeriod time.Duration
 	var namespacedIterations bool
 	var iterationsPerNamespace int
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:          "node-density-heavy",
 		Short:        "Runs node-density-heavy workload",
@@ -49,7 +50,7 @@ func NewNodeDensityHeavy(wh *workloads.WorkloadHelper) *cobra.Command {
 			os.Setenv("ITERATIONS_PER_NAMESPACE", fmt.Sprint(iterationsPerNamespace))
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
@@ -58,5 +59,6 @@ func NewNodeDensityHeavy(wh *workloads.WorkloadHelper) *cobra.Command {
 	cmd.Flags().IntVar(&podsPerNode, "pods-per-node", 245, "Pods per node")
 	cmd.Flags().BoolVar(&namespacedIterations, "namespaced-iterations", true, "Namespaced iterations")
 	cmd.Flags().IntVar(&iterationsPerNamespace, "iterations-per-namespace", 1000, "Iterations per namespace")
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use")
 	return cmd
 }

node-density.go

@@ -30,6 +30,7 @@ func NewNodeDensity(wh *workloads.WorkloadHelper) *cobra.Command {
 	var podsPerNode int
 	var podReadyThreshold time.Duration
 	var containerImage string
+	var metricsProfiles []string
 	cmd := &cobra.Command{
 		Use:          "node-density",
 		Short:        "Runs node-density workload",
@@ -45,12 +46,13 @@ func NewNodeDensity(wh *workloads.WorkloadHelper) *cobra.Command {
 			os.Setenv("CONTAINER_IMAGE", containerImage)
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
 	cmd.Flags().IntVar(&podsPerNode, "pods-per-node", 245, "Pods per node")
 	cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 15*time.Second, "Pod ready timeout threshold")
 	cmd.Flags().StringVar(&containerImage, "container-image", "gcr.io/google_containers/pause:3.1", "Container image")
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use")
 	return cmd
 }

pvc-density.go

@@ -39,7 +39,7 @@ var dynamicStorageProvisioners = map[string]string{
 func NewPVCDensity(wh *workloads.WorkloadHelper) *cobra.Command {
 
 	var iterations int
-	var storageProvisioners []string
+	var storageProvisioners, metricsProfiles []string
 	var claimSize string
 	var containerImage string
 	provisioner := "aws"
@@ -64,7 +64,7 @@ func NewPVCDensity(wh *workloads.WorkloadHelper) *cobra.Command {
 			os.Setenv("STORAGE_PROVISIONER", fmt.Sprint(dynamicStorageProvisioners[provisioner]))
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			setMetrics(cmd, "metrics.yml")
+			setMetrics(cmd, metricsProfiles)
 			wh.Run(cmd.Name())
 		},
 	}
@@ -73,6 +73,6 @@ func NewPVCDensity(wh *workloads.WorkloadHelper) *cobra.Command {
 	cmd.Flags().StringVar(&provisioner, "provisioner", provisioner, fmt.Sprintf("[%s]", strings.Join(storageProvisioners, " ")))
 	cmd.Flags().StringVar(&claimSize, "claim-size", "256Mi", "claim-size=256Mi")
 	cmd.Flags().StringVar(&containerImage, "container-image", "gcr.io/google_containers/pause:3.1", "Container image")
-
+	cmd.Flags().StringSliceVar(&metricsProfiles, "metrics-profile", []string{"metrics.yml"}, "Comma separated list of metrics profiles to use")
 	return cmd
 }

test/ocp/custom-metrics.yml

@@ -0,0 +1,9 @@
+- query: process_resident_memory_bytes{job="prometheus-k8s"}
+  metricName: prometheusRSS
+
+- query: irate(process_cpu_seconds_total{job="prometheus-k8s"}[2m]) and on (job) topk(2,avg_over_time(process_cpu_seconds_total{job="prometheus-k8s"}[{{.elapsed}}:]))
+  metricName: top2PrometheusCPU
+
+- query: prometheus_build_info
+  metricName: prometheusBuildInfo
+  instant: true

test/ocp/metrics-endpoints.yaml

@@ -1,6 +1,6 @@
 - endpoint: http://localhost:9090
   metrics:
-    - metrics.yml
+    - custom-metrics.yml
   alerts:
     - alerts.yml
   indexer:
@@ -10,7 +10,7 @@
       type: opensearch
 - endpoint: http://localhost:9090
   metrics:
-    - metrics.yml
+    - custom-metrics.yml
   indexer:
       esServers: ["{{.ES_SERVER}}"]
       insecureSkipVerify: true