diff --git a/README.md b/README.md index f8af913a..23fcc1ad 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Documentation is [available here](https://kube-burner.github.io/kube-burner-ocp/ Some of the benefits the OCP wrapper provides are: - Simplified execution of the supported workloads. (Only some flags are required) -- Indexes OpenShift metadata along with the Benchmark result. This document can be found with the following query: `uuid: AND metricName.keyword: "clusterMetadata"` +- Adds OpenShift metadata to the generated documents. - Prevents modifying configuration files to tweak some of the parameters of the workloads. - Discovers the Prometheus URL and authentication token, so the user does not have to perform those operations before using them. - Workloads configuration is directly embedded in the binary. @@ -83,7 +83,8 @@ kube-burner-ocp cluster-density-v2 --iterations=1 --churn-duration=2m0s --churn- ``` ### metrics-endpoints.yaml -``` + +```yaml - endpoint: prometheus-k8s-openshift-monitoring.apps.rook.devshift.org metrics: - metrics.yml @@ -211,11 +212,13 @@ kube-burner-ocp egressip --addresses-per-iteration=1 --iterations=1 --external-s With the command above, each namespace has one pod with a dedicated egress IP. OVN will use this dedicated egress IP for the http requests from client pod's to 10.0.34.43. ## Web-burner workloads + This workload is meant to emulate some telco specific workloads. Before running *web-burner-node-density* or *web-burner-cluster-density* load the environment with *web-burner-init* first (without the garbage collection flag: `--gc=false`). Pre-requisites: - - At least two worker nodes - - At least one of the worker nodes must have the `node-role.kubernetes.io/worker-spk` label + +- At least two worker nodes +- At least one of the worker nodes must have the `node-role.kubernetes.io/worker-spk` label ### web-burner-init @@ -226,12 +229,14 @@ Pre-requisites: - 1 emulated lb pod on each namespace for bfd session ### web-burner-node-density + - 35 app-ns - 3 app pods and services on each namespace - 35 normal-ns - 1 service with 60 normal pod endpoints on each namespace ### web-burner-cluster-density + - 20 normal-ns - 30 configmaps, 38 secrets, 38 normal pods and services, 5 deployments with 2 replica pods on each namespace - 35 served-ns @@ -244,8 +249,10 @@ Pre-requisites: - 29 service(15 ports each) with 4 pod endpoints, 29 service(15 ports each) with 6 pod endpoints ## Custom Workload: Bring your own workload + To kickstart kube-burner-ocp with a custom workload, `init` becomes your go-to command. This command is equipped with flags that enable to seamlessly integrate and run your personalized workloads. Here's a breakdown of the flags accepted by the init command: -``` + +```console $ kube-burner-ocp init --help Runs custom workload @@ -294,10 +301,9 @@ jobs: replicas: inputVars: : - ``` -You can start from scratch or explore pre-built workloads in the /config folder, offering a variety of examples used by kube-burner-ocp. Dive into the details of each section in the template to tailor the workload precisely to your requirements. Experiment, iterate, and discover the optimal configuration for your workload to seamlessly integrate with kube-burner-ocp. +You can start from scratch or explore pre-built workloads in the /config folder, offering a variety of examples used by kube-burner-ocp. Dive into the details of each section in the template to tailor the workload precisely to your requirements. Experiment, iterate, and discover the optimal configuration for your workload to seamlessly integrate with kube-burner-ocp. ## Index diff --git a/cmd/config/metrics-aggregated.yml b/cmd/config/metrics-aggregated.yml index 2207f5e1..eeed7654 100644 --- a/cmd/config/metrics-aggregated.yml +++ b/cmd/config/metrics-aggregated.yml @@ -147,27 +147,34 @@ - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="worker",role!="infra"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Workers instant: true + captureStart: true - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Masters instant: true + captureStart: true - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Infra instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "worker",role != "infra" } ) by ( id ) metricName: cgroupCPUSeconds-Workers instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "master" } ) by ( id ) metricName: cgroupCPUSeconds-Masters instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "infra" } ) by ( id ) metricName: cgroupCPUSeconds-Infra instant: true + captureStart: true - query: sum( container_cpu_usage_seconds_total{container!~"POD|",namespace=~"openshift-.*"} ) by (namespace) metricName: cgroupCPUSeconds-namespaces instant: true + captureStart: true diff --git a/cmd/config/metrics-egressip.yml b/cmd/config/metrics-egressip.yml index 4e149292..7a9c3385 100644 --- a/cmd/config/metrics-egressip.yml +++ b/cmd/config/metrics-egressip.yml @@ -159,27 +159,34 @@ - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="worker",role!="infra"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Workers instant: true + captureStart: true - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Masters instant: true + captureStart: true - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Infra instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "worker",role != "infra" } ) by ( id ) metricName: cgroupCPUSeconds-Workers instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "master" } ) by ( id ) metricName: cgroupCPUSeconds-Masters instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "infra" } ) by ( id ) metricName: cgroupCPUSeconds-Infra instant: true + captureStart: true - query: sum( container_cpu_usage_seconds_total{container!~"POD|",namespace=~"openshift-.*"} ) by (namespace) metricName: cgroupCPUSeconds-namespaces instant: true + captureStart: true diff --git a/cmd/config/metrics-report.yml b/cmd/config/metrics-report.yml index 4df1fd63..6133c193 100644 --- a/cmd/config/metrics-report.yml +++ b/cmd/config/metrics-report.yml @@ -16,7 +16,7 @@ metricName: max-mutating-apicalls-latency instant: true - # Kubelet & CRI-O +# Kubelet & CRI-O # Average and max of the CPU usage from all worker's kubelet - query: avg(avg_over_time(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m])[{{.elapsed}}:]) and on (node) kube_node_role{role="worker"}) @@ -376,27 +376,34 @@ - query: sum(node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="worker",role!="infra"}, "instance", "$1", "node", "(.+)")) by (mode) metricName: nodeCPUSeconds-Workers instant: true + captureStart: true - query: sum(node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) by (mode) metricName: nodeCPUSeconds-Masters instant: true + captureStart: true - query: sum(node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (mode) metricName: nodeCPUSeconds-Infra instant: true + captureStart: true - query: sum(container_cpu_usage_seconds_total{id=~"/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice"} and on (node) kube_node_role{role="worker",role!="infra"}) by (id) metricName: cgroupCPUSeconds-Workers instant: true + captureStart: true - query: sum(container_cpu_usage_seconds_total{id=~"/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice"} and on (node) kube_node_role{role="master"}) by (id) metricName: cgroupCPUSeconds-Masters instant: true + captureStart: true - query: sum(container_cpu_usage_seconds_total{id=~"/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice"} and on (node) kube_node_role{role="infra"}) by (id) metricName: cgroupCPUSeconds-Infra instant: true + captureStart: true - query: sum(container_cpu_usage_seconds_total{container!~"POD|",namespace=~"openshift-.*"}) by (namespace) metricName: cgroupCPUSeconds-namespaces instant: true + captureStart: true diff --git a/cmd/config/metrics.yml b/cmd/config/metrics.yml index a88f4392..0c803e6f 100644 --- a/cmd/config/metrics.yml +++ b/cmd/config/metrics.yml @@ -138,27 +138,34 @@ - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="worker",role!="infra"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Workers instant: true + captureStart: true - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Masters instant: true + captureStart: true - query: sum( node_cpu_seconds_total and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)") ) by (mode) metricName: nodeCPUSeconds-Infra instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "worker",role != "infra" } ) by ( id ) metricName: cgroupCPUSeconds-Workers instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "master" } ) by ( id ) metricName: cgroupCPUSeconds-Masters instant: true + captureStart: true - query: sum ( container_cpu_usage_seconds_total { id =~ "/system.slice|/system.slice/kubelet.service|/system.slice/ovs-vswitchd.service|/system.slice/crio.service|/kubepods.slice" } and on (node) kube_node_role{ role = "infra" } ) by ( id ) metricName: cgroupCPUSeconds-Infra instant: true + captureStart: true - query: sum( container_cpu_usage_seconds_total{container!~"POD|",namespace=~"openshift-.*"} ) by (namespace) metricName: cgroupCPUSeconds-namespaces instant: true + captureStart: true diff --git a/cmd/ocp.go b/cmd/ocp.go index be106575..022a3e88 100644 --- a/cmd/ocp.go +++ b/cmd/ocp.go @@ -78,6 +78,7 @@ func openShiftCmd() *cobra.Command { workloadConfig.ConfigDir = configDir kubeClientProvider := config.NewKubeClientProvider("", "") wh = workloads.NewWorkloadHelper(workloadConfig, ocpConfig, kubeClientProvider) + wh.MetricsMetadata = make(map[string]interface{}) envVars := map[string]string{ "UUID": workloadConfig.UUID, "ES_SERVER": esServer, diff --git a/common.go b/common.go index 2a3110d0..79cf5bc5 100644 --- a/common.go +++ b/common.go @@ -25,7 +25,6 @@ import ( "github.com/spf13/cobra" ) - func setMetrics(cmd *cobra.Command, metricsProfile string) { var metricsProfiles []string profileType, _ := cmd.Root().PersistentFlags().GetString("profile-type") @@ -60,14 +59,5 @@ func GatherMetadata(wh *workloads.WorkloadHelper, alerting bool) error { if err != nil { return err } - wh.Config.UUID = wh.UUID - wh.MetricsMetadata = map[string]interface{}{ - "platform": wh.ClusterMetadata.Platform, - "ocpVersion": wh.ClusterMetadata.OCPVersion, - "ocpMajorVersion": wh.ClusterMetadata.OCPMajorVersion, - "k8sVersion": wh.ClusterMetadata.K8SVersion, - "totalNodes": wh.ClusterMetadata.TotalNodes, - "sdnType": wh.ClusterMetadata.SDNType, - } return nil } diff --git a/egressip.go b/egressip.go index 12c44751..42845e38 100644 --- a/egressip.go +++ b/egressip.go @@ -15,11 +15,11 @@ package ocp import ( + "context" + "encoding/json" "fmt" "net" - "context" "os" - "encoding/json" "strings" "time" @@ -34,7 +34,7 @@ import ( // get egress IP cidr, node IPs from worker node annotations func getEgressIPCidrNodeIPs() ([]string, string) { kubeClientProvider := config.NewKubeClientProvider("", "") - clientSet, _:= kubeClientProvider.ClientSet(0, 0) + clientSet, _ := kubeClientProvider.ClientSet(0, 0) workers, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { log.Errorf("Error retrieving workers: %v", err) @@ -55,7 +55,7 @@ func getEgressIPCidrNodeIPs() ([]string, string) { } // For cloud based OCP deployedments, egress IP cidr is added as part of cloud.network.openshift.io/egress-ipconfig annotation // For baremetal, read the cidr from k8s.ovn.org/node-primary-ifaddr - if egressIPCidr == "" { + if egressIPCidr == "" { eipconfig, exist := worker.ObjectMeta.Annotations["cloud.network.openshift.io/egress-ipconfig"] if exist { var items []map[string]interface{} @@ -93,21 +93,21 @@ func getFirstUsableAddr(cidr string) uint32 { } // Calculate the first usable IP address by skipping first 4 addresses. - // For example, OVN didn't assign eip to node when eip was in between 10.0.0.0 and 10.0.0.3 for cidr 10.0.0.0/19 + // For example, OVN didn't assign eip to node when eip was in between 10.0.0.0 and 10.0.0.3 for cidr 10.0.0.0/19 firstUsableIP := make(net.IP, len(networkBytes)) copy(firstUsableIP, networkBytes) firstUsableIP[3] += 4 // Increment the last byte by 1 for the first usable IP address // Output the network address and the first usable IP address in CIDR notation - baseAddrInt, err := ipconv.IPv4ToInt(firstUsableIP) + baseAddrInt, err := ipconv.IPv4ToInt(firstUsableIP) if err != nil { log.Fatal("Error converting IP to int: ", err) os.Exit(1) - } - return baseAddrInt + } + return baseAddrInt } -// egress IPs and node IPs will be in same cidr. So we need to exclude node IPs from CIDR to generate list of avaialble egress IPs. +// egress IPs and node IPs will be in same cidr. So we need to exclude node IPs from CIDR to generate list of avaialble egress IPs. func generateEgressIPs(numJobIterations int, addressesPerIteration int, externalServerIP string) { nodeIPs, egressIPCidr := getEgressIPCidrNodeIPs() @@ -122,16 +122,16 @@ func generateEgressIPs(numJobIterations int, addressesPerIteration int, external for _, nodeip := range nodeIPs { nodeipuint32, err := ipconv.IPv4ToInt(net.ParseIP(nodeip)) if err != nil { - log.Fatal("Error: ", err) + log.Fatal("Error: ", err) os.Exit(1) - } + } nodeMap[nodeipuint32] = true } // Generate ip addresses from CIDR by excluding nodeIPs // Extra iterations needed in for loop if we come across node IPs while generating egress IP list var newAddr uint32 - for i := 0; i < ((numJobIterations * addressesPerIteration) + len(nodeIPs) ); i++ { + for i := 0; i < ((numJobIterations * addressesPerIteration) + len(nodeIPs)); i++ { newAddr = baseAddrInt + uint32(i) if !nodeMap[newAddr] { addrSlice = append(addrSlice, ipconv.IntToIPv4(newAddr).String()) @@ -146,11 +146,10 @@ func generateEgressIPs(numJobIterations int, addressesPerIteration int, external os.Setenv("EIP_ADDRESSES", strings.Join(addrSlice, " ")) } - // NewClusterDensity holds cluster-density workload func NewEgressIP(wh *workloads.WorkloadHelper, variant string) *cobra.Command { var iterations, addressesPerIteration int - var externalServerIP string + var externalServerIP string var podReadyThreshold time.Duration cmd := &cobra.Command{ Use: variant,