From 150dd0b2904ca03e102380a0fe0beefb59f7c033 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 26 Sep 2024 10:16:11 +0200 Subject: [PATCH 1/4] Subset of metadata fields Signed-off-by: Raul Sevilla --- common.go | 6 +++++- go.mod | 2 ++ go.sum | 4 ++-- index.go | 4 ++-- test/helpers.bash | 2 +- workers-scale.go | 7 ++++--- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/common.go b/common.go index 6ed51322..7621f734 100644 --- a/common.go +++ b/common.go @@ -63,6 +63,10 @@ func GatherMetadata(wh *workloads.WorkloadHelper, alerting bool) error { if err != nil { return err } - json.Unmarshal(jsonData, &wh.Metadata) + json.Unmarshal(jsonData, &wh.SummaryMetadata) + wh.MetricsMetadata = map[string]interface{}{ + "ocpMajorVersion": clusterMetadata.OCPMajorVersion, + "ocpVersion": clusterMetadata.OCPVersion, + } return nil } diff --git a/go.mod b/go.mod index a19ab129..69192af1 100644 --- a/go.mod +++ b/go.mod @@ -85,3 +85,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/kube-burner/kube-burner => github.com/rsevilla87/kube-burner v1.7.6-0.20240926073724-2b0caacee619 diff --git a/go.sum b/go.sum index 65539519..af151742 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kube-burner/kube-burner v1.10.7 h1:QcEQt0x7AxoPT9VTzyO1TZNAdQYAKk4Fz9NJ/uSunjA= -github.com/kube-burner/kube-burner v1.10.7/go.mod h1:NKtSusPSi2rmS5kYAYC1A+JTsC4RsI0VIHFt6ET4EJE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -486,6 +484,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rsevilla87/kube-burner v1.7.6-0.20240926073724-2b0caacee619 h1:B4QRIyyym1mo5dS8hNr8zzbDDghjbh8oUdb8zZukZnY= +github.com/rsevilla87/kube-burner v1.7.6-0.20240926073724-2b0caacee619/go.mod h1:NKtSusPSi2rmS5kYAYC1A+JTsC4RsI0VIHFt6ET4EJE= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= diff --git a/index.go b/index.go index 6b2b76bf..cbcbaf64 100644 --- a/index.go +++ b/index.go @@ -105,7 +105,7 @@ func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobr ConfigSpec: &workloads.ConfigSpec, MetricsEndpoint: *metricsEndpoint, UserMetaData: userMetadata, - RawMetadata: metadata, + MetricsMetadata: metadata, }) for _, prometheusClient := range metricsScraper.PrometheusClients { prometheusJob := prometheus.Job{ @@ -137,7 +137,7 @@ func NewIndex(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata) *cobr JobConfig: config.Job{ Name: jobName, }, - Metadata: metricsScraper.Metadata, + Metadata: metricsScraper.SummaryMetadata, MetricName: "jobSummary", Version: fmt.Sprintf("%v@%v", version.Version, version.GitCommit), Passed: rc == 0, diff --git a/test/helpers.bash b/test/helpers.bash index 9ceab3ad..bbf7fc46 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -90,7 +90,7 @@ print_events() { check_metric_value() { sleep 3s # There's some delay on the documents to show up in OpenSearch for metric in "${@}"; do - endpoint="${ES_SERVER}/${ES_INDEX}/_search?q=uuid.keyword:${UUID}+AND+metricName.keyword:${metric}" + endpoint="${ES_SERVER}/${ES_INDEX}/_search?q=uuid.keyword:${UUID}+AND+metricName.keyword:${metric}+AND+ocpVersion.keyword:*" RESULT=$(curl -sS ${endpoint} | jq '.hits.total.value // error') RETURN_CODE=$? if [ "${RETURN_CODE}" -ne 0 ]; then diff --git a/workers-scale.go b/workers-scale.go index 47fa8725..c51b541b 100644 --- a/workers-scale.go +++ b/workers-scale.go @@ -119,7 +119,8 @@ func NewWorkersScale(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata ConfigSpec: &workloads.ConfigSpec, MetricsEndpoint: *metricsEndpoint, UserMetaData: userMetadata, - RawMetadata: metadata, + MetricsMetadata: metadata, + SummaryMetadata: metadata, }) var indexerValue indexers.Indexer for _, value := range metricsScraper.IndexerList { @@ -130,7 +131,7 @@ func NewWorkersScale(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata scenario.OrchestrateWorkload(wscale.ScaleConfig{ UUID: uuid, AdditionalWorkerNodes: additionalWorkerNodes, - Metadata: metricsScraper.Metadata, + Metadata: metricsScraper.MetricsMetadata, Indexer: indexerValue, GC: gc, ScaleEventEpoch: scaleEventEpoch, @@ -162,7 +163,7 @@ func NewWorkersScale(metricsEndpoint *string, ocpMetaAgent *ocpmetadata.Metadata JobConfig: config.Job{ Name: wscale.JobName, }, - Metadata: metricsScraper.Metadata, + Metadata: metricsScraper.SummaryMetadata, MetricName: "jobSummary", Version: fmt.Sprintf("%v@%v", version.Version, version.GitCommit), Passed: rc == 0, From 8521609dc9b08dc399819e41258728e88eed5dc5 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Thu, 26 Sep 2024 10:19:22 +0200 Subject: [PATCH 2/4] Update docs Signed-off-by: Raul Sevilla --- README.md | 2 +- go.mod | 2 +- go.sum | 4 ++-- test/helpers.bash | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d56deb06..0f242598 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Documentation is [available here](https://kube-burner.github.io/kube-burner-ocp/ Some of the benefits the OCP wrapper provides are: - Simplified execution of the supported workloads. (Only some flags are required) -- Adds OpenShift metadata to the generated documents. +- Adds OpenShift metadata to generated jobSummary and a small subset of metadata fields to the remaining metrics. - Prevents modifying configuration files to tweak some of the parameters of the workloads. - Discovers the Prometheus URL and authentication token, so the user does not have to perform those operations before using them. - Workloads configuration is directly embedded in the binary. diff --git a/go.mod b/go.mod index 69192af1..6dc03b71 100644 --- a/go.mod +++ b/go.mod @@ -86,4 +86,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/kube-burner/kube-burner => github.com/rsevilla87/kube-burner v1.7.6-0.20240926073724-2b0caacee619 +replace github.com/kube-burner/kube-burner => github.com/rsevilla87/kube-burner v1.7.6-0.20240926090502-2e59036445b9 diff --git a/go.sum b/go.sum index af151742..18ddcc13 100644 --- a/go.sum +++ b/go.sum @@ -484,8 +484,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rsevilla87/kube-burner v1.7.6-0.20240926073724-2b0caacee619 h1:B4QRIyyym1mo5dS8hNr8zzbDDghjbh8oUdb8zZukZnY= -github.com/rsevilla87/kube-burner v1.7.6-0.20240926073724-2b0caacee619/go.mod h1:NKtSusPSi2rmS5kYAYC1A+JTsC4RsI0VIHFt6ET4EJE= +github.com/rsevilla87/kube-burner v1.7.6-0.20240926090502-2e59036445b9 h1:ttXzGjTFu7ooOfHq4jZWDEZbO/aUxY4NbBnXMwZAnYw= +github.com/rsevilla87/kube-burner v1.7.6-0.20240926090502-2e59036445b9/go.mod h1:NKtSusPSi2rmS5kYAYC1A+JTsC4RsI0VIHFt6ET4EJE= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= diff --git a/test/helpers.bash b/test/helpers.bash index bbf7fc46..b29dc09e 100644 --- a/test/helpers.bash +++ b/test/helpers.bash @@ -90,7 +90,8 @@ print_events() { check_metric_value() { sleep 3s # There's some delay on the documents to show up in OpenSearch for metric in "${@}"; do - endpoint="${ES_SERVER}/${ES_INDEX}/_search?q=uuid.keyword:${UUID}+AND+metricName.keyword:${metric}+AND+ocpVersion.keyword:*" + query="uuid.keyword:${UUID}+AND+metricName.keyword:${metric}+AND+(metadata.ocpVersion.keyword:*+OR+ocpVersion.keyword:*)" + endpoint="${ES_SERVER}/${ES_INDEX}/_search?q=${query}" RESULT=$(curl -sS ${endpoint} | jq '.hits.total.value // error') RETURN_CODE=$? if [ "${RETURN_CODE}" -ne 0 ]; then From 28cd61e22c190d61bacf7d1ed064f3143859c086 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Tue, 8 Oct 2024 16:53:12 +0200 Subject: [PATCH 3/4] Update kube-burner reference Signed-off-by: Raul Sevilla --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 6dc03b71..ce7362fd 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.3 require ( github.com/cloud-bulldozer/go-commons v1.0.16 github.com/google/uuid v1.6.0 - github.com/kube-burner/kube-burner v1.10.7 + github.com/kube-burner/kube-burner v1.10.8 github.com/openshift/api v0.0.0-20240527133614-ba11c1587003 github.com/openshift/client-go v0.0.0-20240821135114-75c118605d5f github.com/praserx/ipconv v1.2.1 @@ -85,5 +85,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/kube-burner/kube-burner => github.com/rsevilla87/kube-burner v1.7.6-0.20240926090502-2e59036445b9 diff --git a/go.sum b/go.sum index 18ddcc13..3ec765db 100644 --- a/go.sum +++ b/go.sum @@ -358,6 +358,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kube-burner/kube-burner v1.10.8 h1:WD93tpAbwMWige449qLYUn5T7DRShKQQRepNLiPK/YE= +github.com/kube-burner/kube-burner v1.10.8/go.mod h1:NKtSusPSi2rmS5kYAYC1A+JTsC4RsI0VIHFt6ET4EJE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -484,8 +486,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rsevilla87/kube-burner v1.7.6-0.20240926090502-2e59036445b9 h1:ttXzGjTFu7ooOfHq4jZWDEZbO/aUxY4NbBnXMwZAnYw= -github.com/rsevilla87/kube-burner v1.7.6-0.20240926090502-2e59036445b9/go.mod h1:NKtSusPSi2rmS5kYAYC1A+JTsC4RsI0VIHFt6ET4EJE= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= From 19b0ffe9ce896bb7f7b0a743057d95deb19c67f0 Mon Sep 17 00:00:00 2001 From: Raul Sevilla Date: Wed, 9 Oct 2024 00:33:34 +0200 Subject: [PATCH 4/4] RC Signed-off-by: Raul Sevilla --- cluster-density.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cluster-density.go b/cluster-density.go index 8577dfc1..4249faeb 100644 --- a/cluster-density.go +++ b/cluster-density.go @@ -34,6 +34,7 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm var churnDeletionStrategy string var podReadyThreshold time.Duration var metricsProfiles []string + var rc int cmd := &cobra.Command{ Use: variant, Short: fmt.Sprintf("Runs %v workload", variant), @@ -62,7 +63,10 @@ func NewClusterDensity(wh *workloads.WorkloadHelper, variant string) *cobra.Comm } } setMetrics(cmd, metricsProfiles) - wh.Run(cmd.Name()) + rc = wh.Run(cmd.Name()) + }, + PostRun: func(cmd *cobra.Command, args []string) { + os.Exit(rc) }, } cmd.Flags().DurationVar(&podReadyThreshold, "pod-ready-threshold", 2*time.Minute, "Pod ready timeout threshold")