From 446545c71fcb87fea46e5a14ee7520e1f40384e7 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 17 Mar 2021 11:11:13 +0200 Subject: [PATCH] Expose suspended status as Prometheus metric Signed-off-by: Stefan Prodan --- api/go.mod | 2 +- api/go.sum | 4 +- controllers/kustomization_controller.go | 50 +++++++++++++++++++------ go.mod | 2 +- go.sum | 4 +- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/api/go.mod b/api/go.mod index dab7aef7..49bbe3d1 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,7 +5,7 @@ go 1.15 require ( github.com/fluxcd/pkg/apis/kustomize v0.0.1 github.com/fluxcd/pkg/apis/meta v0.8.0 - github.com/fluxcd/pkg/runtime v0.8.4 + github.com/fluxcd/pkg/runtime v0.8.5 k8s.io/apiextensions-apiserver v0.20.2 k8s.io/apimachinery v0.20.2 sigs.k8s.io/controller-runtime v0.8.3 diff --git a/api/go.sum b/api/go.sum index 553b0e3a..2b0ee6ee 100644 --- a/api/go.sum +++ b/api/go.sum @@ -92,8 +92,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0= github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk= github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po= -github.com/fluxcd/pkg/runtime v0.8.4 h1:amuhfoHGCUfFCPXg3Zrcyy7f9J+fho+/+FbQDDyewko= -github.com/fluxcd/pkg/runtime v0.8.4/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw= +github.com/fluxcd/pkg/runtime v0.8.5 h1:ynh8fszbLQ3QSisQBNOABEUTnvt+/QfCdaL6gOJQcoQ= +github.com/fluxcd/pkg/runtime v0.8.5/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 6c5bd83d..a0a42223 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -136,6 +136,9 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, client.IgnoreNotFound(err) } + // Record suspended status metric + defer r.recordSuspension(ctx, kustomization) + // Add our finalizer if it does not exist if !controllerutil.ContainsFinalizer(&kustomization, kustomizev1.KustomizationFinalizer) { controllerutil.AddFinalizer(&kustomization, kustomizev1.KustomizationFinalizer) @@ -221,7 +224,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques // set the reconciliation status to progressing kustomization = kustomizev1.KustomizationProgressing(kustomization) if err := r.patchStatus(ctx, req, kustomization.Status); err != nil { - (logr.FromContext(ctx)).Error(err, "unable to update status to progressing") + log.Error(err, "unable to update status to progressing") return ctrl.Result{Requeue: true}, err } r.recordReadiness(ctx, kustomization) @@ -572,6 +575,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku return nil } + log := logr.FromContext(ctx) timeout := kustomization.GetTimeout() + (time.Second * 1) applyCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -580,7 +584,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku if validation == "server" && kustomization.Spec.Force { // Use client-side validation with force validation = "client" - (logr.FromContext(ctx)).Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled")) + log.Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled")) } cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t", @@ -616,6 +620,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku } func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kustomizev1.Kustomization, imp *KustomizeImpersonation, dirPath string) (string, error) { + log := logr.FromContext(ctx) start := time.Now() timeout := kustomization.GetTimeout() + (time.Second * 1) applyCtx, cancel := context.WithTimeout(ctx, timeout) @@ -658,7 +663,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto } resources := parseApplyOutput(output) - (logr.FromContext(ctx)).Info( + log.Info( fmt.Sprintf("Kustomization applied in %s", time.Now().Sub(start).String()), "output", resources, @@ -674,12 +679,13 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto } func (r *KustomizationReconciler) applyWithRetry(ctx context.Context, kustomization kustomizev1.Kustomization, imp *KustomizeImpersonation, revision, dirPath string, delay time.Duration) (string, error) { + log := logr.FromContext(ctx) changeSet, err := r.apply(ctx, kustomization, imp, dirPath) if err != nil { // retry apply due to CRD/CR race if strings.Contains(err.Error(), "could not find the requested resource") || strings.Contains(err.Error(), "no matches for kind") { - (logr.FromContext(ctx)).Info("retrying apply", "error", err.Error()) + log.Info("retrying apply", "error", err.Error()) time.Sleep(delay) if changeSet, err := r.apply(ctx, kustomization, imp, dirPath); err != nil { return "", err @@ -707,6 +713,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.C return nil } + log := logr.FromContext(ctx) gc := NewGarbageCollector(kubeClient, *kustomization.Status.Snapshot, newChecksum, logr.FromContext(ctx)) if output, ok := gc.Prune(kustomization.GetTimeout(), @@ -716,7 +723,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.C return fmt.Errorf("garbage collection failed: %s", output) } else { if output != "" { - (logr.FromContext(ctx)).Info(fmt.Sprintf("garbage collection completed: %s", output)) + log.Info(fmt.Sprintf("garbage collection completed: %s", output)) r.event(ctx, kustomization, newChecksum, events.EventSeverityInfo, output, nil) } } @@ -744,13 +751,14 @@ func (r *KustomizationReconciler) checkHealth(ctx context.Context, statusPoller } func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomization kustomizev1.Kustomization) (ctrl.Result, error) { + log := logr.FromContext(ctx) if kustomization.Spec.Prune && !kustomization.Spec.Suspend { // create any necessary kube-clients imp := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, "") client, _, err := imp.GetClient(ctx) if err != nil { err = fmt.Errorf("failed to build kube client for Kustomization: %w", err) - (logr.FromContext(ctx)).Error(err, "Unable to prune for finalizer") + log.Error(err, "Unable to prune for finalizer") return ctrl.Result{}, err } if err := r.prune(ctx, client, kustomization, ""); err != nil { @@ -774,13 +782,11 @@ func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomiza } func (r *KustomizationReconciler) event(ctx context.Context, kustomization kustomizev1.Kustomization, revision, severity, msg string, metadata map[string]string) { + log := logr.FromContext(ctx) r.EventRecorder.Event(&kustomization, "Normal", severity, msg) objRef, err := reference.GetReference(r.Scheme, &kustomization) if err != nil { - (logr.FromContext(ctx)).WithValues( - strings.ToLower(kustomization.Kind), - fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()), - ).Error(err, "unable to send event") + log.Error(err, "unable to send event") return } @@ -798,7 +804,7 @@ func (r *KustomizationReconciler) event(ctx context.Context, kustomization kusto } if err := r.ExternalEventRecorder.Eventf(*objRef, metadata, severity, reason, msg); err != nil { - (logr.FromContext(ctx)).Error(err, "unable to send event") + log.Error(err, "unable to send event") return } } @@ -808,10 +814,11 @@ func (r *KustomizationReconciler) recordReadiness(ctx context.Context, kustomiza if r.MetricsRecorder == nil { return } + log := logr.FromContext(ctx) objRef, err := reference.GetReference(r.Scheme, &kustomization) if err != nil { - (logr.FromContext(ctx)).Error(err, "unable to record readiness metric") + log.Error(err, "unable to record readiness metric") return } if rc := apimeta.FindStatusCondition(kustomization.Status.Conditions, meta.ReadyCondition); rc != nil { @@ -824,6 +831,25 @@ func (r *KustomizationReconciler) recordReadiness(ctx context.Context, kustomiza } } +func (r *KustomizationReconciler) recordSuspension(ctx context.Context, kustomization kustomizev1.Kustomization) { + if r.MetricsRecorder == nil { + return + } + log := logr.FromContext(ctx) + + objRef, err := reference.GetReference(r.Scheme, &kustomization) + if err != nil { + log.Error(err, "unable to record suspended metric") + return + } + + if !kustomization.DeletionTimestamp.IsZero() { + r.MetricsRecorder.RecordSuspend(*objRef, false) + } else { + r.MetricsRecorder.RecordSuspend(*objRef, kustomization.Spec.Suspend) + } +} + func (r *KustomizationReconciler) patchStatus(ctx context.Context, req ctrl.Request, newStatus kustomizev1.KustomizationStatus) error { var kustomization kustomizev1.Kustomization if err := r.Get(ctx, req.NamespacedName, &kustomization); err != nil { diff --git a/go.mod b/go.mod index b2364e6b..691e1636 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/fluxcd/kustomize-controller/api v0.9.2 github.com/fluxcd/pkg/apis/kustomize v0.0.1 github.com/fluxcd/pkg/apis/meta v0.8.0 - github.com/fluxcd/pkg/runtime v0.8.4 + github.com/fluxcd/pkg/runtime v0.8.5 github.com/fluxcd/pkg/testserver v0.0.2 github.com/fluxcd/pkg/untar v0.0.5 github.com/fluxcd/source-controller/api v0.9.1 diff --git a/go.sum b/go.sum index 66793cc1..90ff4d55 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0= github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk= github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po= -github.com/fluxcd/pkg/runtime v0.8.4 h1:amuhfoHGCUfFCPXg3Zrcyy7f9J+fho+/+FbQDDyewko= -github.com/fluxcd/pkg/runtime v0.8.4/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw= +github.com/fluxcd/pkg/runtime v0.8.5 h1:ynh8fszbLQ3QSisQBNOABEUTnvt+/QfCdaL6gOJQcoQ= +github.com/fluxcd/pkg/runtime v0.8.5/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw= github.com/fluxcd/pkg/testserver v0.0.2 h1:SoaMtO9cE5p/wl2zkGudzflnEHd9mk68CGjZOo7w0Uk= github.com/fluxcd/pkg/testserver v0.0.2/go.mod h1:pgUZTh9aQ44FSTQo+5NFlh7YMbUfdz1B80DalW7k96Y= github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk=