Expose suspended status as Prometheus metric

Signed-off-by: Stefan Prodan <[email protected]>
fluxcd · Mar 17, 2021 · 446545c · 446545c
1 parent c1ffccb
commit 446545c
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 18 deletions.
diff --git a/api/go.mod b/api/go.mod
@@ -5,7 +5,7 @@ go 1.15
 require (
  github.com/fluxcd/pkg/apis/kustomize v0.0.1
  github.com/fluxcd/pkg/apis/meta v0.8.0
- github.com/fluxcd/pkg/runtime v0.8.4
+ github.com/fluxcd/pkg/runtime v0.8.5
  k8s.io/apiextensions-apiserver v0.20.2
  k8s.io/apimachinery v0.20.2
  sigs.k8s.io/controller-runtime v0.8.3

diff --git a/api/go.sum b/api/go.sum
@@ -92,8 +92,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7
 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0=
 github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk=
 github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po=
-github.com/fluxcd/pkg/runtime v0.8.4 h1:amuhfoHGCUfFCPXg3Zrcyy7f9J+fho+/+FbQDDyewko=
-github.com/fluxcd/pkg/runtime v0.8.4/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
+github.com/fluxcd/pkg/runtime v0.8.5 h1:ynh8fszbLQ3QSisQBNOABEUTnvt+/QfCdaL6gOJQcoQ=
+github.com/fluxcd/pkg/runtime v0.8.5/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=

diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go
@@ -136,6 +136,9 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
  return ctrl.Result{}, client.IgnoreNotFound(err)
  }
 
+ // Record suspended status metric
+ defer r.recordSuspension(ctx, kustomization)
+
  // Add our finalizer if it does not exist
  if !controllerutil.ContainsFinalizer(&kustomization, kustomizev1.KustomizationFinalizer) {
  controllerutil.AddFinalizer(&kustomization, kustomizev1.KustomizationFinalizer)
@@ -221,7 +224,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
  // set the reconciliation status to progressing
  kustomization = kustomizev1.KustomizationProgressing(kustomization)
  if err := r.patchStatus(ctx, req, kustomization.Status); err != nil {
- (logr.FromContext(ctx)).Error(err, "unable to update status to progressing")
+ log.Error(err, "unable to update status to progressing")
  return ctrl.Result{Requeue: true}, err
  }
  r.recordReadiness(ctx, kustomization)
@@ -572,6 +575,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
  return nil
  }
 
+ log := logr.FromContext(ctx)
  timeout := kustomization.GetTimeout() + (time.Second * 1)
  applyCtx, cancel := context.WithTimeout(ctx, timeout)
  defer cancel()
@@ -580,7 +584,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
  if validation == "server" && kustomization.Spec.Force {
  // Use client-side validation with force
  validation = "client"
- (logr.FromContext(ctx)).Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled"))
+ log.Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled"))
  }
 
  cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t",
@@ -616,6 +620,7 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
 }
 
 func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kustomizev1.Kustomization, imp *KustomizeImpersonation, dirPath string) (string, error) {
+ log := logr.FromContext(ctx)
  start := time.Now()
  timeout := kustomization.GetTimeout() + (time.Second * 1)
  applyCtx, cancel := context.WithTimeout(ctx, timeout)
@@ -658,7 +663,7 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto
  }
 
  resources := parseApplyOutput(output)
- (logr.FromContext(ctx)).Info(
+ log.Info(
  fmt.Sprintf("Kustomization applied in %s",
  time.Now().Sub(start).String()),
  "output", resources,
@@ -674,12 +679,13 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto
 }
 
 func (r *KustomizationReconciler) applyWithRetry(ctx context.Context, kustomization kustomizev1.Kustomization, imp *KustomizeImpersonation, revision, dirPath string, delay time.Duration) (string, error) {
+ log := logr.FromContext(ctx)
  changeSet, err := r.apply(ctx, kustomization, imp, dirPath)
  if err != nil {
  // retry apply due to CRD/CR race
  if strings.Contains(err.Error(), "could not find the requested resource") ||
  strings.Contains(err.Error(), "no matches for kind") {
- (logr.FromContext(ctx)).Info("retrying apply", "error", err.Error())
+ log.Info("retrying apply", "error", err.Error())
  time.Sleep(delay)
  if changeSet, err := r.apply(ctx, kustomization, imp, dirPath); err != nil {
  return "", err
@@ -707,6 +713,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.C
  return nil
  }
 
+ log := logr.FromContext(ctx)
  gc := NewGarbageCollector(kubeClient, *kustomization.Status.Snapshot, newChecksum, logr.FromContext(ctx))
 
  if output, ok := gc.Prune(kustomization.GetTimeout(),
@@ -716,7 +723,7 @@ func (r *KustomizationReconciler) prune(ctx context.Context, kubeClient client.C
  return fmt.Errorf("garbage collection failed: %s", output)
  } else {
  if output != "" {
- (logr.FromContext(ctx)).Info(fmt.Sprintf("garbage collection completed: %s", output))
+ log.Info(fmt.Sprintf("garbage collection completed: %s", output))
  r.event(ctx, kustomization, newChecksum, events.EventSeverityInfo, output, nil)
  }
  }
@@ -744,13 +751,14 @@ func (r *KustomizationReconciler) checkHealth(ctx context.Context, statusPoller
 }
 
 func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomization kustomizev1.Kustomization) (ctrl.Result, error) {
+ log := logr.FromContext(ctx)
  if kustomization.Spec.Prune && !kustomization.Spec.Suspend {
  // create any necessary kube-clients
  imp := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, "")
  client, _, err := imp.GetClient(ctx)
  if err != nil {
  err = fmt.Errorf("failed to build kube client for Kustomization: %w", err)
- (logr.FromContext(ctx)).Error(err, "Unable to prune for finalizer")
+ log.Error(err, "Unable to prune for finalizer")
  return ctrl.Result{}, err
  }
  if err := r.prune(ctx, client, kustomization, ""); err != nil {
@@ -774,13 +782,11 @@ func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomiza
 }
 
 func (r *KustomizationReconciler) event(ctx context.Context, kustomization kustomizev1.Kustomization, revision, severity, msg string, metadata map[string]string) {
+ log := logr.FromContext(ctx)
  r.EventRecorder.Event(&kustomization, "Normal", severity, msg)
  objRef, err := reference.GetReference(r.Scheme, &kustomization)
  if err != nil {
- (logr.FromContext(ctx)).WithValues(
- strings.ToLower(kustomization.Kind),
- fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
- ).Error(err, "unable to send event")
+ log.Error(err, "unable to send event")
  return
  }
 
@@ -798,7 +804,7 @@ func (r *KustomizationReconciler) event(ctx context.Context, kustomization kusto
  }
 
  if err := r.ExternalEventRecorder.Eventf(*objRef, metadata, severity, reason, msg); err != nil {
- (logr.FromContext(ctx)).Error(err, "unable to send event")
+ log.Error(err, "unable to send event")
  return
  }
  }
@@ -808,10 +814,11 @@ func (r *KustomizationReconciler) recordReadiness(ctx context.Context, kustomiza
  if r.MetricsRecorder == nil {
  return
  }
+ log := logr.FromContext(ctx)
 
  objRef, err := reference.GetReference(r.Scheme, &kustomization)
  if err != nil {
- (logr.FromContext(ctx)).Error(err, "unable to record readiness metric")
+ log.Error(err, "unable to record readiness metric")
  return
  }
  if rc := apimeta.FindStatusCondition(kustomization.Status.Conditions, meta.ReadyCondition); rc != nil {
@@ -824,6 +831,25 @@ func (r *KustomizationReconciler) recordReadiness(ctx context.Context, kustomiza
  }
 }
 
+func (r *KustomizationReconciler) recordSuspension(ctx context.Context, kustomization kustomizev1.Kustomization) {
+ if r.MetricsRecorder == nil {
+ return
+ }
+ log := logr.FromContext(ctx)
+
+ objRef, err := reference.GetReference(r.Scheme, &kustomization)
+ if err != nil {
+ log.Error(err, "unable to record suspended metric")
+ return
+ }
+
+ if !kustomization.DeletionTimestamp.IsZero() {
+ r.MetricsRecorder.RecordSuspend(*objRef, false)
+ } else {
+ r.MetricsRecorder.RecordSuspend(*objRef, kustomization.Spec.Suspend)
+ }
+}
+
 func (r *KustomizationReconciler) patchStatus(ctx context.Context, req ctrl.Request, newStatus kustomizev1.KustomizationStatus) error {
  var kustomization kustomizev1.Kustomization
  if err := r.Get(ctx, req.NamespacedName, &kustomization); err != nil {

diff --git a/go.mod b/go.mod
@@ -10,7 +10,7 @@ require (
  github.com/fluxcd/kustomize-controller/api v0.9.2
  github.com/fluxcd/pkg/apis/kustomize v0.0.1
  github.com/fluxcd/pkg/apis/meta v0.8.0
- github.com/fluxcd/pkg/runtime v0.8.4
+ github.com/fluxcd/pkg/runtime v0.8.5
  github.com/fluxcd/pkg/testserver v0.0.2
  github.com/fluxcd/pkg/untar v0.0.5
  github.com/fluxcd/source-controller/api v0.9.1

diff --git a/go.sum b/go.sum
@@ -197,8 +197,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7
 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0=
 github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk=
 github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po=
-github.com/fluxcd/pkg/runtime v0.8.4 h1:amuhfoHGCUfFCPXg3Zrcyy7f9J+fho+/+FbQDDyewko=
-github.com/fluxcd/pkg/runtime v0.8.4/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
+github.com/fluxcd/pkg/runtime v0.8.5 h1:ynh8fszbLQ3QSisQBNOABEUTnvt+/QfCdaL6gOJQcoQ=
+github.com/fluxcd/pkg/runtime v0.8.5/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw=
 github.com/fluxcd/pkg/testserver v0.0.2 h1:SoaMtO9cE5p/wl2zkGudzflnEHd9mk68CGjZOo7w0Uk=
 github.com/fluxcd/pkg/testserver v0.0.2/go.mod h1:pgUZTh9aQ44FSTQo+5NFlh7YMbUfdz1B80DalW7k96Y=
 github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk=