From 0fc3adb2a609a7dadba77ee06949f15f06d5ca11 Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Wed, 25 Sep 2024 15:56:22 +0800 Subject: [PATCH] chore: Add missing operator metrics (#5752) Co-authored-by: csuzhangxc --- cmd/br-federation-manager/main.go | 5 +++++ cmd/controller-manager/main.go | 5 +++++ .../autoscaler/tidbcluster_autoscaler_controller.go | 12 +++++++++++- pkg/controller/backup/backup_controller.go | 12 +++++++++++- .../backupschedule/backup_schedule_controller.go | 12 +++++++++++- pkg/controller/dmcluster/dm_cluster_controller.go | 12 +++++++++++- .../fedvolumebackup/fed_volume_backup_controller.go | 12 +++++++++++- .../fed_volume_backup_schedule_controller.go | 12 +++++++++++- .../fed_volume_restore_controller.go | 12 +++++++++++- pkg/controller/restore/restore_controller.go | 12 +++++++++++- pkg/controller/tidbcluster/pod_control.go | 12 +++++++++++- .../tidbcluster/tidb_cluster_controller.go | 12 +++++++++++- .../tidbdashboard/tidb_dashboard_controller.go | 12 +++++++++++- .../tidbinitializer/tidb_initializer_controller.go | 12 +++++++++++- .../tidbmonitor/tidb_monitor_controller.go | 12 +++++++++++- .../tidb_ng_monitoring_controller.go | 12 +++++++++++- pkg/metrics/metrics.go | 4 ++++ 17 files changed, 168 insertions(+), 14 deletions(-) diff --git a/cmd/br-federation-manager/main.go b/cmd/br-federation-manager/main.go index fe6e044d98..8bfa4fe171 100644 --- a/cmd/br-federation-manager/main.go +++ b/cmd/br-federation-manager/main.go @@ -133,6 +133,11 @@ func main() { initMetrics := func(c Controller) { metrics.ActiveWorkers.WithLabelValues(c.Name()).Set(0) + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Add(0) + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Add(0) + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Add(0) + metrics.ReconcileErrors.WithLabelValues(c.Name()).Add(0) + metrics.WorkerCount.WithLabelValues(c.Name()).Set(float64(cliCfg.Workers)) } // Initialize all controllers diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index 3153008229..e5e235a1d2 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -168,6 +168,11 @@ func main() { initMetrics := func(c Controller) { metrics.ActiveWorkers.WithLabelValues(c.Name()).Set(0) + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Add(0) + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Add(0) + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Add(0) + metrics.ReconcileErrors.WithLabelValues(c.Name()).Add(0) + metrics.WorkerCount.WithLabelValues(c.Name()).Set(float64(cliCfg.Workers)) } // Initialize all controllers diff --git a/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go b/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go index bfb6fd3512..c631da5496 100644 --- a/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go +++ b/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go @@ -94,11 +94,21 @@ func (c *Controller) processNextWorkItem() bool { return true } -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TidbClusterAutoScaler %q (%v)", key, duration) }() diff --git a/pkg/controller/backup/backup_controller.go b/pkg/controller/backup/backup_controller.go index 6bf0778e9e..41a27dfbd3 100644 --- a/pkg/controller/backup/backup_controller.go +++ b/pkg/controller/backup/backup_controller.go @@ -124,11 +124,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given backup. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing Backup %q (%v)", key, duration) }() diff --git a/pkg/controller/backupschedule/backup_schedule_controller.go b/pkg/controller/backupschedule/backup_schedule_controller.go index edb560bda2..dde25ee99c 100644 --- a/pkg/controller/backupschedule/backup_schedule_controller.go +++ b/pkg/controller/backupschedule/backup_schedule_controller.go @@ -117,11 +117,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given backupSchedule. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing BackupSchedule %q (%v)", key, duration) }() diff --git a/pkg/controller/dmcluster/dm_cluster_controller.go b/pkg/controller/dmcluster/dm_cluster_controller.go index 36a11be5f1..ba41da51dd 100644 --- a/pkg/controller/dmcluster/dm_cluster_controller.go +++ b/pkg/controller/dmcluster/dm_cluster_controller.go @@ -138,11 +138,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given dmcluster. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing DMCluster %q (%v)", key, duration) }() diff --git a/pkg/controller/fedvolumebackup/fed_volume_backup_controller.go b/pkg/controller/fedvolumebackup/fed_volume_backup_controller.go index 813c56207a..ba1fe21f72 100644 --- a/pkg/controller/fedvolumebackup/fed_volume_backup_controller.go +++ b/pkg/controller/fedvolumebackup/fed_volume_backup_controller.go @@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given VolumeBackup. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing VolumeBackup %q (%v)", key, duration) }() diff --git a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go index 705500e0aa..5f6d2cf69e 100644 --- a/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go +++ b/pkg/controller/fedvolumebackupschedule/fed_volume_backup_schedule_controller.go @@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given VolumeBackupSchedule. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing VolumeBackupSchedule %q (%v)", key, duration) }() diff --git a/pkg/controller/fedvolumerestore/fed_volume_restore_controller.go b/pkg/controller/fedvolumerestore/fed_volume_restore_controller.go index 164bbb0b97..0accbfd890 100644 --- a/pkg/controller/fedvolumerestore/fed_volume_restore_controller.go +++ b/pkg/controller/fedvolumerestore/fed_volume_restore_controller.go @@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given VolumeRestore. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing VolumeRestore %q (%v)", key, duration) }() diff --git a/pkg/controller/restore/restore_controller.go b/pkg/controller/restore/restore_controller.go index 5faeeae822..215bf13241 100644 --- a/pkg/controller/restore/restore_controller.go +++ b/pkg/controller/restore/restore_controller.go @@ -118,11 +118,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given restore. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing Restore %q (%v)", key, duration) }() diff --git a/pkg/controller/tidbcluster/pod_control.go b/pkg/controller/tidbcluster/pod_control.go index 866c72123c..9a64cb4628 100644 --- a/pkg/controller/tidbcluster/pod_control.go +++ b/pkg/controller/tidbcluster/pod_control.go @@ -170,7 +170,7 @@ func (c *PodController) processNextWorkItem() bool { return true } -func (c *PodController) sync(key string) (reconcile.Result, error) { +func (c *PodController) sync(key string) (result reconcile.Result, err error) { ns, name, err := cache.SplitMetaNamespaceKey(key) if err != nil { return reconcile.Result{}, err @@ -213,6 +213,16 @@ func (c *PodController) sync(key string) (reconcile.Result, error) { defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TidbCluster pod %q (%v)", key, duration) }() diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 3b31eed966..5c1b631f1e 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -150,11 +150,21 @@ func (c *Controller) processNextWorkItem() bool { } // sync syncs the given tidbcluster. -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TidbCluster %q (%v)", key, duration) }() diff --git a/pkg/controller/tidbdashboard/tidb_dashboard_controller.go b/pkg/controller/tidbdashboard/tidb_dashboard_controller.go index 6ce01a9f63..d0119c50ab 100644 --- a/pkg/controller/tidbdashboard/tidb_dashboard_controller.go +++ b/pkg/controller/tidbdashboard/tidb_dashboard_controller.go @@ -123,11 +123,21 @@ func (c *Controller) processNextWorkItem() bool { return true } -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TidbDashboard %s (%v)", key, duration) }() diff --git a/pkg/controller/tidbinitializer/tidb_initializer_controller.go b/pkg/controller/tidbinitializer/tidb_initializer_controller.go index c31dbffa5c..f55d82ece5 100644 --- a/pkg/controller/tidbinitializer/tidb_initializer_controller.go +++ b/pkg/controller/tidbinitializer/tidb_initializer_controller.go @@ -112,11 +112,21 @@ func (c *Controller) processNextWorkItem() bool { return true } -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TiDBInitializer %q (%v)", key, duration) }() diff --git a/pkg/controller/tidbmonitor/tidb_monitor_controller.go b/pkg/controller/tidbmonitor/tidb_monitor_controller.go index b08d216f52..3dd9644a9b 100644 --- a/pkg/controller/tidbmonitor/tidb_monitor_controller.go +++ b/pkg/controller/tidbmonitor/tidb_monitor_controller.go @@ -107,11 +107,21 @@ func (c *Controller) processNextWorkItem() bool { return true } -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TidbMonitor %q (%v)", key, duration) }() diff --git a/pkg/controller/tidbngmonitoring/tidb_ng_monitoring_controller.go b/pkg/controller/tidbngmonitoring/tidb_ng_monitoring_controller.go index 2f635beb91..aeac258d50 100644 --- a/pkg/controller/tidbngmonitoring/tidb_ng_monitoring_controller.go +++ b/pkg/controller/tidbngmonitoring/tidb_ng_monitoring_controller.go @@ -122,11 +122,21 @@ func (c *Controller) processNextWorkItem() bool { return true } -func (c *Controller) sync(key string) error { +func (c *Controller) sync(key string) (err error) { startTime := time.Now() defer func() { duration := time.Since(startTime) metrics.ReconcileTime.WithLabelValues(c.Name()).Observe(duration.Seconds()) + + if err == nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelSuccess).Inc() + } else if perrors.Find(err, controller.IsRequeueError) != nil { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelRequeue).Inc() + } else { + metrics.ReconcileTotal.WithLabelValues(c.Name(), metrics.LabelError).Inc() + metrics.ReconcileErrors.WithLabelValues(c.Name()).Inc() + } + klog.V(4).Infof("Finished syncing TidbNGMonitoring %s (%v)", key, duration) }() diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0a89193f2e..0789f94331 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -22,6 +22,10 @@ const ( LabelNamespace = "namespace" LabelName = "name" LabelComponent = "component" + + LabelError = "error" + LabelRequeue = "requeue" + LabelSuccess = "success" ) var (