[fix] all probers of a probing type should have the same labels

metric/prometheus.go

@@ -33,7 +33,13 @@ type MetricsType interface {
  *prometheus.CounterVec | *prometheus.GaugeVec | *prometheus.HistogramVec | *prometheus.SummaryVec
 }
 
+type Label struct {
+ Name string `yaml:"name" json:"name" jsonschema:"required,title=Label Name,description=the name of label must be unique"`
+ Value string `yaml:"value" json:"value" jsonschema:"required,title=Label Value,description=the value of label"`
+}
+
 var (
+ registries = make([]*prometheus.Registry, 0)
  counterMap = make(map[string]*prometheus.CounterVec)
  gaugeMap = make(map[string]*prometheus.GaugeVec)
  histogramMap = make(map[string]*prometheus.HistogramVec)
@@ -57,7 +63,7 @@ func Gauge(key string) *prometheus.GaugeVec {
 
 // NewCounter create the counter metric
 func NewCounter(namespace, subsystem, name, metric string,
- help string, labels []string, constLabels map[string]string) *prometheus.CounterVec {
+ help string, labels []string, constLabels []Label) *prometheus.CounterVec {
 
  metricName, err := getAndValid(namespace, subsystem, name, metric, labels)
  if err != nil {
@@ -72,46 +78,58 @@ func NewCounter(namespace, subsystem, name, metric string,
 
  counterMap[metricName] = prometheus.NewCounterVec(
  prometheus.CounterOpts{
- Name: metricName,
- Help: help,
- ConstLabels: constLabels,
+ Name: metricName,
+ Help: help,
  },
- labels,
+ mergeLabels(labels, constLabels),
  )
- prometheus.MustRegister(counterMap[metricName])
 
+ // registries[len(registries)-1].MustRegister(m)
+ prometheus.MustRegister(counterMap[metricName])
  log.Infof("[%s] Counter <%s> is created!", module, metricName)
  return counterMap[metricName]
 }
 
 // NewGauge create the gauge metric
 func NewGauge(namespace, subsystem, name, metric string,
- help string, labels []string, constLabels map[string]string) *prometheus.GaugeVec {
+ help string, labels []string, constLabels []Label) *prometheus.GaugeVec {
 
  metricName, err := getAndValid(namespace, subsystem, name, metric, labels)
  if err != nil {
  log.Errorf("[%s] %v", module, err)
  return nil
  }
+
  if m, find := gaugeMap[metricName]; find {
  log.Debugf("[%s] Gauge <%s> already created!", module, metricName)
  return m
  }
 
  gaugeMap[metricName] = prometheus.NewGaugeVec(
  prometheus.GaugeOpts{
- Name: metricName,
- Help: help,
- ConstLabels: constLabels,
+ Name: metricName,
+ Help: help,
  },
- labels,
+ mergeLabels(labels, constLabels),
  )
+
  prometheus.MustRegister(gaugeMap[metricName])
 
  log.Infof("[%s] Gauge <%s> is created!", module, metricName)
  return gaugeMap[metricName]
 }
 
+func mergeLabels(labels []string, constLabels []Label) []string {
+ l := make([]string, 0, len(labels)+len(constLabels))
+ l = append(l, labels...)
+
+ for _, v := range constLabels {
+ l = append(l, v.Name)
+ }
+
+ return l
+}
+
 func getAndValid(namespace, subsystem, name, metric string, labels []string) (string, error) {
  metricName := GetName(namespace, subsystem, name, metric)
  if ValidMetricName(metricName) == false {
@@ -181,3 +199,10 @@ func RemoveInvalidChars(name string) string {
  }
  return string(result)
 }
+
+func AddConstLabels(labels prometheus.Labels, constLabels []Label) prometheus.Labels {
+ for _, v := range constLabels {
+ labels[v.Name] = v.Value
+ }
+ return labels
+}

metric/prometheus_test.go

@@ -65,12 +65,12 @@ func TestGetName(t *testing.T) {
 
 func TestNewMetrics(t *testing.T) {
  NewCounter("namespace", "subsystem", "counter", "metric",
- "help", []string{"label1", "label2"}, make(map[string]string))
+ "help", []string{"label1", "label2"}, make([]Label, 0))
  assert.NotNil(t, GetName("namespace_subsystem_counter_metric"))
  assert.NotNil(t, Counter("namespace_subsystem_counter_metric"))
 
  NewGauge("namespace", "subsystem", "gauge", "metric",
- "help", []string{"label1", "label2"}, make(map[string]string))
+ "help", []string{"label1", "label2"}, make([]Label, 0))
  assert.NotNil(t, GetName("namespace_subsystem_gauge_metric"))
  assert.NotNil(t, Gauge("namespace_subsystem_gauge_metric"))
 }
@@ -110,34 +110,34 @@ func TestName(t *testing.T) {
 
 func TestDuplicateName(t *testing.T) {
  counter1 := NewCounter("namespace", "subsystem", "counter", "metric",
- "help", []string{}, make(map[string]string))
+ "help", []string{}, make([]Label, 0))
  counter2 := NewCounter("namespace", "subsystem", "counter", "metric",
- "help", []string{}, make(map[string]string))
+ "help", []string{}, make([]Label, 0))
  assert.Equal(t, counter1, counter2)
 
  gauge1 := NewGauge("namespace", "subsystem", "gauge", "metric",
- "help", []string{}, make(map[string]string))
+ "help", []string{}, make([]Label, 0))
  gauge2 := NewGauge("namespace", "subsystem", "gauge", "metric",
- "help", []string{}, make(map[string]string))
+ "help", []string{}, make([]Label, 0))
  assert.Equal(t, gauge1, gauge2)
 }
 
 func TestInvalidName(t *testing.T) {
 
  //label errors
  counter := NewCounter("namespace", "subsystem", "counter", "metric",
- "help", []string{"label-1", "label:2"}, make(map[string]string))
+ "help", []string{"label-1", "label:2"}, make([]Label, 0))
  assert.Nil(t, counter)
 
  gauge := NewGauge("namespace", "subsystem", "gauge", "metric",
- "help", []string{"label-1", "label:2"}, make(map[string]string))
+ "help", []string{"label-1", "label:2"}, make([]Label, 0))
  assert.Nil(t, gauge)
 
  monkey.Patch(ValidMetricName, func(name string) bool {
  return false
  })
  counter = NewCounter("namespace", "subsystem", "counter", "metric",
- "help", []string{}, make(map[string]string))
+ "help", []string{}, make([]Label, 0))
  assert.Nil(t, counter)
 
  monkey.UnpatchAll()

probe/base/base.go

@@ -20,6 +20,7 @@ package base
 
 import (
  "fmt"
+ "github.com/megaease/easeprobe/metric"
  "math"
  "net"
  "net/url"
@@ -45,13 +46,13 @@ type ProbeFuncType func() (bool, string)
 
 // DefaultProbe is the default options for all probe
 type DefaultProbe struct {
- ProbeKind string  `yaml:"-" json:"-"`
- ProbeTag string  `yaml:"-" json:"-"`
- ProbeName string  `yaml:"name" json:"name" jsonschema:"required,title=Probe Name,description=the name of probe must be unique"`
- ProbeChannels []string  `yaml:"channels" json:"channels,omitempty" jsonschema:"title=Probe Channels,description=the channels of probe message need to send to"`
- ProbeTimeout time.Duration  `yaml:"timeout,omitempty" json:"timeout,omitempty" jsonschema:"type=string,format=duration,title=Probe Timeout,description=the timeout of probe"`
- ProbeTimeInterval time.Duration  `yaml:"interval,omitempty" json:"interval,omitempty" jsonschema:"type=string,format=duration,title=Probe Interval,description=the interval of probe"`
- Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty" jsonschema:"title=Probe Labels,description=the labels of probe"`
+ ProbeKind string `yaml:"-" json:"-"`
+ ProbeTag string `yaml:"-" json:"-"`
+ ProbeName string `yaml:"name" json:"name" jsonschema:"required,title=Probe Name,description=the name of probe must be unique"`
+ ProbeChannels []string `yaml:"channels" json:"channels,omitempty" jsonschema:"title=Probe Channels,description=the channels of probe message need to send to"`
+ ProbeTimeout time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty" jsonschema:"type=string,format=duration,title=Probe Timeout,description=the timeout of probe"`
+ ProbeTimeInterval time.Duration `yaml:"interval,omitempty" json:"interval,omitempty" jsonschema:"type=string,format=duration,title=Probe Interval,description=the interval of probe"`
+ Labels []metric.Label `yaml:"labels,omitempty" json:"labels,omitempty" jsonschema:"title=Probe Labels,description=the labels of probe"`
  global.StatusChangeThresholdSettings `yaml:",inline" json:",inline"`
  global.NotificationStrategySettings `yaml:"alert" json:"alert" jsonschema:"title=Probe Alert,description=the alert strategy of probe"`
  ProbeFunc ProbeFuncType `yaml:"-" json:"-"`
@@ -232,37 +233,37 @@ func (d *DefaultProbe) ExportMetrics() {
  }
 
  // Add endpoint label according to ProbeKind(tcp/http/ping/host/...)
- d.metrics.TotalCnt.With(prometheus.Labels{
+ d.metrics.TotalCnt.With(metric.AddConstLabels(prometheus.Labels{
  "name": d.ProbeName,
  "status": d.ProbeResult.Status.String(),
  "endpoint": d.ProbeResult.Endpoint,
- }).Set(float64(cnt))
+ }, d.Labels)).Set(float64(cnt))
 
- d.metrics.TotalTime.With(prometheus.Labels{
+ d.metrics.TotalTime.With(metric.AddConstLabels(prometheus.Labels{
  "name": d.ProbeName,
  "status": d.ProbeResult.Status.String(),
  "endpoint": d.ProbeResult.Endpoint,
- }).Set(float64(time.Seconds()))
+ }, d.Labels)).Set(float64(time.Seconds()))
 
- d.metrics.Duration.With(prometheus.Labels{
+ d.metrics.Duration.With(metric.AddConstLabels(prometheus.Labels{
  "name": d.ProbeName,
  "status": d.ProbeResult.Status.String(),
  "endpoint": d.ProbeResult.Endpoint,
- }).Set(float64(d.ProbeResult.RoundTripTime.Milliseconds()))
+ }, d.Labels)).Set(float64(d.ProbeResult.RoundTripTime.Milliseconds()))
 
  status := ServiceUp // up
  if d.ProbeResult.Status != probe.StatusUp {
  status = ServiceDown // down
  }
- d.metrics.Status.With(prometheus.Labels{
+ d.metrics.Status.With(metric.AddConstLabels(prometheus.Labels{
  "name": d.ProbeName,
  "endpoint": d.ProbeResult.Endpoint,
- }).Set(float64(status))
+ }, d.Labels)).Set(float64(status))
 
- d.metrics.SLA.With(prometheus.Labels{
+ d.metrics.SLA.With(metric.AddConstLabels(prometheus.Labels{
  "name": d.ProbeName,
  "endpoint": d.ProbeResult.Endpoint,
- }).Set(float64(d.ProbeResult.SLAPercent()))
+ }, d.Labels)).Set(float64(d.ProbeResult.SLAPercent()))
 }
 
 // DownTimeCalculation calculate the down time

probe/base/metrics.go

@@ -33,7 +33,7 @@ type metrics struct {
 }
 
 // newMetrics create the metrics
-func newMetrics(subsystem, name string, constLabels map[string]string) *metrics {
+func newMetrics(subsystem, name string, constLabels []metric.Label) *metrics {
  namespace := global.GetEaseProbe().Name
  return &metrics{
  TotalCnt: metric.NewGauge(namespace, subsystem, name, "total",

probe/host/basic.go

@@ -93,8 +93,8 @@ func (b *Basic) CreateMetrics(subsystem, name string) {
 
 // ExportMetrics export the cpu metrics
 func (b *Basic) ExportMetrics(name string) {
- b.metrics.With(prometheus.Labels{
+ b.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "cpu_core",
- }).Set(float64(b.Core))
+ }, b.Labels)).Set(float64(b.Core))
 }

probe/host/cpu.go

@@ -118,48 +118,48 @@ func (c *CPU) CreateMetrics(subsystem, name string) {
 // ExportMetrics export the cpu metrics
 func (c *CPU) ExportMetrics(name string) {
  // CPU metrics
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "usage",
- }).Set(100 - c.Idle)
+ }, c.Labels)).Set(100 - c.Idle)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "idle",
- }).Set(c.Idle)
+ }, c.Labels)).Set(c.Idle)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "user",
- }).Set(c.User)
+ }, c.Labels)).Set(c.User)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "sys",
- }).Set(c.Sys)
+ }, c.Labels)).Set(c.Sys)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "nice",
- }).Set(c.Nice)
+ }, c.Labels)).Set(c.Nice)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "wait",
- }).Set(c.Wait)
+ }, c.Labels)).Set(c.Wait)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "hard",
- }).Set(c.Hard)
+ }, c.Labels)).Set(c.Hard)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "soft",
- }).Set(c.Soft)
+ }, c.Labels)).Set(c.Soft)
 
- c.metrics.With(prometheus.Labels{
+ c.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "steal",
- }).Set(c.Steal)
+ }, c.Labels)).Set(c.Steal)
 }

probe/host/disk.go

@@ -128,28 +128,28 @@ func (d *Disks) CreateMetrics(subsystem, name string) {
 // ExportMetrics export the disk metrics
 func (d *Disks) ExportMetrics(name string) {
  for _, disk := range d.Usage {
- d.metrics.With(prometheus.Labels{
+ d.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "disk": disk.Tag,
  "state": "used",
- }).Set(float64(disk.Used))
+ }, d.Labels)).Set(float64(disk.Used))
 
- d.metrics.With(prometheus.Labels{
+ d.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "disk": disk.Tag,
  "state": "available",
- }).Set(float64(disk.Total - disk.Used))
+ }, d.Labels)).Set(float64(disk.Total - disk.Used))
 
- d.metrics.With(prometheus.Labels{
+ d.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "disk": disk.Tag,
  "state": "total",
- }).Set(float64(disk.Total))
+ }, d.Labels)).Set(float64(disk.Total))
 
- d.metrics.With(prometheus.Labels{
+ d.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "disk": disk.Tag,
  "state": "usage",
- }).Set(disk.Usage)
+ }, d.Labels)).Set(disk.Usage)
  }
 }

probe/host/load.go

@@ -135,18 +135,18 @@ func (l *Load) CreateMetrics(subsystem, name string) {
 
 // ExportMetrics export the load average metrics
 func (l *Load) ExportMetrics(name string) {
- l.metrics.With(prometheus.Labels{
+ l.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "m1",
- }).Set(l.Metrics["m1"])
+ }, l.Labels)).Set(l.Metrics["m1"])
 
- l.metrics.With(prometheus.Labels{
+ l.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "m5",
- }).Set(l.Metrics["m5"])
+ }, l.Labels)).Set(l.Metrics["m5"])
 
- l.metrics.With(prometheus.Labels{
+ l.metrics.With(metric.AddConstLabels(prometheus.Labels{
  "host": name,
  "state": "m15",
- }).Set(l.Metrics["m15"])
+ }, l.Labels)).Set(l.Metrics["m15"])
 }