From 6fd248dae1179441a22af9a9064d153c1eafe099 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 7 Mar 2023 14:47:10 +0100 Subject: [PATCH] Fix service host Signed-off-by: Julien Pivotto --- README.md | 20 +++++++++++++++++ config/config.go | 2 ++ serve.go | 2 ++ webhook/hook.go | 57 +++++++++++++++++++++++++++--------------------- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 5c95d0b..7e06e02 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,12 @@ Optional The maximum number of checks which are executed before changing to a hard state. * `--icinga_reconnect`/`SIGNALILO_ICINGA_RECONNECT`: If it's set, Signalilo to waits for a reconnect instead of switching immediately to another URL. +Here are the new bullet points with the flags added in pull request #126: +* `--icinga_create_hosts`/`SIGNALILO_ICINGA_CREATE_HOSTS`: + If true, Signalilo will automatically create hosts dynamically based on a label (default: false). +* `--icinga_create_hosts_label`/`SIGNALILO_ICINGA_CREATE_HOSTS_LABEL`: + Label used as hostname to create hosts if `--icinga_create_hosts` is enabled (default: instance). +Note: when using environment variables, replace `-` with `_` and lowercase the flag name. * `--alertmanager_port`/`SIGNALILO_ALERTMANAGER_PORT`: Port on which Signalilo listens to incoming webhooks (default 8888). * `--alertmanager_bearer_token`/`SIGNALILO_ALERTMANAGER_BEARER_TOKEN`: @@ -307,6 +313,20 @@ Signalilo will try to parse the value of that label as a [Go duration]. If the value is parsed successfully, Signalilo will create an Icinga service check with active checks enabled and with the check interval set to the parsed duration plus ten percent. We add ten percent to the parsed duration to account for network latencies etc., which could otherwise lead to flapping heartbeat checks. +### Dynamic Host Creation + +Signalilo supports dynamic creation of Icinga2 hosts based on a label +included in the alert. This allows for easy and automatic creation of host +objects in Icinga2 as needed, simplifying the process of associating alerts and +systems. + +To enable this feature, set the `icinga_create_hosts` flag to true and set +`icinga_create_hosts_label` to the label that should be used as the hostname to +create the host (default: **instance**). When an alert is received with a label +matching the specified label, Signalilo will attempt to create a host with the +specified name. If a host with that name already exists, the alert will be +associated with that host. If the host does not exist, Signalilo will attempt +to create one before associating the alert with it. [Go duration]: https://golang.org/pkg/time/#ParseDuration diff --git a/config/config.go b/config/config.go index e95d337..cd54df2 100644 --- a/config/config.go +++ b/config/config.go @@ -74,6 +74,8 @@ type SignaliloConfig struct { CheckCommand string MaxCheckAttempts int Reconnect time.Duration + CreateHosts bool + CreateHostsLabel string } func ConfigInitialize(configuration Configuration) { diff --git a/serve.go b/serve.go index e0040a9..efb109c 100644 --- a/serve.go +++ b/serve.go @@ -225,6 +225,8 @@ func configureServeCommand(app *kingpin.Application) { serve.Flag("icinga_password", "Icinga Password").Envar("SIGNALILO_ICINGA_PASSWORD").Required().StringVar(&s.config.IcingaConfig.Password) serve.Flag("icinga_insecure_tls", "Skip Icinga TLS verification").Envar("SIGNALILO_ICINGA_INSECURE_TLS").Default("false").BoolVar(&s.config.IcingaConfig.InsecureTLS) serve.Flag("icinga_x509_verify_cn", "Use CN when verifying certificates. Overrides the default go1.15 behavior of rejecting certificates without matching SAN.").Envar("SIGNALILO_ICINGA_X509_VERIFY_CN").Default("true").BoolVar(&s.config.IcingaConfig.X509VerifyCN) + serve.Flag("icinga_create_hosts", "Create hosts dynamically based on a label").Envar("SIGNALILO_ICINGA_CREATE_HOSTS").Default("false").BoolVar(&s.config.CreateHosts) + serve.Flag("icinga_create_hosts_label", "Label used as hostname to create hosts").Envar("SIGNALILO_ICINGA_CREATE_HOSTS_LABEL").Default("instance").StringVar(&s.config.CreateHostsLabel) serve.Flag("icinga_disable_keepalives", "Disable HTTP keepalives").Envar("SIGNALILO_ICINGA_DISABLE_KEEPALIVES").Default("false").BoolVar(&s.config.IcingaConfig.DisableKeepAlives) serve.Flag("icinga_display_name_as_service_name", "Leave display name as service name").Envar("SIGNALILO_ICINGA_DISPLAY_NAME_AS_SERVICE_NAME").Default("false").BoolVar(&s.config.DisplayNameAsServiceName) serve.Flag("icinga_debug", "Enable debug-level logging for icinga2 client library").Envar("SIGNALILO_ICINGA_DEBUG").Default("false").BoolVar(&s.config.IcingaConfig.Debug) diff --git a/webhook/hook.go b/webhook/hook.go index 439f204..8e6d738 100644 --- a/webhook/hook.go +++ b/webhook/hook.go @@ -99,31 +99,6 @@ func Webhook(w http.ResponseWriter, r *http.Request, c config.Configuration) { return } - if name, ok := data.CommonLabels["instance"]; ok { - l.V(2).Infof("Creating host %v", name) - host, err = icinga.GetHost(serviceHost) - if err != nil { - err := icinga.CreateHost(icinga2.Host{ - Name: name, - DisplayName: name, - Notes: "Created by signalio", - }) - if err != nil { - l.Errorf("Could not create service host %v: %v\n", host, err) - asJSON(w, http.StatusInternalServerError, err.Error()) - return - } - host, err = icinga.GetHost(serviceHost) - if err != nil { - l.Errorf("Did not find service host %v: %v\n", host, err) - asJSON(w, http.StatusInternalServerError, err.Error()) - return - } - } - } else { - l.V(2).Infof("No instance label") - } - sameAlertName := false groupedAlertName, sameAlertName := data.GroupLabels["alertname"] if sameAlertName { @@ -132,7 +107,39 @@ func Webhook(w http.ResponseWriter, r *http.Request, c config.Configuration) { l.V(2).Infof("Grouped alerts without matching alertname: %d alerts", len(data.Alerts)) } + // For dynamic creation of hosts. This map contains the hosts we have + // already seen. + hosts := map[string]struct{}{ + serviceHost: {}, + } + for _, alert := range data.Alerts { + if c.GetConfig().CreateHosts { + if name, ok := alert.Labels[c.GetConfig().CreateHostsLabel]; ok { + l.V(2).Infof("Using dynamic host %v", name) + if _, ok := hosts[name]; !ok { + // We haven't seen this host yet. + host, err = icinga.GetHost(name) + if err != nil { + // Host does not exist. + err := icinga.CreateHost(icinga2.Host{ + Name: name, + DisplayName: name, + Notes: "Created by signalilo.", + Address: name, + }) + if err != nil { + l.Errorf("Could not create service host %v: %v\n", host, err) + asJSON(w, http.StatusInternalServerError, err.Error()) + return + } + } + hosts[name] = struct{}{} + } + serviceHost = name + } + } + l.V(2).Infof("Processing %v alert: alertname=%v, severity=%v, message=%v", alert.Status, alert.Labels["alertname"],