Fix service host

Signed-off-by: Julien Pivotto <[email protected]>

README.md

@@ -87,6 +87,10 @@ Optional
  The maximum number of checks which are executed before changing to a hard state.
 * `--icinga_reconnect`/`SIGNALILO_ICINGA_RECONNECT`:
  If it's set, Signalilo to waits for a reconnect instead of switching immediately to another URL.
+* `--icinga_create_hosts`/`SIGNALILO_ICINGA_CREATE_HOSTS`:
+ If true, Signalilo will automatically create hosts dynamically based on a label (default: false).
+* `--icinga_create_hosts_label`/`SIGNALILO_ICINGA_CREATE_HOSTS_LABEL`:
+ Label used as hostname to create hosts if `--icinga_create_hosts` is enabled (default: instance).
 * `--alertmanager_port`/`SIGNALILO_ALERTMANAGER_PORT`:
  Port on which Signalilo listens to incoming webhooks (default 8888).
 * `--alertmanager_bearer_token`/`SIGNALILO_ALERTMANAGER_BEARER_TOKEN`:
@@ -307,6 +311,20 @@ Signalilo will try to parse the value of that label as a [Go duration].
 If the value is parsed successfully, Signalilo will create an Icinga service check with active checks enabled and with the check interval set to the parsed duration plus ten percent.
 We add ten percent to the parsed duration to account for network latencies etc., which could otherwise lead to flapping heartbeat checks.
 
+### Dynamic Host Creation
+
+Signalilo supports dynamic creation of Icinga2 hosts based on a label
+included in the alert. This allows for easy and automatic creation of host
+objects in Icinga2 as needed, simplifying the process of associating alerts and
+systems.
+
+To enable this feature, set the `icinga_create_hosts` flag to true and set
+`icinga_create_hosts_label` to the label that should be used as the hostname to
+create the host (default: **instance**). When an alert is received with a label
+matching the specified label, Signalilo will attempt to create a host with the
+specified name. If a host with that name already exists, the alert will be
+associated with that host. If the host does not exist, Signalilo will attempt
+to create one before associating the alert with it.
 
 [Go duration]: https://golang.org/pkg/time/#ParseDuration
 

config/config.go

@@ -74,6 +74,8 @@ type SignaliloConfig struct {
  CheckCommand string
  MaxCheckAttempts int
  Reconnect time.Duration
+ CreateHosts bool
+ CreateHostsLabel string
 }
 
 func ConfigInitialize(configuration Configuration) {

serve.go

@@ -225,6 +225,8 @@ func configureServeCommand(app *kingpin.Application) {
  serve.Flag("icinga_password", "Icinga Password").Envar("SIGNALILO_ICINGA_PASSWORD").Required().StringVar(&s.config.IcingaConfig.Password)
  serve.Flag("icinga_insecure_tls", "Skip Icinga TLS verification").Envar("SIGNALILO_ICINGA_INSECURE_TLS").Default("false").BoolVar(&s.config.IcingaConfig.InsecureTLS)
  serve.Flag("icinga_x509_verify_cn", "Use CN when verifying certificates. Overrides the default go1.15 behavior of rejecting certificates without matching SAN.").Envar("SIGNALILO_ICINGA_X509_VERIFY_CN").Default("true").BoolVar(&s.config.IcingaConfig.X509VerifyCN)
+ serve.Flag("icinga_create_hosts", "Create hosts dynamically based on a label").Envar("SIGNALILO_ICINGA_CREATE_HOSTS").Default("false").BoolVar(&s.config.CreateHosts)
+ serve.Flag("icinga_create_hosts_label", "Label used as hostname to create hosts").Envar("SIGNALILO_ICINGA_CREATE_HOSTS_LABEL").Default("instance").StringVar(&s.config.CreateHostsLabel)
  serve.Flag("icinga_disable_keepalives", "Disable HTTP keepalives").Envar("SIGNALILO_ICINGA_DISABLE_KEEPALIVES").Default("false").BoolVar(&s.config.IcingaConfig.DisableKeepAlives)
  serve.Flag("icinga_display_name_as_service_name", "Leave display name as service name").Envar("SIGNALILO_ICINGA_DISPLAY_NAME_AS_SERVICE_NAME").Default("false").BoolVar(&s.config.DisplayNameAsServiceName)
  serve.Flag("icinga_debug", "Enable debug-level logging for icinga2 client library").Envar("SIGNALILO_ICINGA_DEBUG").Default("false").BoolVar(&s.config.IcingaConfig.Debug)

webhook/hook.go

@@ -99,31 +99,6 @@ func Webhook(w http.ResponseWriter, r *http.Request, c config.Configuration) {
  return
  }
 
- if name, ok := data.CommonLabels["instance"]; ok {
- l.V(2).Infof("Creating host %v", name)
- host, err = icinga.GetHost(serviceHost)
- if err != nil {
- err := icinga.CreateHost(icinga2.Host{
- Name: name,
- DisplayName: name,
- Notes: "Created by signalio",
- })
- if err != nil {
- l.Errorf("Could not create service host %v: %v\n", host, err)
- asJSON(w, http.StatusInternalServerError, err.Error())
- return
- }
- host, err = icinga.GetHost(serviceHost)
- if err != nil {
- l.Errorf("Did not find service host %v: %v\n", host, err)
- asJSON(w, http.StatusInternalServerError, err.Error())
- return
- }
- }
- } else {
- l.V(2).Infof("No instance label")
- }
-
  sameAlertName := false
  groupedAlertName, sameAlertName := data.GroupLabels["alertname"]
  if sameAlertName {
@@ -132,7 +107,39 @@ func Webhook(w http.ResponseWriter, r *http.Request, c config.Configuration) {
  l.V(2).Infof("Grouped alerts without matching alertname: %d alerts", len(data.Alerts))
  }
 
+ // For dynamic creation of hosts. This map contains the hosts we have
+ // already seen.
+ hosts := map[string]struct{}{
+ serviceHost: {},
+ }
+
  for _, alert := range data.Alerts {
+ if c.GetConfig().CreateHosts {
+ if name, ok := alert.Labels[c.GetConfig().CreateHostsLabel]; ok {
+ l.V(2).Infof("Using dynamic host %v", name)
+ if _, ok := hosts[name]; !ok {
+ // We haven't seen this host yet.
+ host, err = icinga.GetHost(name)
+ if err != nil {
+ // Host does not exist.
+ err := icinga.CreateHost(icinga2.Host{
+ Name: name,
+ DisplayName: name,
+ Notes: "Created by signalilo.",
+ Address: name,
+ })
+ if err != nil {
+ l.Errorf("Could not create service host %v: %v\n", host, err)
+ asJSON(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+ }
+ hosts[name] = struct{}{}
+ }
+ serviceHost = name
+ }
+ }
+
  l.V(2).Infof("Processing %v alert: alertname=%v, severity=%v, message=%v",
  alert.Status,
  alert.Labels["alertname"],