feat: allow silencing Grafana alerts

README.md

@@ -9,6 +9,7 @@ grafana-interacter is a tool to interact with your Grafana instance via a Telegr
 - `/dashboard <name>` - will return a link to a dashboard and its panels
 - `/datasources` - will return Grafana datasources
 - `/alerts` - will list both Grafana alerts and Prometheus alerts from all Prometheus datasources, if any
+- `/silence <duration> <params>` - creates a silence for Grafana alert. You need to pass a duration (like `/silence 2h test alert`) and some params for matching alerts to silence. You may use `=` for matching the value exactly (example: `/silence 2h host=localhost`), `!=` for matching everything except this value (example: `/silence 2h host!=localhost`), `=~` for matching everything that matches the regexp (example: `/silence 2h host=~local`), , `!~` for matching everything that doesn't the regexp (example: `/silence 2h host!~local`), or just provide a string that will be treated as an alert name (example: `/silence 2h test alert`).
 
 ## How can I set it up?
 

grafana.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+ "bytes"
  "context"
  "encoding/json"
  "fmt"
@@ -198,6 +199,13 @@ func (g *GrafanaStruct) GetAllAlertingRules() ([]GrafanaAlertGroup, error) {
  return append(grafanaRules, prometheusRules...), nil
 }
 
+func (g *GrafanaStruct) CreateSilence(silence Silence) error {
+ url := g.RelativeLink("/api/alertmanager/grafana/api/v2/silences")
+ res := Silence{}
+ err := g.QueryAndDecodePost(url, silence, res)
+ return err
+}
+
 func (g *GrafanaStruct) Query(url string) (io.ReadCloser, error) {
  client := &http.Client{}
  req, _ := http.NewRequest("GET", url, nil)
@@ -220,6 +228,36 @@ func (g *GrafanaStruct) Query(url string) (io.ReadCloser, error) {
  return resp.Body, nil
 }
 
+func (g *GrafanaStruct) QueryPost(url string, body interface{}) (io.ReadCloser, error) {
+ client := &http.Client{}
+
+ buffer := new(bytes.Buffer)
+
+ if err := json.NewEncoder(buffer).Encode(body); err != nil {
+ return nil, err
+ }
+
+ req, _ := http.NewRequest("POST", url, buffer)
+ req.Header.Set("Content-Type", "application/json")
+
+ g.Logger.Trace().Str("url", url).Msg("Doing a Grafana API query")
+
+ if g.UseAuth() {
+ req.SetBasicAuth(g.Config.User, g.Config.Password)
+ }
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode >= 400 {
+ return nil, fmt.Errorf("Could not fetch request. Status code: %d", resp.StatusCode)
+ }
+
+ return resp.Body, nil
+}
+
 func (g *GrafanaStruct) QueryAndDecode(url string, output interface{}) error {
  body, err := g.Query(url)
  if err != nil {
@@ -229,3 +267,13 @@ func (g *GrafanaStruct) QueryAndDecode(url string, output interface{}) error {
  defer body.Close()
  return json.NewDecoder(body).Decode(&output)
 }
+
+func (g *GrafanaStruct) QueryAndDecodePost(url string, postBody interface{}, output interface{}) error {
+ body, err := g.QueryPost(url, postBody)
+ if err != nil {
+ return err
+ }
+
+ defer body.Close()
+ return json.NewDecoder(body).Decode(&output)
+}

main.go

@@ -80,6 +80,7 @@ func Execute(cmd *cobra.Command, args []string) {
  b.Handle("/datasources", HandleListDatasources)
  b.Handle("/alerts", HandleListAlerts)
  b.Handle("/alert", HandleSingleAlert)
+ b.Handle("/silence", HandleNewSilence)
 
  log.Info().Msg("Telegram bot listening")
 
@@ -284,6 +285,25 @@ func HandleSingleAlert(c tele.Context) error {
  return BotReply(c, sb.String())
 }
 
+func HandleNewSilence(c tele.Context) error {
+ log.Info().
+ Str("sender", c.Sender().Username).
+ Str("text", c.Text()).
+ Msg("Got new silence query")
+
+ silenceInfo, err := ParseSilenceOptions(c.Text(), c)
+ if err != "" {
+ return c.Reply(err)
+ }
+
+ silenceErr := Grafana.CreateSilence(*silenceInfo)
+ if silenceErr != nil {
+ return c.Reply(fmt.Sprintf("Error creating silence: %s", silenceErr))
+ }
+
+ return c.Reply("Silence created.")
+}
+
 func main() {
  rootCmd.PersistentFlags().StringVar(&ConfigPath, "config", "", "Config file path")
 

types.go

@@ -1,6 +1,9 @@
 package main
 
-import "fmt"
+import (
+ "fmt"
+ "time"
+)
 
 type ConfigStruct struct {
  LogLevel string `yaml:"log_level" default:"info"`
@@ -93,6 +96,22 @@ type RenderOptions struct {
  Params map[string]string
 }
 
+type Silence struct {
+ Comment string `json:"comment"`
+ CreatedBy string `json:"createdBy"`
+ StartsAt time.Time `json:"startsAt"`
+ EndsAt time.Time `json:"endsAt"`
+ ID string `json:"id,omitempty"`
+ Matchers []SilenceMatcher `json:"matchers"`
+}
+
+type SilenceMatcher struct {
+ IsEqual bool `json:"isEqual"`
+ IsRegex bool `json:"isRegex"`
+ Name string `json:"name"`
+ Value string `json:"value"`
+}
+
 func (rule *GrafanaAlertRule) Serialize(groupName string) string {
  return fmt.Sprintf("- %s %s -> %s\n", GetEmojiByStatus(rule.State), groupName, rule.Name)
 }

utils.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+ "fmt"
  "regexp"
  "strings"
+ "time"
 
  tele "gopkg.in/telebot.v3"
 )
@@ -154,3 +156,90 @@ func BotReply(c tele.Context, msg string) error {
 
  return nil
 }
+
+func ParseSilenceOptions(query string, c tele.Context) (*Silence, string) {
+ args := strings.Split(query, " ")
+ if len(args) <= 2 {
+ return nil, "Usage: /silence <duration> <params>"
+ }
+
+ _, args = args[0], args[1:] // removing first argument as it's always /silence
+ durationString, args := args[0], args[1:]
+
+ duration, err := time.ParseDuration(durationString)
+ if err != nil {
+ return nil, "Invalid duration provided"
+ }
+
+ silence := Silence{
+ StartsAt: time.Now(),
+ EndsAt: time.Now().Add(duration),
+ Matchers: []SilenceMatcher{},
+ CreatedBy: c.Sender().FirstName,
+ Comment: fmt.Sprintf(
+ "Muted using grafana-interacter for %s by %s",
+ duration,
+ c.Sender().FirstName,
+ ),
+ }
+
+ for len(args) > 0 {
+ if strings.Contains(args[0], "!=") {
+ // not equals
+ argsSplit := strings.SplitN(args[0], "!=", 2)
+ silence.Matchers = append(silence.Matchers, SilenceMatcher{
+ IsEqual: false,
+ IsRegex: false,
+ Name: argsSplit[0],
+ Value: argsSplit[1],
+ })
+ } else if strings.Contains(args[0], "!~") {
+ // not matches regexp
+ argsSplit := strings.SplitN(args[0], "!~", 2)
+ silence.Matchers = append(silence.Matchers, SilenceMatcher{
+ IsEqual: false,
+ IsRegex: true,
+ Name: argsSplit[0],
+ Value: argsSplit[1],
+ })
+ } else if strings.Contains(args[0], "=~") {
+ // matches regexp
+ argsSplit := strings.SplitN(args[0], "=~", 2)
+ silence.Matchers = append(silence.Matchers, SilenceMatcher{
+ IsEqual: true,
+ IsRegex: true,
+ Name: argsSplit[0],
+ Value: argsSplit[1],
+ })
+ } else if strings.Contains(args[0], "=") {
+ // equals
+ argsSplit := strings.SplitN(args[0], "=", 2)
+ silence.Matchers = append(silence.Matchers, SilenceMatcher{
+ IsEqual: true,
+ IsRegex: false,
+ Name: argsSplit[0],
+ Value: argsSplit[1],
+ })
+ } else {
+ break
+ }
+
+ _, args = args[0], args[1:]
+ }
+
+ if len(args) > 0 {
+ // plain string, silencing by alertname
+ silence.Matchers = append(silence.Matchers, SilenceMatcher{
+ IsEqual: true,
+ IsRegex: false,
+ Name: "alertname",
+ Value: strings.Join(args, " "),
+ })
+ }
+
+ if len(silence.Matchers) == 0 {
+ return nil, "Usage: /silence <duration> <params>"
+ }
+
+ return &silence, ""
+}