From cbd8ee4c6602ad9232a1f6a8639c48e4b49ea412 Mon Sep 17 00:00:00 2001 From: Bin Yi Date: Thu, 23 Jan 2020 15:29:55 -0800 Subject: [PATCH] initial commit --- README.md | 29 ++++++- go.mod | 5 ++ go.sum | 9 +++ hook.go | 219 +++++++++++++++++++++++++++++++++++++++++++++++++++ hook_test.go | 124 +++++++++++++++++++++++++++++ 5 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hook.go create mode 100644 hook_test.go diff --git a/README.md b/README.md index acdc631..e5142ef 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,29 @@ # logrus-datadog-hook -Sending logrus log to Datadog log API endpoint + +Shipping log entries from [logrus](https://github.com/sirupsen/logrus) to Datadog log API [HTTP endpoint](https://docs.datadoghq.com/api/?lang=bash#send-logs-over-http) + +## Example + +```golang + // Sending log in JSON format + hostName, _ := os.Hostname() + // When failure, retry up to 3 times with 5s interval + hook := datadog.NewHook(datadog.DatadogUSHost, apiKey, true, 3, 5*time.Second) + hook.Hostname = hostName + l := logrus.New() + l.Formatter = &logrus.JSONFormatter{} + l.Hooks.Add(hook) + l.WithField("from", "unitest").Infof("TestSendingJSON - %d", i) +``` + +```golang + // Sending log in plain text + hostName, _ := os.Hostname() + // When failure, retry up to 3 times with 5s interval + hook := datadog.NewHook(datadog.DatadogUSHost, apiKey, false, 3, 5*time.Second) + hook.Hostname = hostName + l := logrus.New() + l.Formatter = &logrus.TextFormatter{DisableColors: true} + l.Hooks.Add(hook) + l.WithField("from", "unitest").Infof("TestSendingText - %d", i) +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c8e4efb --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/bin3377/logrus-datadog-hook + +go 1.13 + +require github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..46e5ef0 --- /dev/null +++ b/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/hook.go b/hook.go new file mode 100644 index 0000000..56e6dee --- /dev/null +++ b/hook.go @@ -0,0 +1,219 @@ +package datadog + +import ( + "bytes" + "log" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +// Hook is the struct holding connect information to Datadog backend +type Hook struct { + Source string + Service string + Hostname string + Tags []string + + host string + apiKey string + isJSON bool + maxRetry int + buffer [][]byte + m sync.Mutex + ch chan string + err error +} + +const ( + // DatadogUSHost - Host For Datadog US + DatadogUSHost = "http-intake.logs.datadoghq.com" + // DatadogEUHost - Host For Datadog EU + DatadogEUHost = "http-intake.logs.datadoghq.eu" + + basePath = "/v1/input" + apiKeyHeader = "DD-API-KEY" + defaultTimeout = time.Second * 30 + + // ContentTypePlain - content is plain text + contentTypePlain = "text/plain" + + // ContentTypeJSON - content is JSON + contentTypeJSON = "application/json" + + // Maximum content size per payload: 5MB + maxContentByteSize = 5*1024*1024 - 2 + + // Maximum size for a single log: 256kB + maxEntryByteSize = 256 * 1024 + + // Maximum array size if sending multiple logs in an array: 500 entries + maxArraySize = 500 +) + +var ( + // Debug - print out debug log if true + Debug = false +) + +// NewHook - create hook with input +func NewHook( + host string, + apiKey string, + isJSON bool, + maxRetry int, + batchTimeout time.Duration, +) *Hook { + h := &Hook{ + host: host, + apiKey: apiKey, + isJSON: isJSON, + maxRetry: maxRetry, + } + + if batchTimeout <= 0 { + batchTimeout = defaultTimeout + } + h.ch = make(chan string, 1) + go h.pile(time.Tick(batchTimeout)) + return h +} + +// Levels - implement Hook interface supporting all levels +func (h *Hook) Levels() []logrus.Level { + return []logrus.Level{ + logrus.PanicLevel, + logrus.FatalLevel, + logrus.ErrorLevel, + logrus.WarnLevel, + logrus.InfoLevel, + logrus.DebugLevel, + } +} + +// Fire - implement Hook interface fire the entry +func (h *Hook) Fire(entry *logrus.Entry) error { + line, err := entry.String() + if err != nil { + dbg("Unable to read entry, %v", err) + return err + } + h.ch <- line + return h.err +} + +func (h *Hook) pile(ticker <-chan time.Time) { + var pile [][]byte + size := 0 + for { + select { + case str := <-h.ch: + if str == "" { + continue + } + if h.isJSON { + str = strings.TrimRight(str, "\n") + str += "," + } else if !strings.HasSuffix(str, "\n") { + str += "\n" + } + bytes := []byte(str) + messageSize := len(bytes) + if size+messageSize >= maxContentByteSize || len(pile) == maxArraySize { + go h.send(pile) + pile = make([][]byte, 0, maxArraySize) + size = 0 + } + pile = append(pile, bytes) + size += messageSize + case <-ticker: + go h.send(pile) + pile = make([][]byte, 0, maxArraySize) + size = 0 + } + } +} + +func (h *Hook) send(pile [][]byte) { + h.m.Lock() + defer h.m.Unlock() + if len(pile) == 0 { + return + } + + buf := make([]byte, 0) + for _, line := range pile { + buf = append(buf, line...) + } + if len(buf) == 0 { + return + } + if h.isJSON { + if buf[len(buf)-1] == ',' { + buf = buf[:len(buf)-1] + } + buf = append(buf, ']') + buf = append([]byte{'['}, buf...) + } + + dbg(string(buf)) + + req, err := http.NewRequest("POST", h.datadogURL(), bytes.NewBuffer(buf)) + if err != nil { + dbg(err.Error()) + return + } + header := http.Header{} + header.Add(apiKeyHeader, h.apiKey) + if h.isJSON { + header.Add("Content-Type", contentTypeJSON) + } else { + header.Add("Content-Type", contentTypePlain) + } + header.Add("charset", "UTF-8") + req.Header = header + + for i := 0; i <= h.maxRetry; i++ { + resp, err := http.DefaultClient.Do(req) + if err == nil { + dbg("%v", resp) + return + } + dbg(err.Error()) + } +} + +func (h *Hook) datadogURL() string { + u, err := url.Parse("https://" + h.host) + if err != nil { + dbg(err.Error()) + return "" + } + u.Path += basePath + parameters := url.Values{} + if h.Source != "" { + parameters.Add("ddsource", h.Source) + } + if h.Service != "" { + parameters.Add("service", h.Service) + } + if h.Hostname != "" { + parameters.Add("hostname", h.Hostname) + } + if h.Tags != nil { + tags := strings.Join(h.Tags, ",") + parameters.Add("ddtags", tags) + } + u.RawQuery = parameters.Encode() + return u.String() +} + +func dbg(format string, a ...interface{}) { + if Debug { + log.Printf(format+"\n", a...) + } +} diff --git a/hook_test.go b/hook_test.go new file mode 100644 index 0000000..32c5b36 --- /dev/null +++ b/hook_test.go @@ -0,0 +1,124 @@ +package datadog + +import ( + "log" + "os" + "path/filepath" + "reflect" + "runtime" + "sync" + "testing" + "time" + + "github.com/sirupsen/logrus" +) + +// assert fails the test if the condition is false. +func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { + if !condition { + _, file, line, _ := runtime.Caller(1) + log.Printf("%s:%d: "+msg+"\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) + tb.FailNow() + } +} + +// ok fails the test if an err is not nil. +func ok(tb testing.TB, err error) { + if err != nil { + _, file, line, _ := runtime.Caller(1) + log.Printf("%s:%d: unexpected error: %s\n\n", filepath.Base(file), line, err.Error()) + tb.FailNow() + } +} + +// equals fails the test if exp is not equal to act. +func equals(tb testing.TB, exp, act interface{}) { + if !reflect.DeepEqual(exp, act) { + _, file, line, _ := runtime.Caller(1) + log.Printf("%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\n\n", filepath.Base(file), line, exp, act) + tb.FailNow() + } +} + +func getTextLogger(t *testing.T) (*Hook, *logrus.Logger) { + host := os.Getenv("DATADOG_HOST") + apiKey := os.Getenv("DATADOG_APIKEY") + Debug = true + + if host == "" { + host = DatadogUSHost + } + if apiKey == "" { + t.Fatal("skipping test; DATADOG_APIKEY not set") + } + + hostName, _ := os.Hostname() + hook := NewHook(host, apiKey, false, 3, 5*time.Second) + hook.Hostname = hostName + l := logrus.New() + l.Formatter = &logrus.TextFormatter{DisableColors: true} + l.Hooks.Add(hook) + return hook, l +} + +func getJSONLogger(t *testing.T) (*Hook, *logrus.Logger) { + host := os.Getenv("DATADOG_HOST") + apiKey := os.Getenv("DATADOG_APIKEY") + Debug = true + + if host == "" { + host = DatadogUSHost + } + if apiKey == "" { + t.Fatal("skipping test; DATADOG_APIKEY not set") + } + + hostName, _ := os.Hostname() + hook := NewHook(host, apiKey, true, 3, 5*time.Second) + hook.Hostname = hostName + l := logrus.New() + l.Formatter = &logrus.JSONFormatter{} + l.Hooks.Add(hook) + return hook, l +} + +func TestHook(t *testing.T) { + hook, l := getTextLogger(t) + + for _, level := range hook.Levels() { + if len(l.Hooks[level]) != 1 { + t.Errorf("Hook was not added. The length of l.Hooks[%v]: %v", level, len(l.Hooks[level])) + } + } +} +func TestSendingJSON(t *testing.T) { + _, l := getJSONLogger(t) + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + l.WithField("from", "unitest").Infof("TestSendingJSON - %d", i) + }() + time.Sleep(1 * time.Second) + } + + wg.Wait() +} + +func TestSendingPlain(t *testing.T) { + _, l := getTextLogger(t) + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + l.WithField("from", "unitest").Infof("TestSendingPlain - %d", i) + }() + time.Sleep(1 * time.Second) + } + + wg.Wait() +}