initial commit

README.md

@@ -1,2 +1,29 @@
 # logrus-datadog-hook
-Sending logrus log to Datadog log API endpoint
+
+Shipping log entries from [logrus](https://github.com/sirupsen/logrus) to Datadog log API [HTTP endpoint](https://docs.datadoghq.com/api/?lang=bash#send-logs-over-http)
+
+## Example
+
+```golang
+    // Sending log in JSON format
+    hostName, _ := os.Hostname()
+    // When failure, retry up to 3 times with 5s interval
+    hook := datadog.NewHook(datadog.DatadogUSHost, apiKey, true, 3, 5*time.Second)
+    hook.Hostname = hostName
+    l := logrus.New()
+    l.Formatter = &logrus.JSONFormatter{}
+    l.Hooks.Add(hook)
+    l.WithField("from", "unitest").Infof("TestSendingJSON - %d", i)
+```
+
+```golang
+    // Sending log in plain text
+    hostName, _ := os.Hostname()
+    // When failure, retry up to 3 times with 5s interval
+    hook := datadog.NewHook(datadog.DatadogUSHost, apiKey, false, 3, 5*time.Second)
+    hook.Hostname = hostName
+    l := logrus.New()
+    l.Formatter = &logrus.TextFormatter{DisableColors: true}
+    l.Hooks.Add(hook)
+    l.WithField("from", "unitest").Infof("TestSendingText - %d", i)
+```

go.mod

@@ -0,0 +1,5 @@
+module github.com/bin3377/logrus-datadog-hook
+
+go 1.13
+
+require github.com/sirupsen/logrus v1.4.2

go.sum

@@ -0,0 +1,9 @@
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

hook.go

@@ -0,0 +1,219 @@
+package datadog
+
+import (
+	"bytes"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Hook is the struct holding connect information to Datadog backend
+type Hook struct {
+	Source   string
+	Service  string
+	Hostname string
+	Tags     []string
+
+	host     string
+	apiKey   string
+	isJSON   bool
+	maxRetry int
+	buffer   [][]byte
+	m        sync.Mutex
+	ch       chan string
+	err      error
+}
+
+const (
+	// DatadogUSHost - Host For Datadog US
+	DatadogUSHost = "http-intake.logs.datadoghq.com"
+	// DatadogEUHost - Host For Datadog EU
+	DatadogEUHost = "http-intake.logs.datadoghq.eu"
+
+	basePath       = "/v1/input"
+	apiKeyHeader   = "DD-API-KEY"
+	defaultTimeout = time.Second * 30
+
+	// ContentTypePlain - content is plain text
+	contentTypePlain = "text/plain"
+
+	// ContentTypeJSON - content is JSON
+	contentTypeJSON = "application/json"
+
+	// Maximum content size per payload: 5MB
+	maxContentByteSize = 5*1024*1024 - 2
+
+	// Maximum size for a single log: 256kB
+	maxEntryByteSize = 256 * 1024
+
+	// Maximum array size if sending multiple logs in an array: 500 entries
+	maxArraySize = 500
+)
+
+var (
+	// Debug - print out debug log if true
+	Debug = false
+)
+
+// NewHook - create hook with input
+func NewHook(
+	host string,
+	apiKey string,
+	isJSON bool,
+	maxRetry int,
+	batchTimeout time.Duration,
+) *Hook {
+	h := &Hook{
+		host:     host,
+		apiKey:   apiKey,
+		isJSON:   isJSON,
+		maxRetry: maxRetry,
+	}
+
+	if batchTimeout <= 0 {
+		batchTimeout = defaultTimeout
+	}
+	h.ch = make(chan string, 1)
+	go h.pile(time.Tick(batchTimeout))
+	return h
+}
+
+// Levels - implement Hook interface supporting all levels
+func (h *Hook) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}
+
+// Fire - implement Hook interface fire the entry
+func (h *Hook) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		dbg("Unable to read entry, %v", err)
+		return err
+	}
+	h.ch <- line
+	return h.err
+}
+
+func (h *Hook) pile(ticker <-chan time.Time) {
+	var pile [][]byte
+	size := 0
+	for {
+		select {
+		case str := <-h.ch:
+			if str == "" {
+				continue
+			}
+			if h.isJSON {
+				str = strings.TrimRight(str, "\n")
+				str += ","
+			} else if !strings.HasSuffix(str, "\n") {
+				str += "\n"
+			}
+			bytes := []byte(str)
+			messageSize := len(bytes)
+			if size+messageSize >= maxContentByteSize || len(pile) == maxArraySize {
+				go h.send(pile)
+				pile = make([][]byte, 0, maxArraySize)
+				size = 0
+			}
+			pile = append(pile, bytes)
+			size += messageSize
+		case <-ticker:
+			go h.send(pile)
+			pile = make([][]byte, 0, maxArraySize)
+			size = 0
+		}
+	}
+}
+
+func (h *Hook) send(pile [][]byte) {
+	h.m.Lock()
+	defer h.m.Unlock()
+	if len(pile) == 0 {
+		return
+	}
+
+	buf := make([]byte, 0)
+	for _, line := range pile {
+		buf = append(buf, line...)
+	}
+	if len(buf) == 0 {
+		return
+	}
+	if h.isJSON {
+		if buf[len(buf)-1] == ',' {
+			buf = buf[:len(buf)-1]
+		}
+		buf = append(buf, ']')
+		buf = append([]byte{'['}, buf...)
+	}
+
+	dbg(string(buf))
+
+	req, err := http.NewRequest("POST", h.datadogURL(), bytes.NewBuffer(buf))
+	if err != nil {
+		dbg(err.Error())
+		return
+	}
+	header := http.Header{}
+	header.Add(apiKeyHeader, h.apiKey)
+	if h.isJSON {
+		header.Add("Content-Type", contentTypeJSON)
+	} else {
+		header.Add("Content-Type", contentTypePlain)
+	}
+	header.Add("charset", "UTF-8")
+	req.Header = header
+
+	for i := 0; i <= h.maxRetry; i++ {
+		resp, err := http.DefaultClient.Do(req)
+		if err == nil {
+			dbg("%v", resp)
+			return
+		}
+		dbg(err.Error())
+	}
+}
+
+func (h *Hook) datadogURL() string {
+	u, err := url.Parse("https://" + h.host)
+	if err != nil {
+		dbg(err.Error())
+		return ""
+	}
+	u.Path += basePath
+	parameters := url.Values{}
+	if h.Source != "" {
+		parameters.Add("ddsource", h.Source)
+	}
+	if h.Service != "" {
+		parameters.Add("service", h.Service)
+	}
+	if h.Hostname != "" {
+		parameters.Add("hostname", h.Hostname)
+	}
+	if h.Tags != nil {
+		tags := strings.Join(h.Tags, ",")
+		parameters.Add("ddtags", tags)
+	}
+	u.RawQuery = parameters.Encode()
+	return u.String()
+}
+
+func dbg(format string, a ...interface{}) {
+	if Debug {
+		log.Printf(format+"\n", a...)
+	}
+}

hook_test.go

@@ -0,0 +1,124 @@
+package datadog
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// assert fails the test if the condition is false.
+func assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
+	if !condition {
+		_, file, line, _ := runtime.Caller(1)
+		log.Printf("%s:%d: "+msg+"\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
+		tb.FailNow()
+	}
+}
+
+// ok fails the test if an err is not nil.
+func ok(tb testing.TB, err error) {
+	if err != nil {
+		_, file, line, _ := runtime.Caller(1)
+		log.Printf("%s:%d: unexpected error: %s\n\n", filepath.Base(file), line, err.Error())
+		tb.FailNow()
+	}
+}
+
+// equals fails the test if exp is not equal to act.
+func equals(tb testing.TB, exp, act interface{}) {
+	if !reflect.DeepEqual(exp, act) {
+		_, file, line, _ := runtime.Caller(1)
+		log.Printf("%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\n\n", filepath.Base(file), line, exp, act)
+		tb.FailNow()
+	}
+}
+
+func getTextLogger(t *testing.T) (*Hook, *logrus.Logger) {
+	host := os.Getenv("DATADOG_HOST")
+	apiKey := os.Getenv("DATADOG_APIKEY")
+	Debug = true
+
+	if host == "" {
+		host = DatadogUSHost
+	}
+	if apiKey == "" {
+		t.Fatal("skipping test; DATADOG_APIKEY not set")
+	}
+
+	hostName, _ := os.Hostname()
+	hook := NewHook(host, apiKey, false, 3, 5*time.Second)
+	hook.Hostname = hostName
+	l := logrus.New()
+	l.Formatter = &logrus.TextFormatter{DisableColors: true}
+	l.Hooks.Add(hook)
+	return hook, l
+}
+
+func getJSONLogger(t *testing.T) (*Hook, *logrus.Logger) {
+	host := os.Getenv("DATADOG_HOST")
+	apiKey := os.Getenv("DATADOG_APIKEY")
+	Debug = true
+
+	if host == "" {
+		host = DatadogUSHost
+	}
+	if apiKey == "" {
+		t.Fatal("skipping test; DATADOG_APIKEY not set")
+	}
+
+	hostName, _ := os.Hostname()
+	hook := NewHook(host, apiKey, true, 3, 5*time.Second)
+	hook.Hostname = hostName
+	l := logrus.New()
+	l.Formatter = &logrus.JSONFormatter{}
+	l.Hooks.Add(hook)
+	return hook, l
+}
+
+func TestHook(t *testing.T) {
+	hook, l := getTextLogger(t)
+
+	for _, level := range hook.Levels() {
+		if len(l.Hooks[level]) != 1 {
+			t.Errorf("Hook was not added. The length of l.Hooks[%v]: %v", level, len(l.Hooks[level]))
+		}
+	}
+}
+func TestSendingJSON(t *testing.T) {
+	_, l := getJSONLogger(t)
+
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			l.WithField("from", "unitest").Infof("TestSendingJSON - %d", i)
+		}()
+		time.Sleep(1 * time.Second)
+	}
+
+	wg.Wait()
+}
+
+func TestSendingPlain(t *testing.T) {
+	_, l := getTextLogger(t)
+
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			l.WithField("from", "unitest").Infof("TestSendingPlain - %d", i)
+		}()
+		time.Sleep(1 * time.Second)
+	}
+
+	wg.Wait()
+}