fix: logging and pf improvements, skip namespaces & non-ClusterIP svcs

This PR contains a number of reliability and UX improvements for using
localizer in a larger cluster.

* When creating a port-forward, we don't pass a stop channel anymore.
  This was previously passed `ctx.Done()` which would silently stop the
  port-forward on ^C. This caused invalid "use of closed connection"
  logging that was confusing to the user but also possibly could end up
  in a port-forward not being fully stopped.
* When shutting down, pass a temporary context that has a timeout of 30
  seconds. This ensures that `/etc/host` modifications, ip pool
  cleanups, and other shutdown functions properly finish instead of
  possibly being terminated midway through during normal shutdown.
* Enabled delibird to remove tracing/metrics information being sent by
  default in localizer binaries. This is a OSS friendly way of us
  getting telemetry by logging to disk instead, which a user can opt to
  send us when reporting bugs (more on that in future work!)
* Upgraded all dependencies, including `client-go`. We still need a
  fork, but thankfully our fork is a single line now so it's much more
  likely we can upstream the change we made (the other change was
  upstreamed a year ago by another user!)
* Exposes `stderr` from the port-forwarder to the console with a colored
  prefix. This makes it easier to tell when a port-forward was busted
  but localizer didn't detect it (or if it did, why it failed!)
* Skipped namespaces by default (`kube-system`) and enabled the user to
  pass `--skip-namespace` to provide more (helps #212).
* Skip non-clusterIP services. These aren't addressable in the cluster,
  so we shouldn't create a tunnel for them. This helps with `NodePort`
  and `LoadBalancer` service overhead that some larger clusters may
  have.
* Reduced logging to be easier to read, while keeping a lot of helpful
  logs in the `debug` level. Defaults to `info` logging instead of
  `debug` logging (pass `--log-level debug` for that!)

.goreleaser.yml

@@ -14,8 +14,6 @@ builds:
       - arm64
     ldflags:
       - '-w -s -X "github.com/getoutreach/gobox/pkg/app.Version=v{{ .Version }}"'
-      - '-X "main.HoneycombTracingKey={{ .Env.HONEYCOMB_APIKEY }}"'
-      - '-X "main.TeleforkAPIKey={{ .Env.TELEFORK_APIKEY }}"'
     env:
       - CGO_ENABLED=0
 

cmd/localizer/localizer.go

@@ -16,7 +16,6 @@ import (
 	"syscall"
 
 	oapp "github.com/getoutreach/gobox/pkg/app"
-	"github.com/getoutreach/gobox/pkg/cfg"
 	gcli "github.com/getoutreach/gobox/pkg/cli"
 	"github.com/getoutreach/localizer/internal/kevents"
 	"github.com/getoutreach/localizer/internal/kube"
@@ -32,22 +31,6 @@ import (
 	// <</Stencil::Block>>
 )
 
-// HoneycombTracingKey gets set by the Makefile at compile-time which is pulled
-// down by devconfig.sh.
-var HoneycombTracingKey = "NOTSET" //nolint:gochecknoglobals // Why: We can't compile in things as a const.
-
-// TeleforkAPIKey gets set by the Makefile at compile-time which is pulled
-// down by devconfig.sh.
-var TeleforkAPIKey = "NOTSET" //nolint:gochecknoglobals // Why: We can't compile in things as a const.
-
-// <<Stencil::Block(honeycombDataset)>>
-
-// HoneycombDataset is a constant denoting the dataset that traces should be stored
-// in in honeycomb.
-const HoneycombDataset = ""
-
-// <</Stencil::Block>>
-
 // <<Stencil::Block(global)>>
 
 // <</Stencil::Block>>
@@ -78,11 +61,9 @@ func main() {
 			EnvVars: []string{"KUBECONTEXT"},
 		},
 		&cli.StringFlag{
-			Name:        "log-level",
-			Usage:       "Set the log level",
-			EnvVars:     []string{"LOG_LEVEL"},
-			Value:       "DEBUG",
-			DefaultText: "DEBUG",
+			Name:    "log-level",
+			Usage:   "Set the log level. Valid values are: debug",
+			EnvVars: []string{"LOG_LEVEL"},
 		},
 		&cli.StringFlag{
 			Name:        "log-format",
@@ -104,6 +85,10 @@ func main() {
 			Name:  "namespace",
 			Usage: "Restrict forwarding to the given namespace. (default: all namespaces)",
 		},
+		&cli.StringSliceFlag{
+			Name:  "skip-namespace",
+			Usage: "Skip forwarding services from the following namespace",
+		},
 		// <</Stencil::Block>>
 	}
 	app.Commands = []*cli.Command{
@@ -140,7 +125,7 @@ func main() {
 			log.SetFormatter(&logrus.JSONFormatter{})
 		}
 
-		klog.SetLogger(logrusr.New(log))
+		klog.SetLogger(logrusr.New(log, logrusr.WithReportCaller()))
 
 		// setup the global kubernetes cache interface
 		config, k, err := kube.GetKubeClient(c.String("context"))
@@ -182,10 +167,7 @@ func main() {
 	gcli.Run(ctx, cancel, &app, &gcli.Config{
 		Logger: log,
 		Telemetry: gcli.TelemetryConfig{
-			Otel: gcli.TelemetryOtelConfig{
-				Dataset:         HoneycombDataset,
-				HoneycombAPIKey: cfg.SecretData(HoneycombTracingKey),
-			},
+			UseDelibird: true,
 		},
 	})
 }

go.mod

@@ -3,105 +3,117 @@ module github.com/getoutreach/localizer
 go 1.19
 
 require (
-	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
-	github.com/benbjohnson/clock v1.3.0
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
+	github.com/benbjohnson/clock v1.3.5
 	github.com/davecgh/go-spew v1.1.1
 	github.com/elazarl/goproxy v0.0.0-20210110162100-a92cc753f88e // indirect
-	github.com/function61/gokit v0.0.0-20210402130425-341c2c9ecfd0
+	github.com/function61/gokit v0.0.0-20230712092143-d63a51667e64
 	github.com/google/go-cmp v0.5.9
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/metal-stack/go-ipam v1.11.2
+	github.com/metal-stack/go-ipam v1.12.3
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.9.0
-	github.com/urfave/cli/v2 v2.16.3
-	golang.org/x/crypto v0.1.0
-	google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa // indirect
-	google.golang.org/grpc v1.53.0
-	google.golang.org/protobuf v1.28.1
+	github.com/sirupsen/logrus v1.9.3
+	github.com/urfave/cli/v2 v2.25.7
+	golang.org/x/crypto v0.11.0
+	google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/grpc v1.57.0
+	google.golang.org/protobuf v1.31.0
 
 	// kubernetes deps
-	k8s.io/api v0.23.1
-	k8s.io/apimachinery v0.24.0-alpha.3
-	k8s.io/client-go v0.23.1
-	k8s.io/klog/v2 v2.60.1
+	k8s.io/api v0.25.12
+	k8s.io/apimachinery v0.25.12
+	k8s.io/client-go v0.25.12
+	k8s.io/klog/v2 v2.100.1
 )
 
-require github.com/getoutreach/gobox v1.71.0
+require (
+	github.com/egymgmbh/go-prefix-writer v0.0.0-20180609083313-7326ea162eca
+	github.com/fatih/color v1.13.0
+	github.com/getoutreach/gobox v1.73.1
+)
 
 require (
-	cloud.google.com/go/compute v1.15.1 // indirect
+	cloud.google.com/go/compute v1.23.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.12 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.16 // indirect
+	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/alecthomas/chroma v0.10.0 // indirect
-	github.com/avast/retry-go/v4 v4.3.0 // indirect
+	github.com/avast/retry-go/v4 v4.3.4 // indirect
 	github.com/aymanbagabas/go-osc52 v1.0.3 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/briandowns/spinner v1.23.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/charmbracelet/glamour v0.6.0 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.4.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/creack/pty v1.1.17 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
 	github.com/emirpasic/gods v1.12.1 // indirect
-	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-github/v47 v47.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.3 // indirect
 	github.com/honeycombio/beeline-go v1.11.1 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.15.12 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/manifoldco/promptui v0.9.0 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.21 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/montanaflynn/stats v0.6.6 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.13.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/prometheus/client_golang v1.14.0 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/redis/go-redis/v9 v9.0.5 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/schollz/progressbar/v3 v3.13.1 // indirect
@@ -110,58 +122,61 @@ require (
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
-	github.com/xdg-go/scram v1.1.1 // indirect
-	github.com/xdg-go/stringprep v1.0.3 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	github.com/yuin/goldmark v1.5.2 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.5 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.5 // indirect
-	go.mongodb.org/mongo-driver v1.10.3 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.33.0 // indirect
-	go.opentelemetry.io/otel v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect
-	go.opentelemetry.io/otel/metric v0.31.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.14.0 // indirect
-	go.opentelemetry.io/otel/trace v1.14.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.9 // indirect
+	go.mongodb.org/mongo-driver v1.12.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 // indirect
+	go.opentelemetry.io/otel v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect
+	go.opentelemetry.io/otel/metric v1.16.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.16.0 // indirect
+	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.23.0 // indirect
-	go4.org/netipx v0.0.0-20220925034521-797b0c90d8ab // indirect
-	golang.org/x/net v0.7.0 // indirect
-	golang.org/x/oauth2 v0.4.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.25.0 // indirect
+	go4.org/netipx v0.0.0-20230728184502-ec4c8b891b28 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.13.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/tools v0.11.1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 )
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/bombsimon/logrusr/v2 v2.0.1
-	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/lib/pq v1.10.7 // indirect
+	github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	golang.org/x/time v0.1.0 // indirect
-	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace (
-	// We're on a fork, keep version at v0.21.0
-	k8s.io/api => k8s.io/api v0.21.0
-	k8s.io/apimachinery => k8s.io/apimachinery v0.21.0
-	k8s.io/client-go => github.com/jaredallard/client-go v0.21.0-jaredallard
+	// We're on a fork, keep version at v0.25.12
+	k8s.io/api => k8s.io/api v0.25.12
+	k8s.io/apimachinery => k8s.io/apimachinery v0.25.12
+	k8s.io/client-go => github.com/jaredallard/client-go v0.25.12-jaredallard.1
 )