From 11e634dce29e335ba6e79ffe7c29703ff8009a5e Mon Sep 17 00:00:00 2001 From: Vincent van Dam Date: Thu, 20 May 2021 22:22:03 +0200 Subject: [PATCH] Improve resource cleaning --- Makefile | 2 +- README.md | 10 +- cmd/root.go | 6 +- go.sum | 2 - internal/backend/delete.go | 72 ++++++++++---- internal/backend/delete_test.go | 169 ++++++++++++++++++++++++++++++-- internal/backend/main.go | 2 + internal/main.go | 31 ++++++ 8 files changed, 260 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 77c2697..c8747f5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ run: - go run main.go -v 2 + go run main.go -pP -v 2 build: CGO_ENABLED=0 go build -ldflags \ diff --git a/README.md b/README.md index 383745d..193bb61 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Kubedock is an experimental implementation of the docker api that will orchestrate containers into a kubernetes cluster, rather than running containers locally. The main driver for this project is to be able running [testcontainers-java](https://www.testcontainers.org) enabled unit-tests in k8s, without the requirement of running docker-in-docker within resource heavy containers. -The current implementation is limited, but able to run containers that just expose ports, copy resources towards the container, or mount volumes. Containers that 'just' expose ports, require logging and copy resources to running containers will probably work. Volume mounting is implemented by copying the local volume towards the container, changes made by the container to this volume are not synced back. All data is considered emphemeral. +The current implementation is limited, but able to run containers that just expose ports, copy resources towards the container, or mount volumes. Containers that 'just' expose ports, require logging and copy resources to running containers will probably work. Volume mounting is implemented by copying the local volume towards the container, changes made by the container to this volume are not synced back. All data is considered emphemeral. If a container has network aliases configured, it will create k8s services with the alias as name. However, if aliases are present, a port mapping should be configured as well (as a service requires a specific port mapping). ## Quick start @@ -40,7 +40,13 @@ The below use-cases are mostly not working: ## Resource reaping -Kubedock will dynamically create deployments and services in the configured namespace. If kubedock is requested to delete a container, it will remove the deployment and related services. However, if e.g. a test fails and didn't clean up its started containers, these resources will remain in the namespace. To prevent unused deployments and services lingering around, kubedock will automatically delete deployments and services that are older than 5 minutes (default) if it's owned by the current process. If the deployment is not owned by the running process, it will delete it after 10 minutes if the deployment or service has the label `kubedock=true`. +### Automatic reaping + +Kubedock will dynamically create deployments and services in the configured namespace. If kubedock is requested to delete a container, it will remove the deployment and related services. However, if e.g. a test fails and didn't clean up its started containers, these resources will remain in the namespace. To prevent unused deployments and services lingering around, kubedock will automatically delete deployments and services that are older than 15 minutes (default) if it's owned by the current process. If the deployment is not owned by the running process, it will delete it after 30 minutes if the deployment or service has the label `kubedock=true`. + +### Forced cleaning + +The reaping of resources can also be enforced at startup, and at exit. When kubedock is started with the `--prune-start` argument, it will delete all resources that have the `kubedock=true` before starting the API server. If the `--prune-exit` argument is set, kubedock will delete all the resources it created in the running instance before exiting (identified with the `kubedock.id` label). # See also diff --git a/cmd/root.go b/cmd/root.go index 7df2f00..d79e8da 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -39,8 +39,10 @@ func init() { rootCmd.PersistentFlags().StringP("namespace", "n", "default", "Namespace in which containers should be orchestrated") rootCmd.PersistentFlags().String("initimage", config.Image, "Image to use as initcontainer for volume setup") rootCmd.PersistentFlags().DurationP("timeout", "t", 1*time.Minute, "Container creating timeout") - rootCmd.PersistentFlags().DurationP("reapmax", "r", 5*time.Minute, "Reap all resources older than this time") + rootCmd.PersistentFlags().DurationP("reapmax", "r", 15*time.Minute, "Reap all resources older than this time") rootCmd.PersistentFlags().StringP("verbosity", "v", "1", "Log verbosity level") + rootCmd.PersistentFlags().BoolP("prune-start", "P", false, "Prune all existing kubedock resources before starting") + rootCmd.PersistentFlags().BoolP("prune-exit", "p", false, "Prune all created resources on exit") viper.BindPFlag("server.listen-addr", rootCmd.PersistentFlags().Lookup("listen-addr")) viper.BindPFlag("server.socket", rootCmd.PersistentFlags().Lookup("socket")) @@ -52,6 +54,8 @@ func init() { viper.BindPFlag("kubernetes.timeout", rootCmd.PersistentFlags().Lookup("timeout")) viper.BindPFlag("reaper.reapmax", rootCmd.PersistentFlags().Lookup("reapmax")) viper.BindPFlag("verbosity", rootCmd.PersistentFlags().Lookup("verbosity")) + viper.BindPFlag("prune-start", rootCmd.PersistentFlags().Lookup("prune-start")) + viper.BindPFlag("prune-exit", rootCmd.PersistentFlags().Lookup("prune-exit")) viper.BindEnv("server.listen-addr", "SERVER_LISTEN_ADDR") viper.BindEnv("server.socket", "SERVER_SOCKET") diff --git a/go.sum b/go.sum index 4633abd..4ecf0c4 100644 --- a/go.sum +++ b/go.sum @@ -101,8 +101,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.7.1 h1:qC89GU3p8TvKWMAVhEpmpB2CIb1hnqt2UdKZaP93mS8= -github.com/gin-gonic/gin v1.7.1/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/gin-gonic/gin v1.7.2-0.20210519235755-e72e584d1aba h1:2jUZdpT0sXVBSeXbatd/CdRBaOK8YYFlnx5v5LBmOj4= github.com/gin-gonic/gin v1.7.2-0.20210519235755-e72e584d1aba/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= diff --git a/internal/backend/delete.go b/internal/backend/delete.go index e316806..bc97562 100644 --- a/internal/backend/delete.go +++ b/internal/backend/delete.go @@ -10,12 +10,28 @@ import ( "github.com/joyrex2001/kubedock/internal/model/types" ) +// DeleteAll will delete all resources that kubedock=true +func (in *instance) DeleteAll() error { + if err := in.deleteServices("kubedock=true"); err != nil { + klog.Errorf("error deleting services: %s", err) + } + return in.deleteDeployments("kubedock=true") +} + +// DeleteWithKubedockID will delete all resources that have given kubedock.id +func (in *instance) DeleteWithKubedockID(id string) error { + if err := in.deleteServices("kubedock.id=" + id); err != nil { + klog.Errorf("error deleting services: %s", err) + } + return in.deleteDeployments("kubedock.id=" + id) +} + // DeleteContainer will delete given container object in kubernetes. func (in *instance) DeleteContainer(tainr *types.Container) error { - if err := in.deleteServices(tainr.ShortID); err != nil { + if err := in.deleteServices("kubedock.containerid=" + tainr.ShortID); err != nil { klog.Errorf("error deleting services: %s", err) } - return in.cli.AppsV1().Deployments(in.namespace).Delete(context.TODO(), tainr.ShortID, metav1.DeleteOptions{}) + return in.deleteDeployments("kubedock.containerid=" + tainr.ShortID) } // DeleteContainersOlderThan will delete containers than are orchestrated @@ -28,14 +44,9 @@ func (in *instance) DeleteContainersOlderThan(keepmax time.Duration) error { return err } for _, dep := range deps.Items { - if dep.ObjectMeta.DeletionTimestamp != nil { - klog.V(3).Infof("skipping deployment %v, already in deleting state", dep) - continue - } - old := metav1.NewTime(time.Now().Add(-keepmax)) - if dep.ObjectMeta.CreationTimestamp.Before(&old) { + if in.isOlderThan(dep.ObjectMeta, keepmax) { klog.V(3).Infof("deleting deployment: %s", dep.Name) - if err := in.deleteServices(dep.Name); err != nil { + if err := in.deleteServices("kubedock.containerid=" + dep.Name); err != nil { klog.Errorf("error deleting services: %s", err) } if err := in.cli.AppsV1().Deployments(dep.Namespace).Delete(context.TODO(), dep.Name, metav1.DeleteOptions{}); err != nil { @@ -56,12 +67,7 @@ func (in *instance) DeleteServicesOlderThan(keepmax time.Duration) error { return err } for _, svc := range svcs.Items { - if svc.ObjectMeta.DeletionTimestamp != nil { - klog.V(3).Infof("skipping service %v, already in deleting state", svc) - continue - } - old := metav1.NewTime(time.Now().Add(-keepmax)) - if svc.ObjectMeta.CreationTimestamp.Before(&old) { + if in.isOlderThan(svc.ObjectMeta, keepmax) { klog.V(3).Infof("deleting service: %s", svc.Name) if err := in.cli.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}); err != nil { return err @@ -71,11 +77,22 @@ func (in *instance) DeleteServicesOlderThan(keepmax time.Duration) error { return nil } -// deleteServices will delete k8s service resources which have the -// label kubedock with the given id as value. -func (in *instance) deleteServices(id string) error { +// isOlderThan will check if given resource metadata has an older timestamp +// compared to given keepmax duration +func (in *instance) isOlderThan(met metav1.ObjectMeta, keepmax time.Duration) bool { + if met.DeletionTimestamp != nil { + klog.V(3).Infof("ignoring %v, already in deleting state", met) + return false + } + old := metav1.NewTime(time.Now().Add(-keepmax)) + return met.CreationTimestamp.Before(&old) +} + +// deleteServices will delete k8s service resources which match the +// given label selector. +func (in *instance) deleteServices(selector string) error { svcs, err := in.cli.CoreV1().Services(in.namespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "kubedock.containerid=" + id, + LabelSelector: selector, }) if err != nil { return err @@ -87,3 +104,20 @@ func (in *instance) deleteServices(id string) error { } return nil } + +// deleteDeployments will delete k8s deployments resources which match the +// given label selector. +func (in *instance) deleteDeployments(selector string) error { + deps, err := in.cli.AppsV1().Deployments(in.namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return err + } + for _, svc := range deps.Items { + if err := in.cli.AppsV1().Deployments(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{}); err != nil { + return err + } + } + return nil +} diff --git a/internal/backend/delete_test.go b/internal/backend/delete_test.go index 77e6d54..4c14958 100644 --- a/internal/backend/delete_test.go +++ b/internal/backend/delete_test.go @@ -13,11 +13,11 @@ import ( "github.com/joyrex2001/kubedock/internal/model/types" ) -func TestDeleteContainer(t *testing.T) { +func TestDeleteContainerKubedockID(t *testing.T) { tests := []struct { in *types.Container kub *instance - out bool + ins int }{ { kub: &instance{ @@ -27,20 +27,171 @@ func TestDeleteContainer(t *testing.T) { Name: "tb303", Namespace: "default", }, - Status: appsv1.DeploymentStatus{ - ReadyReplicas: 1, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + ins: 1, + }, + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + Labels: map[string]string{"kubedock.containerid": "tb303", "kubedock.id": "6502"}, + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + ins: 1, + }, + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + Labels: map[string]string{"kubedock.containerid": "tb303", "kubedock.id": "z80"}, }, }), }, in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, - out: false, + ins: 0, }, } for i, tst := range tests { - res := tst.kub.DeleteContainer(tst.in) - if (res != nil && !tst.out) || (res == nil && tst.out) { - t.Errorf("failed test %d - unexpected return value %s", i, res) + if err := tst.kub.DeleteWithKubedockID("z80"); err != nil { + t.Errorf("failed test %d - unexpected error %s", i, err) + } + deps, _ := tst.kub.cli.AppsV1().Deployments("default").List(context.TODO(), metav1.ListOptions{}) + cnt := len(deps.Items) + if cnt != tst.ins { + t.Errorf("failed delete instances test %d - expected %d remaining deployments but got %d", i, tst.ins, cnt) + } + } +} + +func TestDeleteContainers(t *testing.T) { + tests := []struct { + in *types.Container + kub *instance + cnt int + }{ + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + cnt: 1, + }, + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + Labels: map[string]string{"kubedock.containerid": "tb303", "kubedock.id": "6502"}, + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + cnt: 0, + }, + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + Labels: map[string]string{"kubedock.containerid": "tb303", "kubedock.id": "z80"}, + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + cnt: 0, + }, + } + + for i, tst := range tests { + if err := tst.kub.DeleteContainer(tst.in); err != nil { + t.Errorf("failed test %d - unexpected error %s", i, err) + } + deps, _ := tst.kub.cli.AppsV1().Deployments("default").List(context.TODO(), metav1.ListOptions{}) + cnt := len(deps.Items) + if cnt != tst.cnt { + t.Errorf("failed test %d - expected %d remaining deployments but got %d", i, tst.cnt, cnt) + } + } +} + +func TestDeleteContainerKubedock(t *testing.T) { + tests := []struct { + in *types.Container + kub *instance + all int + }{ + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + all: 1, + }, + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + Labels: map[string]string{"kubedock": "true", "kubedock.id": "6502"}, + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + all: 0, + }, + { + kub: &instance{ + namespace: "default", + cli: fake.NewSimpleClientset(&appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tb303", + Namespace: "default", + Labels: map[string]string{"kubedock": "true", "kubedock.id": "z80"}, + }, + }), + }, + in: &types.Container{ID: "rc752", ShortID: "tb303", Name: "f1spirit"}, + all: 0, + }, + } + + for i, tst := range tests { + if err := tst.kub.DeleteAll(); err != nil { + t.Errorf("failed test %d - unexpected error %s", i, err) + } + deps, _ := tst.kub.cli.AppsV1().Deployments("default").List(context.TODO(), metav1.ListOptions{}) + cnt := len(deps.Items) + if cnt != tst.all { + t.Errorf("failed delete all test %d - expected %d remaining deployments but got %d", i, tst.all, cnt) } } } @@ -81,7 +232,7 @@ func TestDeleteServices(t *testing.T) { } for i, tst := range tests { - if err := tst.kub.deleteServices(tst.id); err != nil { + if err := tst.kub.deleteServices("kubedock.containerid=" + tst.id); err != nil { t.Errorf("failed test %d - unexpected error %s", i, err) } svcs, _ := tst.kub.cli.CoreV1().Services("default").List(context.TODO(), metav1.ListOptions{}) diff --git a/internal/backend/main.go b/internal/backend/main.go index 71d359b..62c17b9 100644 --- a/internal/backend/main.go +++ b/internal/backend/main.go @@ -14,6 +14,8 @@ import ( type Backend interface { StartContainer(*types.Container) error CreateServices(*types.Container) error + DeleteAll() error + DeleteWithKubedockID(string) error DeleteContainer(*types.Container) error DeleteContainersOlderThan(time.Duration) error DeleteServicesOlderThan(time.Duration) error diff --git a/internal/main.go b/internal/main.go index 396f746..1b79a54 100644 --- a/internal/main.go +++ b/internal/main.go @@ -1,6 +1,10 @@ package internal import ( + "os" + "os/signal" + "syscall" + "github.com/spf13/viper" "k8s.io/client-go/kubernetes" "k8s.io/klog" @@ -28,6 +32,16 @@ func Main() { } rpr.Start() + if viper.GetBool("prune-start") { + if err := kub.DeleteAll(); err != nil { + klog.Fatalf("error pruning resources: %s", err) + } + } + + if viper.GetBool("prune-exit") { + pruneAtExit(kub) + } + svr := server.New(kub) if err := svr.Run(); err != nil { klog.Fatalf("error instantiating server: %s", err) @@ -53,3 +67,20 @@ func getBackend() (backend.Backend, error) { }) return kub, nil } + +// pruneAtExit will clean up resources when kubedock exits +func pruneAtExit(kub backend.Backend) { + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + go func() { + <-sigc + if err := kub.DeleteWithKubedockID(config.DefaultLabels["kubedock.id"]); err != nil { + klog.Fatalf("error pruning resources: %s", err) + } + os.Exit(0) + }() +}