Fix resource consumption issues (#125)

* Use varnishreload-compatible vcl names

#81 (comment)

* Honor varnishd max_vcl

https://varnish-cache.org/docs/6.0/reference/varnishd.html#max-vcl

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/json-iterator/go v1.1.11 // indirect
-	github.com/martin-helmich/go-varnish-client v0.2.1
+	github.com/martin-helmich/go-varnish-client v0.2.2
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect

go.sum

@@ -80,6 +80,8 @@ github.com/martin-helmich/go-varnish-client v0.2.0 h1:UNeB/8PQ+inkn1v57KLg2s3UkB
 github.com/martin-helmich/go-varnish-client v0.2.0/go.mod h1:BPZwupDItHsp9jLWUCFtchhJoJlt1RIjWoPEfeWTRdI=
 github.com/martin-helmich/go-varnish-client v0.2.1 h1:+NTZ+Io0dXMVrLJ534ABt8zr34sZop1fjt683qybmhg=
 github.com/martin-helmich/go-varnish-client v0.2.1/go.mod h1:mTUoKZn5fbP67YKPfKP2Zi8Y6A/+99B0oCQBFuHOhks=
+github.com/martin-helmich/go-varnish-client v0.2.2 h1:WBW/SwBPX+SEJN2nKh1/mhdgRqdNLgndxFE4RyD51hE=
+github.com/martin-helmich/go-varnish-client v0.2.2/go.mod h1:BUVEoli2BmESGMvlRiytvurkdZhJQwlzFtTBmG9mfFA=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -126,6 +128,7 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490 h1:va0qYsIOza3Nlf2IncFyOql4/3XUq3vfge/Ad64bhlM=
 golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -194,6 +197,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.0.0-20181013054003-e94254e9898f h1:kLihNpdVw0H7RpDrS0XH9yI0+TI90p6YbaKlitJkaPc=
 k8s.io/api v0.0.0-20181013054003-e94254e9898f/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
 k8s.io/apimachinery v0.0.0-20181013010248-dcb88206cd7f h1:J4RMsuKRhuF+JxWL8Ip+w+lVxS/kypd+j0R93qPBD2c=

pkg/controller/watch.go

@@ -5,18 +5,18 @@ import (
 	"context"
 	"fmt"
 	"os/exec"
+	"sort"
+	"strconv"
+	"strings"
 	"text/template"
+	"time"
 
 	"github.com/golang/glog"
 	varnishclient "github.com/martin-helmich/go-varnish-client"
 )
 
 func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd, errors chan<- error) {
-	i := 0
-
 	for {
-		i++
-
 		select {
 		case tmplContents := <-v.vclTemplateUpdates:
 			glog.Infof("VCL template has been updated")
@@ -29,7 +29,7 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd,
 
 			v.vclTemplate = tmpl
 
-			errors <- v.rebuildConfig(ctx, i)
+			errors <- v.rebuildConfig(ctx)
 
 		case newConfig := <-v.frontendUpdates:
 			glog.Infof("received new frontend configuration: %+v", newConfig)
@@ -40,14 +40,14 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd,
 				v.varnishSignaller.SetEndpoints(v.frontend)
 			}
 
-			errors <- v.rebuildConfig(ctx, i)
+			errors <- v.rebuildConfig(ctx)
 
 		case newConfig := <-v.backendUpdates:
 			glog.Infof("received new backend configuration: %+v", newConfig)
 
 			v.backend = newConfig
 
-			errors <- v.rebuildConfig(ctx, i)
+			errors <- v.rebuildConfig(ctx)
 
 		case <-ctx.Done():
 			errors <- ctx.Err()
@@ -56,7 +56,7 @@ func (v *VarnishController) watchConfigUpdates(ctx context.Context, c *exec.Cmd,
 	}
 }
 
-func (v *VarnishController) rebuildConfig(ctx context.Context, i int) error {
+func (v *VarnishController) rebuildConfig(ctx context.Context) error {
 	buf := new(bytes.Buffer)
 
 	err := v.renderVCL(buf, v.frontend.Endpoints, v.frontend.Primary, v.backend.Endpoints, v.backend.Primary)
@@ -77,7 +77,46 @@ func (v *VarnishController) rebuildConfig(ctx context.Context, i int) error {
 		return err
 	}
 
-	configname := fmt.Sprintf("k8s-upstreamcfg-%d", i)
+	maxVclParam, err := client.GetParameter(ctx, "max_vcl")
+	if err != nil {
+		return err
+	}
+
+	maxVcl, err := strconv.Atoi(maxVclParam.Value)
+	if err != nil {
+		return err
+	}
+
+	loadedVcl, err := client.ListVCL(ctx)
+	if err != nil {
+		return err
+	}
+
+	availableVcl := make([]varnishclient.VCLConfig, 0)
+
+	for i := range loadedVcl {
+		if loadedVcl[i].Status == varnishclient.VCLAvailable {
+			availableVcl = append(availableVcl, loadedVcl[i])
+		}
+	}
+
+	if len(loadedVcl) >= maxVcl {
+		// we're abusing the fact that "boot" < "reload"
+		sort.Slice(availableVcl, func(i, j int) bool {
+			return availableVcl[i].Name < availableVcl[j].Name
+		})
+
+		for i := 0; i < len(loadedVcl)-maxVcl+1; i++ {
+			glog.V(8).Infof("discarding VCL: %s", availableVcl[i].Name)
+
+			err = client.DiscardVCL(ctx, availableVcl[i].Name)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	configname := strings.ReplaceAll(time.Now().Format("reload_20060102_150405.00000"), ".", "_")
 
 	err = client.DefineInlineVCL(ctx, configname, vcl, varnishclient.VCLStateAuto)
 	if err != nil {