From 863019dfb5ce7141cd1cd6de09ffe39df3f1de51 Mon Sep 17 00:00:00 2001 From: Samuel Lang Date: Thu, 9 Dec 2021 11:46:46 +0100 Subject: [PATCH] Application Load Balancer Support for End-to-End HTTP/2 ref: https://aws.amazon.com/de/blogs/aws/new-application-load-balancer-support-for-end-to-end-http-2-and-grpc/ Currently HTTP/2 is only half way implemented, that is the ALB can be configured for HTTP/2 but the TargetGroup is not configured. This is probably due to the AWS delay of supporting that protocal version for the TargetGroup which is available in meantime. There is however an update missing for `mweagle/go-cloudformation` which is deprecated, thus a fork has been made that includes this Cloudformation feature: https://github.com/o11n/go-cloudformation/commit/b975e65fc6d53c789602ceb28de10974ec69f87e Signed-off-by: Samuel Lang --- aws/adapter.go | 16 ++++++++++++++-- aws/cf.go | 1 + aws/cf_template.go | 6 +++++- aws/cf_template_test.go | 39 ++++++++++++++++++++++++++++++++++++++- aws/cloudwatch.go | 2 +- aws/cloudwatch_test.go | 2 +- controller.go | 10 +++++++++- go.mod | 4 +--- go.sum | 14 +++++--------- worker_test.go | 2 +- 10 files changed, 76 insertions(+), 20 deletions(-) diff --git a/aws/adapter.go b/aws/adapter.go index 2cb9b4ee..42840a53 100644 --- a/aws/adapter.go +++ b/aws/adapter.go @@ -74,6 +74,7 @@ type Adapter struct { denyInternalRespBody string denyInternalRespContentType string denyInternalRespStatusCode int + targetGroupProtocolVersion string } type manifest struct { @@ -121,8 +122,9 @@ const ( DefaultCustomFilter = "" // DefaultNLBCrossZone specifies the default configuration for cross // zone load balancing: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html#load-balancer-attributes - DefaultNLBCrossZone = false - DefaultNLBHTTPEnabled = false + DefaultNLBCrossZone = false + DefaultNLBHTTPEnabled = false + DefaultTargetGroupProtocolVersion = "HTTP1" nameTag = "Name" LoadBalancerTypeApplication = "application" @@ -225,6 +227,7 @@ func NewAdapter(clusterID, newControllerID, vpcID string, debug, disableInstrume nlbCrossZone: DefaultNLBCrossZone, nlbHTTPEnabled: DefaultNLBHTTPEnabled, customFilter: DefaultCustomFilter, + targetGroupProtocolVersion: DefaultTargetGroupProtocolVersion, } adapter.manifest, err = buildManifest(adapter, clusterID, vpcID) @@ -466,6 +469,13 @@ func (a *Adapter) WithInternalDomainsDenyResponseContenType(contentType string) return a } +// WithTargetGroupProtocolVersion returns the receiver +// adapter after setting targetGroupProtocolVersion config. +func (a *Adapter) WithTargetGroupProtocolVersion(pv string) *Adapter { + a.targetGroupProtocolVersion = pv + return a +} + // ClusterID returns the ClusterID tag that all resources from the same Kubernetes cluster share. // It's taken from the current ec2 instance. func (a *Adapter) ClusterID() string { @@ -671,6 +681,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o statusCode: a.denyInternalRespStatusCode, contentType: a.denyInternalRespContentType, }, + targetGroupProtocolVersion: a.targetGroupProtocolVersion, } return createStack(a.cloudformation, spec) @@ -726,6 +737,7 @@ func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time. statusCode: a.denyInternalRespStatusCode, contentType: a.denyInternalRespContentType, }, + targetGroupProtocolVersion: a.targetGroupProtocolVersion, } return updateStack(a.cloudformation, spec) diff --git a/aws/cf.go b/aws/cf.go index 058883dc..2f666e77 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -164,6 +164,7 @@ type stackSpec struct { denyInternalDomainsResponse denyResp internalDomains []string tags map[string]string + targetGroupProtocolVersion string } type healthCheck struct { diff --git a/aws/cf_template.go b/aws/cf_template.go index c5b0af06..69d48644 100644 --- a/aws/cf_template.go +++ b/aws/cf_template.go @@ -8,7 +8,7 @@ import ( "crypto/sha256" "sort" - cloudformation "github.com/mweagle/go-cloudformation" + cloudformation "github.com/o11n/go-cloudformation" ) const ( @@ -447,14 +447,17 @@ func newTargetGroup(spec *stackSpec, targetPortParameter string) *cloudformation protocol := "HTTP" healthCheckProtocol := "HTTP" healthyThresholdCount, unhealthyThresholdCount := spec.albHealthyThresholdCount, spec.albUnhealthyThresholdCount + protocolVersion := cloudformation.String("HTTP1") if spec.loadbalancerType == LoadBalancerTypeNetwork { protocol = "TCP" healthCheckProtocol = "HTTP" // For NLBs the healthy and unhealthy threshold count value must be equal healthyThresholdCount, unhealthyThresholdCount = spec.nlbHealthyThresholdCount, spec.nlbHealthyThresholdCount + protocolVersion = nil } else if spec.targetHTTPS { protocol = "HTTPS" healthCheckProtocol = "HTTPS" + protocolVersion = cloudformation.String(spec.targetGroupProtocolVersion) } targetGroup := &cloudformation.ElasticLoadBalancingV2TargetGroup{ @@ -472,6 +475,7 @@ func newTargetGroup(spec *stackSpec, targetPortParameter string) *cloudformation UnhealthyThresholdCount: cloudformation.Integer(int64(unhealthyThresholdCount)), Port: cloudformation.Ref(targetPortParameter).Integer(), Protocol: cloudformation.String(protocol), + ProtocolVersion: protocolVersion, VPCID: cloudformation.Ref(parameterTargetGroupVPCIDParameter).String(), } diff --git a/aws/cf_template_test.go b/aws/cf_template_test.go index addf8839..08cf4eae 100644 --- a/aws/cf_template_test.go +++ b/aws/cf_template_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - cloudformation "github.com/mweagle/go-cloudformation" + cloudformation "github.com/o11n/go-cloudformation" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -301,6 +301,8 @@ func TestGenerateTemplate(t *testing.T) { attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes) require.Equal(t, attributes[1].Key.Literal, "routing.http2.enabled") require.Equal(t, attributes[1].Value.Literal, "true") + tg := template.Resources["TG"].Properties.(*cloudformation.ElasticLoadBalancingV2TargetGroup) + require.Equal(t, cloudformation.String("HTTP1"), tg.ProtocolVersion) }, }, { @@ -315,6 +317,8 @@ func TestGenerateTemplate(t *testing.T) { attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes) require.Equal(t, attributes[1].Key.Literal, "routing.http2.enabled") require.Equal(t, attributes[1].Value.Literal, "false") + tg := template.Resources["TG"].Properties.(*cloudformation.ElasticLoadBalancingV2TargetGroup) + require.Equal(t, cloudformation.String("HTTP1"), tg.ProtocolVersion) }, }, { @@ -533,6 +537,7 @@ func TestGenerateTemplate(t *testing.T) { require.Equal(t, cloudformation.Ref(parameterTargetGroupTargetPortParameter).Integer(), tg.Port) require.Equal(t, cloudformation.String("TCP"), tg.Protocol) require.Equal(t, cloudformation.String("HTTP"), tg.HealthCheckProtocol) + require.Empty(t, tg.ProtocolVersion) validateTargetGroupListener(t, template, "TG", "HTTPSListener", 443, "TLS") validateTargetGroupOutput(t, template, "TG", "TargetGroupARN") @@ -614,6 +619,38 @@ func TestGenerateTemplate(t *testing.T) { require.NotEqual(t, cloudformation.Integer(3), tg.UnhealthyThresholdCount) }, }, + { + name: "target port http2 when https listener and configured", + spec: &stackSpec{ + loadbalancerType: LoadBalancerTypeApplication, + http2: true, + targetHTTPS: true, + targetGroupProtocolVersion: "HTTP2", + }, + validate: func(t *testing.T, template *cloudformation.Template) { + require.NotNil(t, template.Resources["LB"]) + properties := template.Resources["LB"].Properties.(*cloudformation.ElasticLoadBalancingV2LoadBalancer) + attributes := []cloudformation.ElasticLoadBalancingV2LoadBalancerLoadBalancerAttribute(*properties.LoadBalancerAttributes) + require.Equal(t, attributes[1].Key.Literal, "routing.http2.enabled") + require.Equal(t, attributes[1].Value.Literal, "true") + tg := template.Resources["TG"].Properties.(*cloudformation.ElasticLoadBalancingV2TargetGroup) + require.Equal(t, cloudformation.String("HTTP2"), tg.ProtocolVersion) + }, + }, + { + name: "For Nlbs target protocol is undefined", + spec: &stackSpec{ + loadbalancerType: LoadBalancerTypeNetwork, + http2: true, + targetGroupProtocolVersion: "HTTP2", + targetHTTPS: true, + }, + validate: func(t *testing.T, template *cloudformation.Template) { + require.NotNil(t, template.Resources["LB"]) + tg := template.Resources["TG"].Properties.(*cloudformation.ElasticLoadBalancingV2TargetGroup) + require.Nil(t, tg.ProtocolVersion) + }, + }, } { t.Run(test.name, func(t *testing.T) { generated, err := generateTemplate(test.spec) diff --git a/aws/cloudwatch.go b/aws/cloudwatch.go index e38d4a8d..b0c4cb18 100644 --- a/aws/cloudwatch.go +++ b/aws/cloudwatch.go @@ -6,7 +6,7 @@ import ( "encoding/json" "github.com/ghodss/yaml" - cloudformation "github.com/mweagle/go-cloudformation" + cloudformation "github.com/o11n/go-cloudformation" log "github.com/sirupsen/logrus" ) diff --git a/aws/cloudwatch_test.go b/aws/cloudwatch_test.go index 33b0123b..02e1ec30 100644 --- a/aws/cloudwatch_test.go +++ b/aws/cloudwatch_test.go @@ -3,7 +3,7 @@ package aws import ( "testing" - cloudformation "github.com/mweagle/go-cloudformation" + cloudformation "github.com/o11n/go-cloudformation" "github.com/stretchr/testify/assert" ) diff --git a/controller.go b/controller.go index 1235125a..1a44ef97 100644 --- a/controller.go +++ b/controller.go @@ -87,6 +87,7 @@ var ( denyInternalRespContentType string denyInternalRespStatusCode int defaultInternalDomains = fmt.Sprintf("*%s", kubernetes.DefaultClusterLocalDomain) + targetGroupProtocolVersion string ) var metrics = struct { @@ -280,6 +281,8 @@ func loadSettings() error { Default("text/plain").StringVar(&denyInternalRespContentType) kingpin.Flag("deny-internal-domains-response-status-code", "Defines the response status code for a request identified as to an internal domain when -deny-internal-domains is set."). Default("401").IntVar(&denyInternalRespStatusCode) + kingpin.Flag("target-group-protocol-version", "Defines the target group protocol version for ALB. Specify GRPC to send requests to targets using gRPC. Specify HTTP2 to send requests to targets using HTTP/2. The default is HTTP1, which sends requests to targets using HTTP/1.1. Note the compatibility chart in https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-target-groups.html#target-group-protocol-version."). + Default("HTTP1").StringVar(&targetGroupProtocolVersion) kingpin.Parse() blacklistCertArnMap = make(map[string]bool) @@ -321,6 +324,10 @@ func loadSettings() error { return fmt.Errorf("invalid max number of certificates per ALB: %d. AWS does not allow more than %d", maxCertsPerALB, aws.DefaultMaxCertsPerALB) } + if !strings.Contains("HTTP1 HTTP2 GRPC", targetGroupProtocolVersion) { + return fmt.Errorf("invalid target group protocol version: %s. must be one of HTTP1, HTTP2, GRPC", targetGroupProtocolVersion) + } + if cwAlarmConfigMap != "" { loc, err := kubernetes.ParseResourceLocation(cwAlarmConfigMap) if err != nil { @@ -410,7 +417,8 @@ func main() { WithDenyInternalDomains(denyInternalDomains). WithInternalDomainsDenyResponse(denyInternalRespBody). WithInternalDomainsDenyResponseStatusCode(denyInternalRespStatusCode). - WithInternalDomainsDenyResponseContenType(denyInternalRespContentType) + WithInternalDomainsDenyResponseContenType(denyInternalRespContentType). + WithTargetGroupProtocolVersion(targetGroupProtocolVersion) log.Debug("certs.NewCachingProvider") certificatesProvider, err := certs.NewCachingProvider( diff --git a/go.mod b/go.mod index d023ad88..f4f0d8b8 100644 --- a/go.mod +++ b/go.mod @@ -7,17 +7,15 @@ require ( github.com/aws/aws-sdk-go v1.42.16 github.com/ghodss/yaml v1.0.0 github.com/go-playground/universal-translator v0.17.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 github.com/google/uuid v1.3.0 github.com/leodido/go-urn v1.2.1 // indirect github.com/linki/instrumented_http v0.3.0 - github.com/mweagle/go-cloudformation v0.0.0-20210117063902-00aa242fdc67 + github.com/o11n/go-cloudformation v0.0.0-20211209093817-f39f874a873a github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index dc9cee03..af96ccf2 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -83,10 +81,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mweagle/go-cloudformation v0.0.0-20210117063902-00aa242fdc67 h1:LX4BE6D2CnqgLjh05gAOlok9nEt78wvSF1Bj4pUOkYY= -github.com/mweagle/go-cloudformation v0.0.0-20210117063902-00aa242fdc67/go.mod h1:ZkuUgvDIuRW0sYTRfCz7VmL3IodhIufcb8HNdI6b6AI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/o11n/go-cloudformation v0.0.0-20211209093817-f39f874a873a h1:LtRfd3Ra8EHvTjFkRvIMU9yW1r7vxTT+Yq5pHgksNh4= +github.com/o11n/go-cloudformation v0.0.0-20211209093817-f39f874a873a/go.mod h1:XjqKQN76wMyOXjYp4joytbSjabK9+kzehqjzsGq51Oc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -170,9 +168,8 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -187,8 +184,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/worker_test.go b/worker_test.go index 4bd1ef49..702847f9 100644 --- a/worker_test.go +++ b/worker_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - cloudformation "github.com/mweagle/go-cloudformation" + cloudformation "github.com/o11n/go-cloudformation" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/zalando-incubator/kube-ingress-aws-controller/aws"