From 1e1db901d9de74f1c79c243c282ecb0276e89c2b Mon Sep 17 00:00:00 2001 From: Andre Baptista Aguas Date: Sun, 25 Aug 2024 21:09:31 +0200 Subject: [PATCH] Add reverse proxy support Problem K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively. The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy. Solution To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on the GSLB resource. Example ``` apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: labels: app: ingress annotations: k8gb.io/external-ips: "185.199.110.153" ``` Fixes #1275 Signed-off-by: Andre Baptista Aguas --- .../testdata/invalid_omitempty_empty.yaml | 2 -- controllers/gslb_controller_reconciliation.go | 2 +- controllers/mocks/refresolver_mock.go | 8 +++---- controllers/refresolver/ingress/ingress.go | 15 ++++++++++-- .../refresolver/ingress/ingress_test.go | 24 ++++++++++++++++++- .../istiovirtualservice.go | 14 +++++++++-- .../istiovirtualservice_test.go | 24 ++++++++++++++++++- controllers/refresolver/refresolver.go | 2 +- 8 files changed, 77 insertions(+), 14 deletions(-) diff --git a/controllers/depresolver/testdata/invalid_omitempty_empty.yaml b/controllers/depresolver/testdata/invalid_omitempty_empty.yaml index 7ee8f9801b..981b89b7e3 100644 --- a/controllers/depresolver/testdata/invalid_omitempty_empty.yaml +++ b/controllers/depresolver/testdata/invalid_omitempty_empty.yaml @@ -37,5 +37,3 @@ spec: type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too splitBrainThresholdSeconds: weight: - - diff --git a/controllers/gslb_controller_reconciliation.go b/controllers/gslb_controller_reconciliation.go index c626d74e0b..4488e5b886 100644 --- a/controllers/gslb_controller_reconciliation.go +++ b/controllers/gslb_controller_reconciliation.go @@ -178,7 +178,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } gslb.Status.Servers = servers - loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers) + loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb.Annotations, r.Config.EdgeDNSServers) if err != nil { m.IncrementError(gslb) return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err)) diff --git a/controllers/mocks/refresolver_mock.go b/controllers/mocks/refresolver_mock.go index 2b5ce011f8..6694e0d3c3 100644 --- a/controllers/mocks/refresolver_mock.go +++ b/controllers/mocks/refresolver_mock.go @@ -59,18 +59,18 @@ func (m *MockGslbReferenceResolver) EXPECT() *MockGslbReferenceResolverMockRecor } // GetGslbExposedIPs mocks base method. -func (m *MockGslbReferenceResolver) GetGslbExposedIPs(arg0 utils.DNSList) ([]string, error) { +func (m *MockGslbReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetGslbExposedIPs", arg0) + ret := m.ctrl.Call(m, "GetGslbExposedIPs", gslbAnnotations, edgeDNSServers) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetGslbExposedIPs indicates an expected call of GetGslbExposedIPs. -func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(arg0 any) *gomock.Call { +func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(gslbAnnotations, edgeDNSServers any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), gslbAnnotations, edgeDNSServers) } // GetServers mocks base method. diff --git a/controllers/refresolver/ingress/ingress.go b/controllers/refresolver/ingress/ingress.go index 8c4a4c5034..93141adff2 100644 --- a/controllers/refresolver/ingress/ingress.go +++ b/controllers/refresolver/ingress/ingress.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "reflect" + "strings" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" "github.com/k8gb-io/k8gb/controllers/logging" @@ -35,6 +36,11 @@ import ( var log = logging.Logger() +const ( + // comma separated list of external IP addresses + externalIPsAnnotation = "k8gb.io/exposed-ip-addresses" +) + type ReferenceResolver struct { ingress *netv1.Ingress } @@ -158,9 +164,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { } // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB -func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) { - gslbIngressIPs := []string{} +func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) { + // fetch the IP addresses of the reverse proxy from an annotation if it exists + if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok { + return strings.Split(ingressIPsFromAnnotation, ","), nil + } + // if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource + gslbIngressIPs := []string{} for _, ip := range rr.ingress.Status.LoadBalancer.Ingress { if len(ip.IP) > 0 { gslbIngressIPs = append(gslbIngressIPs, ip.IP) diff --git a/controllers/refresolver/ingress/ingress_test.go b/controllers/refresolver/ingress/ingress_test.go index e288101e81..c6b3d07ce1 100644 --- a/controllers/refresolver/ingress/ingress_test.go +++ b/controllers/refresolver/ingress/ingress_test.go @@ -97,24 +97,46 @@ func TestGetServers(t *testing.T) { func TestGetGslbExposedIPs(t *testing.T) { var tests = []struct { name string + annotations map[string]string ingressYaml string expectedIPs []string }{ { name: "no exposed IPs", + annotations: map[string]string{}, ingressYaml: "./testdata/ingress_no_ips.yaml", expectedIPs: []string{}, }, { name: "single exposed IP", + annotations: map[string]string{}, ingressYaml: "../testdata/ingress_referenced.yaml", expectedIPs: []string{"10.0.0.1"}, }, { name: "multiple exposed IPs", + annotations: map[string]string{}, ingressYaml: "./testdata/ingress_multiple_ips.yaml", expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, }, + { + name: "annotation with no exposed IPs", + annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""}, + ingressYaml: "./testdata/ingress_multiple_ips.yaml", + expectedIPs: []string{""}, + }, + { + name: "annotation with single exposed IP", + annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"}, + ingressYaml: "./testdata/ingress_multiple_ips.yaml", + expectedIPs: []string{"185.199.110.153"}, + }, + { + name: "annotation with multiple exposed IPs", + annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"}, + ingressYaml: "./testdata/ingress_multiple_ips.yaml", + expectedIPs: []string{"185.199.110.153", "185.199.109.153"}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -125,7 +147,7 @@ func TestGetGslbExposedIPs(t *testing.T) { } // act - IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{}) + IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{}) assert.NoError(t, err) // assert diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go index 93f4050f68..6405d3ca5a 100644 --- a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go +++ b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go @@ -37,6 +37,11 @@ import ( var log = logging.Logger() +const ( + // comma separated list of external IP addresses + externalIPsAnnotation = "k8gb.io/exposed-ip-addresses" +) + type ReferenceResolver struct { virtualService *istio.VirtualService lbService *corev1.Service @@ -191,9 +196,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { } // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB -func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) { - gslbIngressIPs := []string{} +func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) { + // fetch the IP addresses of the reverse proxy from an annotation if it exists + if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok { + return strings.Split(ingressIPsFromAnnotation, ","), nil + } + // if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource + gslbIngressIPs := []string{} for _, ip := range rr.lbService.Status.LoadBalancer.Ingress { if len(ip.IP) > 0 { gslbIngressIPs = append(gslbIngressIPs, ip.IP) diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go index 8938e6dd10..8f0e78465a 100644 --- a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go +++ b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go @@ -112,24 +112,46 @@ func TestGetServers(t *testing.T) { func TestGetGslbExposedIPs(t *testing.T) { var tests = []struct { name string + annotations map[string]string serviceYaml string expectedIPs []string }{ { name: "no exposed IPs", serviceYaml: "./testdata/istio_service_no_ips.yaml", + annotations: map[string]string{}, expectedIPs: []string{}, }, { name: "single exposed IP", + annotations: map[string]string{}, serviceYaml: "../testdata/istio_service.yaml", expectedIPs: []string{"10.0.0.1"}, }, { name: "multiple exposed IPs", + annotations: map[string]string{}, serviceYaml: "./testdata/istio_service_multiple_ips.yaml", expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, }, + { + name: "annotation with no exposed IPs", + annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""}, + serviceYaml: "./testdata/istio_service_multiple_ips.yaml", + expectedIPs: []string{""}, + }, + { + name: "annotation with single exposed IP", + annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"}, + serviceYaml: "./testdata/istio_service_multiple_ips.yaml", + expectedIPs: []string{"185.199.110.153"}, + }, + { + name: "annotation with multiple exposed IPs", + annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"}, + serviceYaml: "./testdata/istio_service_multiple_ips.yaml", + expectedIPs: []string{"185.199.110.153", "185.199.109.153"}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { @@ -140,7 +162,7 @@ func TestGetGslbExposedIPs(t *testing.T) { } // act - IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{}) + IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{}) assert.NoError(t, err) // assert diff --git a/controllers/refresolver/refresolver.go b/controllers/refresolver/refresolver.go index 6b9067f1a2..460de7f815 100644 --- a/controllers/refresolver/refresolver.go +++ b/controllers/refresolver/refresolver.go @@ -34,7 +34,7 @@ type GslbReferenceResolver interface { // GetServers retrieves GSLB the server configuration GetServers() ([]*k8gbv1beta1.Server, error) // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB - GetGslbExposedIPs(utils.DNSList) ([]string, error) + GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) } // New creates a new GSLBReferenceResolver