Add reverse proxy support

Problem K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively. The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy. Solution To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on the GSLB resource. Example ``` apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: labels: app: ingress annotations: k8gb.io/external-ips: "185.199.110.153" ``` Fixes #1275 Signed-off-by: Andre Baptista Aguas <[email protected]>
k8gb-io · Nov 7, 2024 · 1e1db90 · 1e1db90
1 parent 31c0d09
commit 1e1db90
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 14 deletions.
diff --git a/controllers/depresolver/testdata/invalid_omitempty_empty.yaml b/controllers/depresolver/testdata/invalid_omitempty_empty.yaml
@@ -37,5 +37,3 @@ spec:
     type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
     splitBrainThresholdSeconds:
     weight:
-
-
diff --git a/controllers/gslb_controller_reconciliation.go b/controllers/gslb_controller_reconciliation.go
@@ -178,7 +178,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	}
 	gslb.Status.Servers = servers
 
-	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers)
+	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb.Annotations, r.Config.EdgeDNSServers)
 	if err != nil {
 		m.IncrementError(gslb)
 		return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err))

diff --git a/controllers/mocks/refresolver_mock.go b/controllers/mocks/refresolver_mock.go
diff --git a/controllers/refresolver/ingress/ingress.go b/controllers/refresolver/ingress/ingress.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
 	"github.com/k8gb-io/k8gb/controllers/logging"
@@ -35,6 +36,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	ingress *netv1.Ingress
 }
@@ -158,9 +164,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return strings.Split(ingressIPsFromAnnotation, ","), nil
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.ingress.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

diff --git a/controllers/refresolver/ingress/ingress_test.go b/controllers/refresolver/ingress/ingress_test.go
@@ -97,24 +97,46 @@ func TestGetServers(t *testing.T) {
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
 		name        string
+		annotations map[string]string
 		ingressYaml string
 		expectedIPs []string
 	}{
 		{
 			name:        "no exposed IPs",
+			annotations: map[string]string{},
 			ingressYaml: "./testdata/ingress_no_ips.yaml",
 			expectedIPs: []string{},
 		},
 		{
 			name:        "single exposed IP",
+			annotations: map[string]string{},
 			ingressYaml: "../testdata/ingress_referenced.yaml",
 			expectedIPs: []string{"10.0.0.1"},
 		},
 		{
 			name:        "multiple exposed IPs",
+			annotations: map[string]string{},
 			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
 			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
 		},
+		{
+			name:        "annotation with no exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs: []string{""},
+		},
+		{
+			name:        "annotation with single exposed IP",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153"},
+		},
+		{
+			name:        "annotation with multiple exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
+			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -125,7 +147,7 @@ func TestGetGslbExposedIPs(t *testing.T) {
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
+			IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{})
 			assert.NoError(t, err)
 
 			// assert

diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go
@@ -37,6 +37,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	virtualService *istio.VirtualService
 	lbService      *corev1.Service
@@ -191,9 +196,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return strings.Split(ingressIPsFromAnnotation, ","), nil
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.lbService.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go
@@ -112,24 +112,46 @@ func TestGetServers(t *testing.T) {
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
 		name        string
+		annotations map[string]string
 		serviceYaml string
 		expectedIPs []string
 	}{
 		{
 			name:        "no exposed IPs",
 			serviceYaml: "./testdata/istio_service_no_ips.yaml",
+			annotations: map[string]string{},
 			expectedIPs: []string{},
 		},
 		{
 			name:        "single exposed IP",
+			annotations: map[string]string{},
 			serviceYaml: "../testdata/istio_service.yaml",
 			expectedIPs: []string{"10.0.0.1"},
 		},
 		{
 			name:        "multiple exposed IPs",
+			annotations: map[string]string{},
 			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
 			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
 		},
+		{
+			name:        "annotation with no exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs: []string{""},
+		},
+		{
+			name:        "annotation with single exposed IP",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153"},
+		},
+		{
+			name:        "annotation with multiple exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
+			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -140,7 +162,7 @@ func TestGetGslbExposedIPs(t *testing.T) {
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
+			IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{})
 			assert.NoError(t, err)
 
 			// assert

diff --git a/controllers/refresolver/refresolver.go b/controllers/refresolver/refresolver.go
@@ -34,7 +34,7 @@ type GslbReferenceResolver interface {
 	// GetServers retrieves GSLB the server configuration
 	GetServers() ([]*k8gbv1beta1.Server, error)
 	// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-	GetGslbExposedIPs(utils.DNSList) ([]string, error)
+	GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error)
 }
 
 // New creates a new GSLBReferenceResolver