terratest roundrobin

Makefile

@@ -452,7 +452,7 @@ terratest: # Run terratest suite
 		echo -e "$(RED)Make sure you run the tests against at least two running clusters$(NC)" ;\
 		exit 1;\
 	fi
-	cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 15m -parallel=12 --tags=$(TEST_TAGS)
+	cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 30m -parallel=12 --tags=$(TEST_TAGS)
 
 .PHONY: website
 website:

controllers/refresolver/ingress.go

@@ -42,7 +42,7 @@ import (
 	"reflect"
 
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
-	"github.com/k8gb-io/k8gb/controllers/internal/utils"
+	"github.com/k8gb-io/k8gb/controllers/utils"
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

controllers/refresolver/ingress_test.go

@@ -42,7 +42,7 @@ import (
 	"testing"
 
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
-	"github.com/k8gb-io/k8gb/controllers/internal/utils"
+	"github.com/k8gb-io/k8gb/controllers/utils"
 	"github.com/stretchr/testify/assert"
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/runtime"

controllers/refresolver/refresolver.go

@@ -41,8 +41,8 @@ import (
 	"reflect"
 
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
-	"github.com/k8gb-io/k8gb/controllers/internal/utils"
 	"github.com/k8gb-io/k8gb/controllers/logging"
+	"github.com/k8gb-io/k8gb/controllers/utils"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )

terratest/examples/failover-playground-ref-gslb.yaml

@@ -0,0 +1,13 @@
+apiVersion: k8gb.absa.oss/v1beta1
+kind: Gslb
+metadata:
+  name: test-gslb
+spec:
+  resourceRef:
+    ingress:
+      matchLabels:
+        app: test-gslb
+  strategy:
+    type: failover
+    dnsTtlSeconds: 5
+    primaryGeoTag: "eu"

terratest/examples/failover-playground-ref-ingress.yaml

@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: test-gslb
+  labels:
+    app: test-gslb
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: playground-failover.cloud.example.com
+      http:
+        paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records
+              port:
+                name: http

terratest/examples/failover-ref-gslb.yaml

@@ -0,0 +1,13 @@
+apiVersion: k8gb.absa.oss/v1beta1
+kind: Gslb
+metadata:
+  name: test-gslb
+spec:
+  resourceRef:
+    ingress:
+      matchLabels:
+        app: test-gslb
+  strategy:
+    type: failover
+    dnsTtlSeconds: 5
+    primaryGeoTag: "eu"

terratest/examples/failover-ref-ingress.yaml

@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: test-gslb
+  labels:
+    app: test-gslb
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: terratest-failover.cloud.example.com
+      http:
+        paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records
+              port:
+                name: http

terratest/examples/roundrobin-weight1-ref-gslb.yaml

@@ -0,0 +1,15 @@
+apiVersion: k8gb.absa.oss/v1beta1
+kind: Gslb
+metadata:
+  name: test-gslb
+spec:
+  resourceRef:
+    ingress:
+      matchLabels:
+        app: test-gslb
+  strategy:
+    type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
+    dnsTtlSeconds: 5
+    weight:
+      eu: 5
+      us: 5

terratest/examples/roundrobin-weight1-ref-ingress.yaml

@@ -0,0 +1,39 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: test-gslb
+  labels:
+    app: test-gslb
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: terratest-notfound.cloud.example.com # This is the GSLB enabled host that clients would use
+      http: # This section mirrors the same structure as that of an Ingress resource and will be used verbatim when creating the corresponding Ingress resource that will match the GSLB host
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: non-existing-app # Gslb should reflect NotFound status
+                port:
+                  name: http
+    - host: terratest-unhealthy.cloud.example.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: unhealthy-app # Gslb should reflect Unhealthy status
+                port:
+                  name: http
+    - host: terratest-roundrobin.cloud.example.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records
+                port:
+                  name: http

terratest/examples/roundrobin2-ref-gslb.yaml

@@ -0,0 +1,12 @@
+apiVersion: k8gb.absa.oss/v1beta1
+kind: Gslb
+metadata:
+  name: roundrobin-test-gslb
+spec:
+  resourceRef:
+    ingress:
+      matchLabels:
+        app: roundrobin-test-gslb
+  strategy:
+    type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
+    dnsTtlSeconds: 5

terratest/examples/roundrobin2-ref-ingress.yaml

@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: roundrobin-test-gslb
+  labels:
+    app: roundrobin-test-gslb
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: roundrobin-test.cloud.example.com
+      http:
+        paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records
+              port:
+                name: http

terratest/test/k8gb_abstract_full_roundrobin_test.go

@@ -26,30 +26,35 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func abstractTestFullRoundRobin(t *testing.T, n int) {
+func abstractTestFullRoundRobin(t *testing.T, n int, host string, gslbPath string, ingressPath string) {
 	if n < 2 || n > 8 {
 		t.Logf("Use value of n that represents the number of clusters from interval [2,8]")
 		t.FailNow()
 	}
 	t.Logf(fmt.Sprintf("Running TestFullRoundRobin for %d clusters", n))
 	tags := []string{"eu", "us", "cz", "af", "ru", "ap", "uk", "ca"}
+	testPrefix := "embedded ingress"
+	if ingressPath != "" {
+		testPrefix = "referenced ingress"
+	}
 	var instances []*utils.Instance
 
-	const host = "roundrobin-test.cloud.example.com"
-	const gslbPath = "../examples/roundrobin2.yaml"
-
 	// start all the test apps on all the clusters
 	for i := 0; i < n; i += 1 {
-		instance, er := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i).
+		workflow := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i).
 			WithGslb(gslbPath, host).
-			WithTestApp(tags[i]).
-			Start()
+			WithTestApp(tags[i])
+		if ingressPath != "" {
+			workflow = workflow.WithIngress(ingressPath)
+		}
+
+		instance, er := workflow.Start()
 		require.NoError(t, er)
 		instances = append(instances, instance)
 		defer instance.Kill()
 	}
 	var err error
-	t.Run(fmt.Sprintf("round-robin on %d concurrent clusters with podinfo running", n), func(t *testing.T) {
+	t.Run(fmt.Sprintf("%s round-robin on %d concurrent clusters with podinfo running", testPrefix, n), func(t *testing.T) {
 		for _, ins := range instances {
 			err = ins.WaitForAppIsRunning()
 			require.NoError(t, err)
@@ -61,13 +66,13 @@ func abstractTestFullRoundRobin(t *testing.T, n int) {
 	for _, ins := range instances {
 		workingTargets = append(workingTargets, ins.GetLocalTargets()...)
 	}
-	t.Run(fmt.Sprintf("all %d clusters should be interconnected", n), func(t *testing.T) {
+	t.Run(fmt.Sprintf("%s all %d clusters should be interconnected", testPrefix, n), func(t *testing.T) {
 		allShouldExpectTheseTargets(t, instances, workingTargets)
 	})
 
 	// kill the apps on clusters one by one and expect less and less targets to be available
 	for i, instance := range instances {
-		t.Run(fmt.Sprintf("kill podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s kill podinfo on cluster %d (%s)", testPrefix, i+1, tags[i]), func(t *testing.T) {
 			workingTargets = distinct(subtract(workingTargets, instance.GetLocalTargets()))
 			t.Logf("New expected targets: %v", workingTargets)
 			instance.StopTestApp()
@@ -77,7 +82,7 @@ func abstractTestFullRoundRobin(t *testing.T, n int) {
 
 	// start the test apps again on each cluster and check if the targets start appearing
 	for i, instance := range instances {
-		t.Run(fmt.Sprintf("start podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s start podinfo on cluster %d (%s)", testPrefix, i+1, tags[i]), func(t *testing.T) {
 			instance.StartTestApp()
 			workingTargets = distinct(append(workingTargets, instance.GetLocalTargets()...))
 			t.Logf("New expected targets: %v", workingTargets)

terratest/test/k8gb_failover_playground_test.go

@@ -22,6 +22,7 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic
 */
 
 import (
+	"fmt"
 	"k8gbterratest/utils"
 	"testing"
 
@@ -33,55 +34,86 @@ import (
 // see: https://github.com/k8gb-io/k8gb/blob/master/docs/local.md#failover
 func TestFailoverPlayground(t *testing.T) {
 	const host = "playground-failover.cloud.example.com"
-	const gslbPath = "../examples/failover-playground.yaml"
+	abstractTestFailoverPlayground(t, host, "../examples/failover-playground.yaml", "")
+	abstractTestFailoverPlayground(t, host, "../examples/failover-playground-ref-gslb.yaml", "../examples/failover-playground-ref-ingress.yaml")
+}
+
+func abstractTestFailoverPlayground(t *testing.T, host string, gslbPath string, ingressPath string) {
+	testPrefix := "embedded ingress"
+	if ingressPath != "" {
+		testPrefix = "referenced ingress"
+	}
 	const euGeoTag = "eu"
 	const usGeoTag = "us"
 
-	instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
+	workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053).
 		WithGslb(gslbPath, host).
-		WithTestApp(euGeoTag).
-		Start()
+		WithTestApp(euGeoTag)
+	if ingressPath != "" {
+		workflowEU = workflowEU.WithIngress(ingressPath)
+	}
+	instanceEU, err := workflowEU.Start()
 	require.NoError(t, err)
 	defer instanceEU.Kill()
-	instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
+
+	workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054).
 		WithGslb(gslbPath, host).
-		WithTestApp(usGeoTag).
-		Start()
+		WithTestApp(usGeoTag)
+	if ingressPath != "" {
+		workflowUS = workflowUS.WithIngress(ingressPath)
+	}
+	instanceUS, err := workflowUS.Start()
 	require.NoError(t, err)
 	defer instanceUS.Kill()
 
+	fmt.Println("aba instances up")
+
 	actAndAssert := func(test, geoTag string, localTargets []string) {
 		// waiting for DNS sync
+		fmt.Println("aba act and assert 1")
 		err = instanceEU.WaitForExpected(localTargets)
 		require.NoError(t, err)
+		fmt.Println("aba act and assert 2")
 		err = instanceUS.WaitForExpected(localTargets)
 		require.NoError(t, err)
+		fmt.Println("aba act and assert 3")
 		// hit testApp from both clusters
 		httpResult := instanceEU.HitTestApp()
 		assert.Equal(t, geoTag, httpResult.Message)
+		fmt.Println("aba act and assert 4")
 		httpResult = instanceUS.HitTestApp()
 		assert.Equal(t, geoTag, httpResult.Message)
+		fmt.Println("aba act and assert 5")
 	}
 
-	t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) {
+	t.Run(fmt.Sprintf("%s failover on two concurrent clusters with TestApp running", testPrefix), func(t *testing.T) {
+		fmt.Println("aba app running 1")
 		err = instanceEU.WaitForAppIsRunning()
 		require.NoError(t, err)
+		fmt.Println("aba app running 2")
 		err = instanceUS.WaitForAppIsRunning()
 		require.NoError(t, err)
+		fmt.Println("aba app running 3")
 	})
 
 	euLocalTargets := instanceEU.GetLocalTargets()
 	usLocalTargets := instanceUS.GetLocalTargets()
 
-	t.Run("stop podinfo on eu cluster", func(t *testing.T) {
+	t.Run(fmt.Sprintf("%s stop podinfo on eu cluster", testPrefix), func(t *testing.T) {
+		fmt.Println("aba stop eu 1")
 		instanceEU.StopTestApp()
 		require.NoError(t, instanceEU.WaitForAppIsStopped())
+		fmt.Println("aba stop eu 2")
 		actAndAssert(t.Name(), usGeoTag, usLocalTargets)
+		fmt.Println("aba stop eu 3")
 	})
 
-	t.Run("start podinfo again on eu cluster", func(t *testing.T) {
+	t.Run(fmt.Sprintf("%s start podinfo again on eu cluster", testPrefix), func(t *testing.T) {
+		fmt.Println("aba start eu 1")
 		instanceEU.StartTestApp()
 		require.NoError(t, instanceEU.WaitForAppIsRunning())
+		fmt.Println("aba start eu 2")
 		actAndAssert(t.Name(), euGeoTag, euLocalTargets)
+		fmt.Println("aba start eu 3")
 	})
 }