ci: workaround sporadic timeout during Rancher deployment

Signed-off-by: Loic Devulder <[email protected]>
rancher · Sep 27, 2024 · 1fce26c · 1fce26c
1 parent e0a3a68
commit 1fce26c
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 9 deletions.
diff --git a/tests/e2e/configure_test.go b/tests/e2e/configure_test.go
@@ -58,8 +58,9 @@ var _ = Describe("E2E - Configure test", Label("configure"), func() {
 			}
 
 			// Apply to k8s
-			err := kubectl.Apply(clusterNS, clusterYaml)
-			Expect(err).To(Not(HaveOccurred()))
+			Eventually(func() error {
+				return kubectl.Apply(clusterNS, clusterYaml)
+			}, tools.SetTimeout(1*time.Minute), 10*time.Second).Should(Not(HaveOccurred()))
 
 			// Check that the cluster is correctly created
 			CheckCreatedCluster(clusterNS, clusterName)

diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
@@ -107,6 +107,16 @@ var (
 	vmName                    string
 )
 
+func CheckDaemonSet(k *kubectl.Kubectl, checkList [][]string) error {
+	// Wait for a maximum of 2*120s (2 retries)
+	for _, check := range checkList {
+		if err := k.WaitForDaemonSet(check[0], check[1]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func CheckBackupRestore(v string) {
 	Eventually(func() string {
 		out, _ := kubectl.RunWithoutErr("logs", "-l app.kubernetes.io/name=rancher-backup",
@@ -452,8 +462,9 @@ Install Rancher Manager
   - @returns Nothing, the function will fail through Ginkgo in case of issue
 */
 func InstallRancher(k *kubectl.Kubectl) {
-	err := rancher.DeployRancherManager(rancherHostname, rancherChannel, rancherVersion, rancherHeadVersion, caType, proxy)
-	Expect(err).To(Not(HaveOccurred()))
+	Eventually(func() error {
+		return rancher.DeployRancherManager(rancherHostname, rancherChannel, rancherVersion, rancherHeadVersion, caType, proxy)
+	}, tools.SetTimeout(5*time.Minute), 1*time.Minute).Should(Not(HaveOccurred()))
 
 	checkList := [][]string{
 		{"cattle-system", "app=rancher"},
@@ -698,8 +709,18 @@ func WaitForRKE2(k *kubectl.Kubectl) {
 		return rancher.CheckPod(k, checkList)
 	}, tools.SetTimeout(4*time.Minute), 30*time.Second).Should(Not(HaveOccurred()))
 
-	err = k.WaitLabelFilter("kube-system", "Ready", "rke2-ingress-nginx-controller", "app.kubernetes.io/name=rke2-ingress-nginx")
-	Expect(err).To(Not(HaveOccurred()))
+	/*
+		err = k.WaitLabelFilter("kube-system", "Ready", "rke2-ingress-nginx-controller", "app.kubernetes.io/name=rke2-ingress-nginx")
+		Expect(err).To(Not(HaveOccurred()))
+	*/
+
+	checkList = [][]string{
+		{"kube-system", "rke2-canal"},
+		{"kube-system", "rke2-ingress-nginx-controller"},
+	}
+	Eventually(func() error {
+		return CheckDaemonSet(k, checkList)
+	}, tools.SetTimeout(4*time.Minute), 30*time.Second).Should(Not(HaveOccurred()))
 }
 
 /*

diff --git a/tests/go.mod b/tests/go.mod
@@ -6,10 +6,12 @@ toolchain go1.22.7
 
 replace go.qase.io/client => github.com/rancher/qase-go/client v0.0.0-20231114201952-65195ec001fa
 
+replace github.com/rancher-sandbox/ele-testhelpers => github.com/ldevulder/ele-testhelpers v0.0.0-20240927154419-af7b6a63a30f
+
 require (
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
-	github.com/rancher-sandbox/ele-testhelpers v0.0.0-20240926104948-8ac88aebed21
+	github.com/rancher-sandbox/ele-testhelpers v0.0.0-00010101000000-000000000000
 	github.com/rancher-sandbox/qase-ginkgo v1.0.1
 	github.com/sirupsen/logrus v1.9.3
 	golang.org/x/mod v0.21.0

diff --git a/tests/go.sum b/tests/go.sum
@@ -118,6 +118,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/ldevulder/ele-testhelpers v0.0.0-20240927154419-af7b6a63a30f h1:JTpK0Jq8nEQVvYFVOVeI/g7sJBp5masLuiHIJMPXUNc=
+github.com/ldevulder/ele-testhelpers v0.0.0-20240927154419-af7b6a63a30f/go.mod h1:Ex+a/ng4u2BvcGQdQjTHI48h88bQ6k2a7q8rnvU0XbQ=
 github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=
 github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
 github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
@@ -127,8 +129,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rancher-sandbox/ele-testhelpers v0.0.0-20240926104948-8ac88aebed21 h1:HiCx5MUE8Y3o9K+YV8fMPPbzqSm7r6klDQ8erJbIw9E=
-github.com/rancher-sandbox/ele-testhelpers v0.0.0-20240926104948-8ac88aebed21/go.mod h1:Ex+a/ng4u2BvcGQdQjTHI48h88bQ6k2a7q8rnvU0XbQ=
 github.com/rancher-sandbox/qase-ginkgo v1.0.1 h1:LB9ITLavX3PmcOe0hp0Y7rwQCjJ3WpL8kG8v1MxPadE=
 github.com/rancher-sandbox/qase-ginkgo v1.0.1/go.mod h1:sIF43xaLHtEzmPqADKlZZV6oatc66GHz1N6gpBNn6QY=
 github.com/rancher/qase-go/client v0.0.0-20231114201952-65195ec001fa h1:/qeYlQVfyvsO5yY0dZmm7mRTAsDm54jACiRDx3LAwsA=