ci: add RunHelmCmdWithRetry function

Helpful to retry Helm command, should avoid sporadic deployment issues. Signed-off-by: Loic Devulder <[email protected]>
rancher · Sep 29, 2023 · a15eb17 · a15eb17
1 parent 4c6986e
commit a15eb17
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 28 deletions.
diff --git a/tests/e2e/backup-restore_test.go b/tests/e2e/backup-restore_test.go
@@ -43,11 +43,8 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 			if backupRestoreVersion != "" {
 				chartRepo = "https://github.com/rancher/backup-restore-operator/releases/download/" + backupRestoreVersion
 			} else {
-				err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", chartRepo, "https://charts.rancher.io")
-				Expect(err).To(Not(HaveOccurred()))
-
-				err = kubectl.RunHelmBinaryWithCustomErr("repo", "update")
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry("repo", "add", chartRepo, "https://charts.rancher.io")
+				RunHelmCmdWithRetry("repo", "update")
 			}
 		})
 
@@ -64,6 +61,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 					"upgrade", "--install", chart, chartRepo + "/" + chartName,
 					"--namespace", "cattle-resources-system",
 					"--create-namespace",
+					"--wait", "--wait-for-jobs",
 				}
 
 				// Add specific options for the rancher-backup chart
@@ -75,8 +73,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 				}
 
 				// Install through Helm
-				err := kubectl.RunHelmBinaryWithCustomErr(flags...)
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry(flags...)
 			}
 		})
 

diff --git a/tests/e2e/install_test.go b/tests/e2e/install_test.go
@@ -162,33 +162,30 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
 			})
 		} else {
 			By("Installing CertManager", func() {
-				err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", "jetstack", "https://charts.jetstack.io")
-				Expect(err).To(Not(HaveOccurred()))
-
-				err = kubectl.RunHelmBinaryWithCustomErr("repo", "update")
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry("repo", "add", "jetstack", "https://charts.jetstack.io")
+				RunHelmCmdWithRetry("repo", "update")
 
 				// Set flags for cert-manager installation
 				flags := []string{
 					"upgrade", "--install", "cert-manager", "jetstack/cert-manager",
 					"--namespace", "cert-manager",
 					"--create-namespace",
 					"--set", "installCRDs=true",
+					"--wait", "--wait-for-jobs",
 				}
 
 				if clusterType == "hardened" {
 					flags = append(flags, "--version", CertManagerVersion)
 				}
 
-				err = kubectl.RunHelmBinaryWithCustomErr(flags...)
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry(flags...)
 
 				checkList := [][]string{
 					{"cert-manager", "app.kubernetes.io/component=controller"},
 					{"cert-manager", "app.kubernetes.io/component=webhook"},
 					{"cert-manager", "app.kubernetes.io/component=cainjector"},
 				}
-				err = rancher.CheckPod(k, checkList)
+				err := rancher.CheckPod(k, checkList)
 				Expect(err).To(Not(HaveOccurred()))
 			})
 		}
@@ -306,13 +303,13 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
 						continue
 					}
 				}
-				Eventually(func() error {
-					return kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart,
-						operatorRepo+"/"+chart+"-chart",
-						"--namespace", "cattle-elemental-system",
-						"--create-namespace",
-					)
-				}, tools.SetTimeout(1*time.Minute), 10*time.Second).Should(BeNil())
+
+				RunHelmCmdWithRetry("upgrade", "--install", chart,
+					operatorRepo+"/"+chart+"-chart",
+					"--namespace", "cattle-elemental-system",
+					"--create-namespace",
+					"--wait", "--wait-for-jobs",
+				)
 			}
 
 			// Wait for pod to be started

diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
@@ -178,6 +178,12 @@ func GetNodeInfo(hostName string) (*tools.Client, string) {
 	return c, hostData.Mac
 }
 
+func RunHelmCmdWithRetry(s ...string) {
+	Eventually(func() error {
+		return kubectl.RunHelmBinaryWithCustomErr(s...)
+	}, tools.SetTimeout(2*time.Minute), 20*time.Second).Should(Not(HaveOccurred()))
+}
+
 func FailWithReport(message string, callerSkip ...int) {
 	// Ensures the correct line numbers are reported
 	Fail(message, callerSkip[0]+1)

diff --git a/tests/e2e/uninstall-operator_test.go b/tests/e2e/uninstall-operator_test.go
@@ -64,11 +64,10 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator
 						continue
 					}
 				}
-				err := kubectl.RunHelmBinaryWithCustomErr(
+				RunHelmCmdWithRetry(
 					"uninstall", chart,
 					"--namespace", "cattle-elemental-system",
 				)
-				Expect(err).To(Not(HaveOccurred()))
 			}
 		})
 
@@ -113,12 +112,12 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator
 						continue
 					}
 				}
-				err := kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart,
+				RunHelmCmdWithRetry("upgrade", "--install", chart,
 					operatorRepo+"/"+chart+"-chart",
 					"--namespace", "cattle-elemental-system",
 					"--create-namespace",
+					"--wait", "--wait-for-jobs",
 				)
-				Expect(err).To(Not(HaveOccurred()))
 			}
 
 			// Wait for pod to be started

diff --git a/tests/e2e/upgrade_test.go b/tests/e2e/upgrade_test.go
@@ -54,12 +54,13 @@ var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator")
 		}
 
 		for _, chart := range upgradeOrder {
-			err := kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart,
+			RunHelmCmdWithRetry(
+				"upgrade", "--install", chart,
 				operatorUpgrade+"/"+chart+"-chart",
 				"--namespace", "cattle-elemental-system",
 				"--create-namespace",
+				"--wait", "--wait-for-jobs",
 			)
-			Expect(err).To(Not(HaveOccurred()))
 		}
 
 		// Delay few seconds before checking, needed because we may have 2 pods at the same time