diff --git a/tests/e2e/backup-restore_test.go b/tests/e2e/backup-restore_test.go index 68a1a997d..8257d1887 100644 --- a/tests/e2e/backup-restore_test.go +++ b/tests/e2e/backup-restore_test.go @@ -43,11 +43,8 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup- if backupRestoreVersion != "" { chartRepo = "https://github.com/rancher/backup-restore-operator/releases/download/" + backupRestoreVersion } else { - err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", chartRepo, "https://charts.rancher.io") - Expect(err).To(Not(HaveOccurred())) - - err = kubectl.RunHelmBinaryWithCustomErr("repo", "update") - Expect(err).To(Not(HaveOccurred())) + RunHelmCmdWithRetry("repo", "add", chartRepo, "https://charts.rancher.io") + RunHelmCmdWithRetry("repo", "update") } }) @@ -64,6 +61,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup- "upgrade", "--install", chart, chartRepo + "/" + chartName, "--namespace", "cattle-resources-system", "--create-namespace", + "--wait", "--wait-for-jobs", } // Add specific options for the rancher-backup chart @@ -75,8 +73,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup- } // Install through Helm - err := kubectl.RunHelmBinaryWithCustomErr(flags...) - Expect(err).To(Not(HaveOccurred())) + RunHelmCmdWithRetry(flags...) } }) diff --git a/tests/e2e/install_test.go b/tests/e2e/install_test.go index 261685533..19988a29d 100644 --- a/tests/e2e/install_test.go +++ b/tests/e2e/install_test.go @@ -162,11 +162,8 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { }) } else { By("Installing CertManager", func() { - err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", "jetstack", "https://charts.jetstack.io") - Expect(err).To(Not(HaveOccurred())) - - err = kubectl.RunHelmBinaryWithCustomErr("repo", "update") - Expect(err).To(Not(HaveOccurred())) + RunHelmCmdWithRetry("repo", "add", "jetstack", "https://charts.jetstack.io") + RunHelmCmdWithRetry("repo", "update") // Set flags for cert-manager installation flags := []string{ @@ -174,21 +171,21 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { "--namespace", "cert-manager", "--create-namespace", "--set", "installCRDs=true", + "--wait", "--wait-for-jobs", } if clusterType == "hardened" { flags = append(flags, "--version", CertManagerVersion) } - err = kubectl.RunHelmBinaryWithCustomErr(flags...) - Expect(err).To(Not(HaveOccurred())) + RunHelmCmdWithRetry(flags...) checkList := [][]string{ {"cert-manager", "app.kubernetes.io/component=controller"}, {"cert-manager", "app.kubernetes.io/component=webhook"}, {"cert-manager", "app.kubernetes.io/component=cainjector"}, } - err = rancher.CheckPod(k, checkList) + err := rancher.CheckPod(k, checkList) Expect(err).To(Not(HaveOccurred())) }) } @@ -306,13 +303,13 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { continue } } - Eventually(func() error { - return kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart, - operatorRepo+"/"+chart+"-chart", - "--namespace", "cattle-elemental-system", - "--create-namespace", - ) - }, tools.SetTimeout(1*time.Minute), 10*time.Second).Should(BeNil()) + + RunHelmCmdWithRetry("upgrade", "--install", chart, + operatorRepo+"/"+chart+"-chart", + "--namespace", "cattle-elemental-system", + "--create-namespace", + "--wait", "--wait-for-jobs", + ) } // Wait for pod to be started diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index bc3309d57..e252aa8e9 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -178,6 +178,12 @@ func GetNodeInfo(hostName string) (*tools.Client, string) { return c, hostData.Mac } +func RunHelmCmdWithRetry(s ...string) { + Eventually(func() error { + return kubectl.RunHelmBinaryWithCustomErr(s...) + }, tools.SetTimeout(2*time.Minute), 20*time.Second).Should(Not(HaveOccurred())) +} + func FailWithReport(message string, callerSkip ...int) { // Ensures the correct line numbers are reported Fail(message, callerSkip[0]+1) diff --git a/tests/e2e/uninstall-operator_test.go b/tests/e2e/uninstall-operator_test.go index a13ebf990..1e7e6b5a9 100644 --- a/tests/e2e/uninstall-operator_test.go +++ b/tests/e2e/uninstall-operator_test.go @@ -64,11 +64,10 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator continue } } - err := kubectl.RunHelmBinaryWithCustomErr( + RunHelmCmdWithRetry( "uninstall", chart, "--namespace", "cattle-elemental-system", ) - Expect(err).To(Not(HaveOccurred())) } }) @@ -113,12 +112,12 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator continue } } - err := kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart, + RunHelmCmdWithRetry("upgrade", "--install", chart, operatorRepo+"/"+chart+"-chart", "--namespace", "cattle-elemental-system", "--create-namespace", + "--wait", "--wait-for-jobs", ) - Expect(err).To(Not(HaveOccurred())) } // Wait for pod to be started diff --git a/tests/e2e/upgrade_test.go b/tests/e2e/upgrade_test.go index 995436d50..0e77d7871 100644 --- a/tests/e2e/upgrade_test.go +++ b/tests/e2e/upgrade_test.go @@ -54,12 +54,13 @@ var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator") } for _, chart := range upgradeOrder { - err := kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart, + RunHelmCmdWithRetry( + "upgrade", "--install", chart, operatorUpgrade+"/"+chart+"-chart", "--namespace", "cattle-elemental-system", "--create-namespace", + "--wait", "--wait-for-jobs", ) - Expect(err).To(Not(HaveOccurred())) } // Delay few seconds before checking, needed because we may have 2 pods at the same time