From b22faa7ee1438d3d7294d54184eece587bd13482 Mon Sep 17 00:00:00 2001 From: Loic Devulder Date: Thu, 20 Jul 2023 11:25:21 +0200 Subject: [PATCH] ci: use new helpers Signed-off-by: Loic Devulder --- tests/e2e/app_test.go | 9 +- tests/e2e/backup-restore_test.go | 13 +- tests/e2e/bootstrap_test.go | 58 +-- tests/e2e/configure_test.go | 18 +- tests/e2e/helpers/elemental/elemental.go | 122 +++++ tests/e2e/helpers/install/install.go | 86 ---- tests/e2e/helpers/misc/misc.go | 540 ----------------------- tests/e2e/helpers/network/network.go | 83 ++++ tests/e2e/install_test.go | 43 +- tests/e2e/logs_test.go | 4 +- tests/e2e/suite_test.go | 16 +- tests/e2e/ui_test.go | 14 +- tests/e2e/uninstall-operator_test.go | 12 +- tests/e2e/upgrade_test.go | 54 ++- 14 files changed, 337 insertions(+), 735 deletions(-) create mode 100644 tests/e2e/helpers/elemental/elemental.go delete mode 100644 tests/e2e/helpers/install/install.go delete mode 100644 tests/e2e/helpers/misc/misc.go create mode 100644 tests/e2e/helpers/network/network.go diff --git a/tests/e2e/app_test.go b/tests/e2e/app_test.go index 56462ca38..f96501748 100644 --- a/tests/e2e/app_test.go +++ b/tests/e2e/app_test.go @@ -23,12 +23,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher-sandbox/ele-testhelpers/rancher" + "github.com/rancher-sandbox/ele-testhelpers/tools" ) var _ = Describe("E2E - Install a simple application", Label("install-app"), func() { It("Install HelloWorld application", func() { - kubeConfig, err := misc.SetClientKubeConfig(clusterNS, clusterName) + kubeConfig, err := rancher.SetClientKubeConfig(clusterNS, clusterName) defer os.Remove(kubeConfig) Expect(err).To(Not(HaveOccurred())) @@ -44,7 +45,7 @@ var _ = Describe("E2E - Checking a simple application", Label("check-app"), func appName := "hello-world" // File where to host client cluster kubeconfig - kubeConfig, err := misc.SetClientKubeConfig(clusterNS, clusterName) + kubeConfig, err := rancher.SetClientKubeConfig(clusterNS, clusterName) defer os.Remove(kubeConfig) Expect(err).To(Not(HaveOccurred())) @@ -72,7 +73,7 @@ var _ = Describe("E2E - Checking a simple application", Label("check-app"), func Eventually(func() error { htmlPage, err = exec.Command("curl", "http://"+ip+":8080").CombinedOutput() return err - }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) // Check HTML page content Expect(string(htmlPage)).To(And( diff --git a/tests/e2e/backup-restore_test.go b/tests/e2e/backup-restore_test.go index ff482e1e2..68a1a997d 100644 --- a/tests/e2e/backup-restore_test.go +++ b/tests/e2e/backup-restore_test.go @@ -21,8 +21,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "github.com/rancher-sandbox/ele-testhelpers/rancher" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/misc" ) var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-restore"), func() { @@ -30,7 +30,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup- // Default timeout is too small, so New() cannot be used k := &kubectl.Kubectl{ Namespace: "", - PollTimeout: misc.SetTimeout(300 * time.Second), + PollTimeout: tools.SetTimeout(300 * time.Second), PollInterval: 500 * time.Millisecond, } @@ -82,7 +82,8 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup- By("Waiting for rancher-backup-operator pod", func() { // Wait for pod to be started - misc.CheckPod(k, [][]string{{"cattle-resources-system", "app.kubernetes.io/name=rancher-backup"}}) + err := rancher.CheckPod(k, [][]string{{"cattle-resources-system", "app.kubernetes.io/name=rancher-backup"}}) + Expect(err).To(Not(HaveOccurred())) }) }) }) @@ -110,7 +111,7 @@ var _ = Describe("E2E - Test Backup/Restore", Label("test-backup-restore"), func "--tail=-1", "--since=5m", "--namespace", "cattle-resources-system") return out - }, misc.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring("Done with backup")) + }, tools.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring("Done with backup")) }) }) @@ -151,7 +152,7 @@ var _ = Describe("E2E - Test Backup/Restore", Label("test-backup-restore"), func out, _ := kubectl.Run("get", "restore", restoreResourceName, "-o", "jsonpath={.metadata.name}") return out - }, misc.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring(restoreResourceName)) + }, tools.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring(restoreResourceName)) // Check operator logs Eventually(func() string { @@ -159,7 +160,7 @@ var _ = Describe("E2E - Test Backup/Restore", Label("test-backup-restore"), func "--tail=-1", "--since=5m", "--namespace", "cattle-resources-system") return out - }, misc.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring("Done restoring")) + }, tools.SetTimeout(5*time.Minute), 10*time.Second).Should(ContainSubstring("Done restoring")) }) By("Checking cluster state after restore", func() { diff --git a/tests/e2e/bootstrap_test.go b/tests/e2e/bootstrap_test.go index f8b99fcd1..9355f1cdd 100644 --- a/tests/e2e/bootstrap_test.go +++ b/tests/e2e/bootstrap_test.go @@ -26,8 +26,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "github.com/rancher-sandbox/ele-testhelpers/rancher" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher/elemental/tests/e2e/helpers/elemental" + "github.com/rancher/elemental/tests/e2e/helpers/network" ) func checkClusterAgent(client *tools.Client) { @@ -35,7 +37,7 @@ func checkClusterAgent(client *tools.Client) { Eventually(func() string { out, _ := client.RunSSH("kubectl get pod -n cattle-system -l app=cattle-cluster-agent") return out - }, misc.SetTimeout(3*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(ContainSubstring("Running")) + }, tools.SetTimeout(3*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(ContainSubstring("Running")) } func getClusterState(ns, cluster, condition string) string { @@ -108,12 +110,12 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { By("Setting emulated TPM to "+strconv.FormatBool(emulateTPM), func() { // Set temporary file - emulatedTmp, err := misc.CreateTemp("emulatedTPM") + emulatedTmp, err := tools.CreateTemp("emulatedTPM") Expect(err).To(Not(HaveOccurred())) defer os.Remove(emulatedTmp) // Save original file as it can be modified multiple time - misc.CopyFile(emulateTPMYaml, emulatedTmp) + tools.CopyFile(emulateTPMYaml, emulatedTmp) // Patch the yaml file err = tools.Sed("%EMULATE_TPM%", strconv.FormatBool(emulateTPM), emulatedTmp) @@ -141,7 +143,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { if isoBoot == "true" { By("Adding SeedImage", func() { // Set temporary file - seedimageTmp, err := misc.CreateTemp("seedimage") + seedimageTmp, err := tools.CreateTemp("seedimage") Expect(err).To(Not(HaveOccurred())) defer os.Remove(seedimageTmp) @@ -154,7 +156,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { } // Save original file as it will have to be modified twice - misc.CopyFile(seedimageYaml, seedimageTmp) + tools.CopyFile(seedimageYaml, seedimageTmp) // Create Yaml file for _, p := range patterns { @@ -173,7 +175,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { seedImageName, "-o", "jsonpath={.status}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("downloadURL")) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("downloadURL")) }) By("Downloading ISO built by SeedImage", func() { @@ -190,7 +192,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { if isoBoot != "true" { By("Configuring iPXE boot script for network installation", func() { - numberOfFile, err := misc.ConfigureiPXE() + numberOfFile, err := network.ConfigureiPXE(httpSrv) Expect(err).To(Not(HaveOccurred())) Expect(numberOfFile).To(BeNumerically(">=", 1)) }) @@ -201,11 +203,11 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { bootstrappedNodes = 0 for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Add node in network configuration - err := misc.AddNode(netDefaultFileName, hostName, index) + err := rancher.AddNode(netDefaultFileName, hostName, index) Expect(err).To(Not(HaveOccurred())) // Get generated MAC address @@ -238,7 +240,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { // Only for master pool if poolType == "master" && isoBoot == "true" { for index := vmIndex; index <= numberOfVMs; index++ { - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) client, _ := GetNodeInfo(hostName) @@ -256,7 +258,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { out, _ := cl.RunSSH("echo SSH_OK") out = strings.Trim(out, "\n") return out - }, misc.SetTimeout(10*time.Minute), 5*time.Second).Should(Equal("SSH_OK")) + }, tools.SetTimeout(10*time.Minute), 5*time.Second).Should(Equal("SSH_OK")) // Check that the cloud-config is correctly applied by checking the presence of a file _, err := cl.RunSSH("ls /etc/elemental-test") @@ -266,7 +268,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { Eventually(func() error { _, err := cl.RunSSH("journalctl -u elemental-register.service --no-pager | grep -i 'elemental installation completed'") return err - }, misc.SetTimeout(8*time.Minute), 10*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(8*time.Minute), 10*time.Second).Should(Not(HaveOccurred())) // Halt the VM _, err = cl.RunSSH("setsid -f init 0") @@ -281,7 +283,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { It("Add the nodes in Rancher Manager", func() { for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Execute node deployment in parallel @@ -292,9 +294,9 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { By("Checking that node "+h+" is available in Rancher", func() { Eventually(func() string { - id, _ := misc.GetServerId(c, i) + id, _ := elemental.GetServerId(c, i) return id - }, misc.SetTimeout(1*time.Minute), 5*time.Second).Should(Not(BeEmpty())) + }, tools.SetTimeout(1*time.Minute), 5*time.Second).Should(Not(BeEmpty())) }) }(clusterNS, hostName, index) } @@ -310,7 +312,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { By("Incrementing number of nodes in "+poolType+" pool", func() { // Increase 'quantity' field - value, err := misc.IncreaseQuantity(clusterNS, + value, err := rancher.SetNodeQuantity(clusterNS, clusterName, "pool-"+poolType+"-"+clusterName, usedNodes) Expect(err).To(Not(HaveOccurred())) @@ -322,7 +324,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { "--namespace", clusterNS, "-o", "jsonpath={.items[*].metadata.name}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("selector-" + poolType + "-" + clusterName)) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("selector-" + poolType + "-" + clusterName)) }) By("Waiting for known cluster state before adding the node(s)", func() { @@ -337,13 +339,13 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { } return clusterMsg - }, misc.SetTimeout(5*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(MatchRegexp(msg)) + }, tools.SetTimeout(5*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(MatchRegexp(msg)) }) bootstrappedNodes = 0 for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Get node information @@ -371,7 +373,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { out, _ := cl.RunSSH("echo SSH_OK") out = strings.Trim(out, "\n") return out - }, misc.SetTimeout(10*time.Minute), 5*time.Second).Should(Equal("SSH_OK")) + }, tools.SetTimeout(10*time.Minute), 5*time.Second).Should(Equal("SSH_OK")) }) By("Checking that TPM is correctly configured on "+h, func() { @@ -382,7 +384,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { Eventually(func() error { _, err := cl.RunSSH("[[ " + testValue + " /dev/tpm0 ]]") return err - }, misc.SetTimeout(1*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(1*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) }) By("Checking OS version on "+h, func() { @@ -402,7 +404,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { if poolType != "worker" { for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Get node information @@ -425,7 +427,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { Eventually(func() error { _, err := cl.RunSSH("[[ -d " + dir + " ]]") return err - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) // Configure kubectl _, err := cl.RunSSH("I=" + dir + "/kubectl; if [[ -x ${I} ]]; then ln -s ${I} bin/; echo " + kubeCfg + " >> .bashrc; fi") @@ -438,7 +440,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { Eventually(func() string { out, _ := cl.RunSSH("kubectl version 2>/dev/null | grep 'Server Version:'") return out - }, misc.SetTimeout(5*time.Minute), 5*time.Second).Should(ContainSubstring(k8sVersion)) + }, tools.SetTimeout(5*time.Minute), 5*time.Second).Should(ContainSubstring(k8sVersion)) }) By("Checking cluster agent on "+h, func() { @@ -458,7 +460,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { if poolType != "worker" { for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Get node information @@ -479,7 +481,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { GinkgoWriter.Printf("K8s version on %s:\n%s\n", h, k8sVer) } return err - }, misc.SetTimeout(1*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(1*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) }) }(hostName, client) } @@ -491,7 +493,7 @@ var _ = Describe("E2E - Bootstrapping node", Label("bootstrap"), func() { bootstrappedNodes = 0 for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Get node information diff --git a/tests/e2e/configure_test.go b/tests/e2e/configure_test.go index 2407318bd..0b4f8d26d 100644 --- a/tests/e2e/configure_test.go +++ b/tests/e2e/configure_test.go @@ -24,7 +24,7 @@ import ( . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher/elemental/tests/e2e/helpers/elemental" ) var _ = Describe("E2E - Configure test", Label("configure"), func() { @@ -63,17 +63,17 @@ var _ = Describe("E2E - Configure test", Label("configure"), func() { "--namespace", clusterNS, clusterName, "-o", "jsonpath={.metadata.name}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(clusterName)) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(clusterName)) }) By("Creating cluster selectors", func() { // Set temporary file - selectorTmp, err := misc.CreateTemp("selector") + selectorTmp, err := tools.CreateTemp("selector") Expect(err).To(Not(HaveOccurred())) defer os.Remove(selectorTmp) // Get elemental-operator version - operatorVersion, err := misc.GetOperatorVersion() + operatorVersion, err := elemental.GetOperatorVersion() Expect(err).To(Not(HaveOccurred())) operatorVersionShort := strings.Split(operatorVersion, ".") @@ -88,7 +88,7 @@ var _ = Describe("E2E - Configure test", Label("configure"), func() { patterns := append(basePatterns, addPatterns...) // Save original file as it will have to be modified twice - misc.CopyFile(selectorYaml, selectorTmp) + tools.CopyFile(selectorYaml, selectorTmp) // Create Yaml file for _, p := range patterns { @@ -112,13 +112,13 @@ var _ = Describe("E2E - Configure test", Label("configure"), func() { "--namespace", clusterNS, "-o", "jsonpath={.items[*].metadata.name}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("selector-" + pool + "-" + clusterName)) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("selector-" + pool + "-" + clusterName)) } }) By("Adding MachineRegistration", func() { // Set temporary file - registrationTmp, err := misc.CreateTemp("machineRegistration") + registrationTmp, err := tools.CreateTemp("machineRegistration") Expect(err).To(Not(HaveOccurred())) defer os.Remove(registrationTmp) @@ -145,7 +145,7 @@ var _ = Describe("E2E - Configure test", Label("configure"), func() { patterns := append(basePatterns, addPatterns...) // Save original file as it will have to be modified twice - misc.CopyFile(registrationYaml, registrationTmp) + tools.CopyFile(registrationYaml, registrationTmp) // Create Yaml file for _, p := range patterns { @@ -163,7 +163,7 @@ var _ = Describe("E2E - Configure test", Label("configure"), func() { "--namespace", clusterNS, "-o", "jsonpath={.items[*].metadata.name}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("machine-registration-" + pool + "-" + clusterName)) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("machine-registration-" + pool + "-" + clusterName)) } }) diff --git a/tests/e2e/helpers/elemental/elemental.go b/tests/e2e/helpers/elemental/elemental.go new file mode 100644 index 000000000..b82223e35 --- /dev/null +++ b/tests/e2e/helpers/elemental/elemental.go @@ -0,0 +1,122 @@ +/* +Copyright © 2022 - 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package elemental + +import ( + "fmt" + "strings" + + "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "gopkg.in/yaml.v3" +) + +/** + * Get MachineInventory name (aka. server id) + * @remarks The repository is added to the local cluster + * @param clusterNS Name of the repository + * @param index URL of the repository + * @returns The name/id of the server or an error + */ +func GetServerId(clusterNS string, index int) (string, error) { + serverId, err := kubectl.Run("get", "MachineInventories", + "--namespace", clusterNS, + "-o", "jsonpath={.items["+fmt.Sprint(index-1)+"].metadata.name}") + if err != nil { + return "", err + } + + return serverId, nil +} + +/** + * Get container image used for Elemental operator + * @remarks Image used is returned + * @returns The container image used or an error + */ +func GetOperatorImage() (string, error) { + operatorImage, err := kubectl.Run("get", "pod", + "--namespace", "cattle-elemental-system", + "-l", "app=elemental-operator", "-o", "jsonpath={.items[*].status.containerStatuses[*].image}") + if err != nil { + return "", err + } + + return operatorImage, nil +} + +/** + * Get Elemental operator version + * @remarks Version is returned + * @returns the Elemental operator version or an error + */ +func GetOperatorVersion() (string, error) { + operatorImage, err := GetOperatorImage() + if err != nil { + return "", err + } + + // Extract version + operatorVersion := strings.Split(operatorImage, ":") + + return operatorVersion[1], nil +} + +/** + * Add node selector + * @remarks A nodeSelector field is added + * @param key key to add in YAML + * @param value value to set the key to + * @returns The YAML structure or an error + */ +func AddSelector(key, value string) ([]byte, error) { + type selectorYaml struct { + MatchLabels map[string]string `yaml:"matchLabels,omitempty"` + } + + type selector struct { + SelectorYaml selectorYaml `yaml:"nodeSelector,omitempty"` + } + + v := selectorYaml{map[string]string{key: value}} + s := selector{v} + out, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + + // Add indent at the beginning + out = append([]byte(" "), out...) + + return out, nil +} + +/** + * Set hostname of the node + * @remarks Define the hostname base on baseName and node index + * @param baseName Basename to use, "empty" if nothing provided + * @param index index of the node + * @returns Full hostname of the node + */ +func SetHostname(baseName string, index int) string { + if baseName == "" { + baseName = "emtpy" + } + + if index < 0 { + index = 0 + } + + return baseName + "-" + fmt.Sprintf("%03d", index) +} diff --git a/tests/e2e/helpers/install/install.go b/tests/e2e/helpers/install/install.go deleted file mode 100644 index c16e5a5c1..000000000 --- a/tests/e2e/helpers/install/install.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright © 2023 SUSE LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package install - -import ( - "strings" - - . "github.com/onsi/gomega" - "github.com/rancher-sandbox/ele-testhelpers/kubectl" -) - -// Install or upgrade Rancher Manager -func DeployRancherManager(hostname, channel, version, ca, proxy string) { - channelName := "rancher-" + channel - - // Add Helm repository - err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", channelName, - "https://releases.rancher.com/server-charts/"+channel, - ) - Expect(err).To(Not(HaveOccurred())) - - err = kubectl.RunHelmBinaryWithCustomErr("repo", "update") - Expect(err).To(Not(HaveOccurred())) - - // Set flags for Rancher Manager installation - flags := []string{ - "upgrade", "--install", "rancher", channelName + "/rancher", - "--namespace", "cattle-system", - "--create-namespace", - "--set", "hostname=" + hostname, - "--set", "extraEnv[0].name=CATTLE_SERVER_URL", - "--set", "extraEnv[0].value=https://" + hostname, - "--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD", - "--set", "extraEnv[1].value=rancherpassword", - "--set", "replicas=1", - "--set", "global.cattle.psp.enabled=false", - } - - // Set specified version if needed - if version != "" && version != "latest" { - if version == "devel" { - flags = append(flags, - "--devel", - "--set", "rancherImageTag=v2.7-head", - ) - } else if strings.Contains(version, "-rc") { - flags = append(flags, - "--devel", - "--version", version, - ) - } else { - flags = append(flags, "--version", version) - } - } - - // For Private CA - if ca == "private" { - flags = append(flags, - "--set", "ingress.tls.source=secret", - "--set", "privateCA=true", - ) - } - - // Use Rancher Manager behind proxy - if proxy == "rancher" { - flags = append(flags, - "--set", "proxy=http://172.17.0.1:3128", - "--set", "noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local", - ) - } - - err = kubectl.RunHelmBinaryWithCustomErr(flags...) - Expect(err).To(Not(HaveOccurred())) -} diff --git a/tests/e2e/helpers/misc/misc.go b/tests/e2e/helpers/misc/misc.go deleted file mode 100644 index 3bc875847..000000000 --- a/tests/e2e/helpers/misc/misc.go +++ /dev/null @@ -1,540 +0,0 @@ -/* -Copyright © 2022 - 2023 SUSE LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package misc - -import ( - "encoding/base64" - "errors" - "fmt" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "reflect" - "strconv" - "strings" - "time" - - "github.com/rancher-sandbox/ele-testhelpers/kubectl" - "github.com/rancher-sandbox/ele-testhelpers/tools" - "gopkg.in/yaml.v3" - libvirtxml "libvirt.org/libvirt-go-xml" -) - -// Cluster is the definition of a K8s cluster -type Cluster struct { - APIVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind,omitempty"` - Metadata Metadata `yaml:"metadata"` - Spec ClusterSpec `yaml:"spec"` - Status ClusterStatus `yaml:"status,omitempty"` -} - -// Metadata is metadata attached to any object -type Metadata struct { - Annotations interface{} `yaml:"annotations"` - Labels interface{} `yaml:"labels,omitempty"` - Finalizers interface{} `yaml:"finalizers,omitempty"` - ManagedFields interface{} `yaml:"managedFields,omitempty"` - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - ResourceVersion string `yaml:"resourceVersion"` - UID string `yaml:"uid"` -} - -// ClusterSpec is a description of a cluster -type ClusterSpec struct { - KubernetesVersion string `yaml:"kubernetesVersion"` - LocalClusterAuthEndpoint interface{} `yaml:"localClusterAuthEndpoint"` - RkeConfig RKEConfig `yaml:"rkeConfig"` -} - -// RKEConfig has all RKE/K3s cluster information -type RKEConfig struct { - Etcd interface{} `yaml:"etcd,omitempty"` - ChartValues map[string]interface{} `yaml:"chartValues,omitempty" wrangler:"nullable"` - MachineGlobalConfig interface{} `yaml:"machineGlobalConfig"` - MachinePools []MachinePools `yaml:"machinePools"` - MachineSelectorConfig interface{} `yaml:"machineSelectorConfig"` - UpgradeStrategy interface{} `yaml:"upgradeStrategy,omitempty"` - Registries interface{} `yaml:"registries"` -} - -// MachinePools has all pools information -type MachinePools struct { - ControlPlaneRole bool `yaml:"controlPlaneRole,omitempty"` - EtcdRole bool `yaml:"etcdRole,omitempty"` - MachineConfigRef MachineConfigRef `yaml:"machineConfigRef"` - Name string `yaml:"name"` - Quantity int `yaml:"quantity"` - UnhealthyNodeTimeout string `yaml:"unhealthyNodeTimeout"` - WorkerRole bool `yaml:"workerRole,omitempty"` -} - -// MachineConfigRef makes the link between the cluster, pool and the Elemental nodes -type MachineConfigRef struct { - APIVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Name string `yaml:"name"` -} - -// ClusterStatus has all the cluster status information -type ClusterStatus struct { - AgentDeployed bool `yaml:"agentDeployed,omitempty"` - ClientSecretName string `yaml:"clientSecretName"` - ClusterName string `yaml:"clusterName"` - Conditions []ClusterCondition `yaml:"conditions,omitempty"` - Ready bool `yaml:"ready,omitempty"` -} - -// ClusterCondition is the cluster condition status -type ClusterCondition struct { - LastUpdateTime string `yaml:"lastUpdateTime"` - Message string `yaml:"message,omitempty"` - Reason string `yaml:"reason,omitempty"` - Status string `yaml:"status"` - Type string `yaml:"type"` -} - -const ( - httpSrv = "http://192.168.122.1:8000" -) - -func (c *Cluster) getCluster(ns, name string) error { - out, err := kubectl.Run("get", "cluster", - "--namespace", ns, name, - "-o", "yaml") - if err != nil { - return err - } - - // Decode content - if err := yaml.Unmarshal([]byte(out), c); err != nil { - return err - } - - return nil -} - -func (c *Cluster) setCluster(ns, name string) error { - // Encode content - out, err := yaml.Marshal(&c) - if err != nil { - return err - } - - // Use temporary file - f, err := os.CreateTemp("", "updatedCluster") - if err != nil { - return err - } - defer os.Remove(f.Name()) - - if _, err := f.Write(out); err != nil { - return err - } - - if err := f.Close(); err != nil { - return err - } - - // Apply new cluster configuration - if err := kubectl.Apply(ns, f.Name()); err != nil { - return err - } - - return nil -} - -func GetServerId(clusterNS string, index int) (string, error) { - serverId, err := kubectl.Run("get", "MachineInventories", - "--namespace", clusterNS, - "-o", "jsonpath={.items["+fmt.Sprint(index-1)+"].metadata.name}") - if err != nil { - return "", err - } - - return serverId, nil -} - -func GetOperatorImage() (string, error) { - operatorImage, err := kubectl.Run("get", "pod", - "--namespace", "cattle-elemental-system", - "-l", "app=elemental-operator", "-o", "jsonpath={.items[*].status.containerStatuses[*].image}") - if err != nil { - return "", err - } - - return operatorImage, nil -} - -func GetOperatorVersion() (string, error) { - operatorImage, err := GetOperatorImage() - if err != nil { - return "", err - } - - // Extract version - operatorVersion := strings.Split(operatorImage, ":") - - return operatorVersion[1], nil -} - -func ConfigureiPXE() (int, error) { - ipxeScript, err := tools.GetFiles("../..", "*.ipxe") - if err != nil { - return 0, err - } - - // NOTE: always use the first ipxe file found! - if len(ipxeScript) >= 1 { - err = tools.Sed("set url.*", "set url "+httpSrv, ipxeScript[0]) - if err != nil { - return 0, err - } - - err = tools.Sed(".*set config.*", "set config $${url}/install-config.yaml", ipxeScript[0]) - if err != nil { - return 0, err - } - - // Delete the previous symlink, use RemoveAll to avoid error if the file doesn't exist - symLink := "../../install-elemental.ipxe" - err = os.RemoveAll(symLink) - if err != nil { - return 0, err - } - - // Create a symlink to the needed ipxe file - scriptName := filepath.Base(ipxeScript[0]) - err = os.Symlink(scriptName, symLink) - if err != nil { - return 0, err - } - } - - // Returns the number of ipxe files found - return len(ipxeScript), nil -} - -// Don't return error, in the worst case return the initial value -// Otherwise an additional step will be needed for some commands (like Eventually) -func SetTimeout(timeout time.Duration) time.Duration { - s, set := os.LookupEnv("TIMEOUT_SCALE") - - // Only if TIMEOUT_SCALE is set - if set { - scale, err := strconv.Atoi(s) - if err != nil { - return timeout - } - - // Return the scaled timeout - return timeout * time.Duration(scale) - } - - // Nothing to do - return timeout -} - -func IncreaseQuantity(ns, name, pool string, quantity int) (int, error) { - c := &Cluster{} - quantitySet := 0 - poolFound := false - - // Get cluster configuration - if err := c.getCluster(ns, name); err != nil { - return 0, err - } - - // Try to increase quantity field - for i := range c.Spec.RkeConfig.MachinePools { - // Only on selected pool - if c.Spec.RkeConfig.MachinePools[i].Name == pool { - // Pool found! - poolFound = true - - // Increase quantity - c.Spec.RkeConfig.MachinePools[i].Quantity += quantity - quantitySet = c.Spec.RkeConfig.MachinePools[i].Quantity - - // Quantity increased, loop can be stopped - break - } - } - - // Throw an error if the pool has not been found - if !poolFound { - return 0, errors.New("pool '" + pool + "' does not exist!") - } - - // Save and apply cluster configuration - if err := c.setCluster(ns, name); err != nil { - return 0, err - } - - return quantitySet, nil -} - -// How to use it, for example: -// err := misc.ToggleRole(clusterNS, clusterName, "pool-worker-"+clusterName, "ControlPlaneRole", true) -func ToggleRole(ns, name, pool, role string, value bool) error { - c := &Cluster{} - poolFound := false - - // Get cluster configuration - if err := c.getCluster(ns, name); err != nil { - return err - } - - // Try to set value to role - for i := range c.Spec.RkeConfig.MachinePools { - // Only on selected pool - if c.Spec.RkeConfig.MachinePools[i].Name == pool { - // Pool found! - poolFound = true - - // Get fields list and check that the role exist - v := reflect.ValueOf(&c.Spec.RkeConfig.MachinePools[i]).Elem() - f := v.FieldByName(role) - if f == (reflect.Value{}) { - // No, return an error - return errors.New("role '" + role + "' does not exist!") - } else { - // Yes, set the value accordingly - v.FieldByName(role).SetBool(value) - - // Role toggled, loop can be stopped - break - } - } - } - - // Throw an error if the pool has not been found - if !poolFound { - return errors.New("pool '" + pool + "' does not exist!") - } - - // Save and apply cluster configuration - if err := c.setCluster(ns, name); err != nil { - return err - } - - return nil -} - -func AddSelector(key, value string) ([]byte, error) { - type selectorYaml struct { - MatchLabels map[string]string `yaml:"matchLabels,omitempty"` - } - - type selector struct { - SelectorYaml selectorYaml `yaml:"nodeSelector,omitempty"` - } - - v := selectorYaml{map[string]string{key: value}} - s := selector{v} - out, err := yaml.Marshal(s) - if err != nil { - return nil, err - } - - // Add indent at the beginning - out = append([]byte(" "), out...) - - return out, nil -} - -func ConcateFiles(srcfile, dstfile string, data []byte) error { - // Open source file - f, err := os.Open(srcfile) - if err != nil { - return err - } - defer f.Close() - - // Open/create destination file - d, err := os.OpenFile(dstfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer d.Close() - - // Copy source to dest - if _, err = io.Copy(d, f); err != nil { - return err - } - - // Add data to dest - if _, err = d.Write([]byte(data)); err != nil { - return err - } - - return nil -} - -func WriteFile(dstfile string, data []byte) error { - // Open/create destination file - d, err := os.OpenFile(dstfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer d.Close() - - // Add data to dest - if _, err = d.Write([]byte(data)); err != nil { - return err - } - - return nil -} - -func CopyFile(srcFile, dstFile string) error { - // Concate files without adding data is in fact a copy - return (ConcateFiles(srcFile, dstFile, []byte(""))) -} - -func TrimStringFromChar(s, c string) string { - if idx := strings.Index(s, c); idx != -1 { - return s[:idx] - } - - return s -} - -func AddNode(file, name string, index int) error { - // Read live XML configuration - fileContent, err := exec.Command("sudo", "virsh", "net-dumpxml", "default").Output() - if err != nil { - return err - } - - // Unmarshal fileContent - netcfg := &libvirtxml.Network{} - if err := netcfg.Unmarshal(string(fileContent)); err != nil { - return err - } - - // Add new host - // NOTE: we only use one network (IPs[0]) - host := libvirtxml.NetworkDHCPHost{ - Name: name, - MAC: "52:54:00:00:00:" + fmt.Sprintf("%02x", index), - IP: "192.168.122." + strconv.Itoa(index+1), - } - netcfg.IPs[0].DHCP.Hosts = append(netcfg.IPs[0].DHCP.Hosts, host) - - // Marshal new content - newFileContent, err := netcfg.Marshal() - if err != nil { - return err - } - - // Re-write XML file - if err := os.WriteFile(file, []byte(newFileContent), 0644); err != nil { - return err - } - - // Update live network configuration - xmlValue, err := host.Marshal() - if err != nil { - return err - } - if err := exec.Command("sudo", "virsh", "net-update", - "default", "add", "ip-dhcp-host", "--live", "--xml", xmlValue).Run(); err != nil { - return err - } - - return nil -} - -func FileShare(directory, listenAddr string) { - fs := http.FileServer(http.Dir(directory)) - - go func() { - if err := http.ListenAndServe(listenAddr, fs); err != nil { - fmt.Printf("Server failed: %s\n", err) - } - }() -} - -func SetHostname(baseName string, index int) string { - if baseName == "" { - baseName = "emtpy" - } - - if index < 0 { - index = 0 - } - - return baseName + "-" + fmt.Sprintf("%03d", index) -} - -func CreateTemp(baseName string) (string, error) { - t, err := os.CreateTemp("", baseName) - if err != nil { - return "", err - } - - return t.Name(), nil -} - -func CheckPod(k *kubectl.Kubectl, checkList [][]string) error { - for _, check := range checkList { - if err := k.WaitForNamespaceWithPod(check[0], check[1]); err != nil { - return err - } - } - - return nil -} - -func SetClientKubeConfig(ns, name string) (string, error) { - clientKubeConfig, err := CreateTemp("clientKubeConfig") - if err != nil { - return "", err - } - - // Get Kubeconfig of client cluster - out, err := kubectl.Run("get", "secret", - "--namespace", ns, - name+"-kubeconfig", "-o", "jsonpath={.data.value}") - if err != nil { - os.Remove(clientKubeConfig) - return "", err - } - - // Decode Kubeconfig data and write into file - data, err := base64.StdEncoding.DecodeString(out) - if err != nil { - os.Remove(clientKubeConfig) - return "", err - } - err = WriteFile(clientKubeConfig, data) - if err != nil { - os.Remove(clientKubeConfig) - return "", err - } - - // Export KUBECONFIG envar - err = os.Setenv("KUBECONFIG", clientKubeConfig) - if err != nil { - os.Remove(clientKubeConfig) - return "", err - } - - return clientKubeConfig, nil -} diff --git a/tests/e2e/helpers/network/network.go b/tests/e2e/helpers/network/network.go new file mode 100644 index 000000000..a45819e53 --- /dev/null +++ b/tests/e2e/helpers/network/network.go @@ -0,0 +1,83 @@ +/* +Copyright © 2022 - 2023 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "fmt" + "net/http" + "os" + "path/filepath" + + "github.com/rancher-sandbox/ele-testhelpers/tools" +) + +/** + * Configure iPXE server for OS provisioning + * @remarks An iPXE server is up and running + * @param httpSrv IP address:port where the files are shared + * @returns The number of .ipxe files found or an error + */ +func ConfigureiPXE(httpSrv string) (int, error) { + ipxeScript, err := tools.GetFiles("../..", "*.ipxe") + if err != nil { + return 0, err + } + + // NOTE: always use the first ipxe file found! + if len(ipxeScript) >= 1 { + err = tools.Sed("set url.*", "set url "+httpSrv, ipxeScript[0]) + if err != nil { + return 0, err + } + + err = tools.Sed(".*set config.*", "set config $${url}/install-config.yaml", ipxeScript[0]) + if err != nil { + return 0, err + } + + // Delete the previous symlink, use RemoveAll to avoid error if the file doesn't exist + symLink := "../../install-elemental.ipxe" + err = os.RemoveAll(symLink) + if err != nil { + return 0, err + } + + // Create a symlink to the needed ipxe file + scriptName := filepath.Base(ipxeScript[0]) + err = os.Symlink(scriptName, symLink) + if err != nil { + return 0, err + } + } + + // Returns the number of ipxe files found + return len(ipxeScript), nil +} + +/** + * Share files through HTTP (simple way, no security at all!) + * @remarks A HTTP server is up and running + * @param directory The directory where is files are + * @param listenAddr Port where to listen to + */ +func HttpShare(directory, listenAddr string) { + fs := http.FileServer(http.Dir(directory)) + + go func() { + if err := http.ListenAndServe(listenAddr, fs); err != nil { + fmt.Printf("Server failed: %s\n", err) + } + }() +} diff --git a/tests/e2e/install_test.go b/tests/e2e/install_test.go index 6ce29d1a0..0048c6d50 100644 --- a/tests/e2e/install_test.go +++ b/tests/e2e/install_test.go @@ -23,9 +23,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "github.com/rancher-sandbox/ele-testhelpers/rancher" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/install" - "github.com/rancher/elemental/tests/e2e/helpers/misc" ) var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { @@ -33,7 +32,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { // Default timeout is too small, so New() cannot be used k := &kubectl.Kubectl{ Namespace: "", - PollTimeout: misc.SetTimeout(300 * time.Second), + PollTimeout: tools.SetTimeout(300 * time.Second), PollInterval: 500 * time.Millisecond, } @@ -56,7 +55,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { GinkgoWriter.Printf("RKE2 installation loop %d:\n%s\n", count, out) count++ return err - }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(BeNil()) + }, tools.SetTimeout(2*time.Minute), 5*time.Second).Should(BeNil()) }) if clusterType == "hardened" { @@ -71,7 +70,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { Expect(err).To(Not(HaveOccurred())) // Delay few seconds before checking - time.Sleep(misc.SetTimeout(20 * time.Second)) + time.Sleep(tools.SetTimeout(20 * time.Second)) err = exec.Command("sudo", "chown", "gh-runner", "/etc/rancher/rke2/rke2.yaml").Run() Expect(err).To(Not(HaveOccurred())) @@ -90,7 +89,8 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { {"kube-system", "app=rke2-metrics-server"}, {"kube-system", "app.kubernetes.io/name=rke2-ingress-nginx"}, } - misc.CheckPod(k, checkList) + err = rancher.CheckPod(k, checkList) + Expect(err).To(Not(HaveOccurred())) err = k.WaitLabelFilter("kube-system", "Ready", "rke2-ingress-nginx-controller", "app.kubernetes.io/name=rke2-ingress-nginx") Expect(err).To(Not(HaveOccurred())) @@ -110,7 +110,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { GinkgoWriter.Printf("K3s installation loop %d:\n%s\n", count, out) count++ return err - }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(BeNil()) + }, tools.SetTimeout(2*time.Minute), 5*time.Second).Should(BeNil()) }) if clusterType == "hardened" { @@ -125,7 +125,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { Expect(err).To(Not(HaveOccurred())) // Delay few seconds before checking - time.Sleep(misc.SetTimeout(20 * time.Second)) + time.Sleep(tools.SetTimeout(20 * time.Second)) }) By("Waiting for K3s to be started", func() { @@ -137,7 +137,8 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { {"kube-system", "app.kubernetes.io/name=traefik"}, {"kube-system", "svccontroller.k3s.cattle.io/svcname=traefik"}, } - misc.CheckPod(k, checkList) + err := rancher.CheckPod(k, checkList) + Expect(err).To(Not(HaveOccurred())) }) } @@ -146,7 +147,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { // NOTE: don't check for error, as it will happen anyway (only K3s or RKE2 is installed at a time) file, _ := exec.Command("bash", "-c", "ls /etc/rancher/{k3s,rke2}/{k3s,rke2}.yaml").Output() Expect(file).To(Not(BeEmpty())) - misc.CopyFile(strings.Trim(string(file), "\n"), localKubeconfig) + tools.CopyFile(strings.Trim(string(file), "\n"), localKubeconfig) err := os.Setenv("KUBECONFIG", localKubeconfig) Expect(err).To(Not(HaveOccurred())) @@ -186,12 +187,14 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { {"cert-manager", "app.kubernetes.io/component=webhook"}, {"cert-manager", "app.kubernetes.io/component=cainjector"}, } - misc.CheckPod(k, checkList) + err = rancher.CheckPod(k, checkList) + Expect(err).To(Not(HaveOccurred())) }) } By("Installing Rancher", func() { - install.DeployRancherManager(rancherHostname, rancherChannel, rancherVersion, caType, proxy) + err := rancher.DeployRancherManager(rancherHostname, rancherChannel, rancherVersion, caType, proxy) + Expect(err).To(Not(HaveOccurred())) // Inject secret for Private CA if caType == "private" { @@ -217,7 +220,8 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { {"cattle-fleet-local-system", "app=fleet-agent"}, {"cattle-system", "app=rancher-webhook"}, } - misc.CheckPod(k, checkList) + err = rancher.CheckPod(k, checkList) + Expect(err).To(Not(HaveOccurred())) // Check issuer for Private CA if caType == "private" { @@ -228,7 +232,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { GinkgoWriter.Printf("%s\n", out) } return err - }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) } }) @@ -255,10 +259,10 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { "-o", "jsonpath={.data.tls\\.crt}", ) return err - }, misc.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) + }, tools.SetTimeout(2*time.Minute), 5*time.Second).Should(Not(HaveOccurred())) // Copy skel file for ~/.kube/config - misc.CopyFile(localKubeconfigYaml, localKubeconfig) + tools.CopyFile(localKubeconfigYaml, localKubeconfig) // Create kubeconfig for local cluster err = tools.Sed("%RANCHER_URL%", rancherHostname, localKubeconfig) @@ -278,7 +282,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { if testType == "ui" { By("Workaround for upgrade test, restart Fleet controller and agent", func() { // https://github.com/rancher/elemental/issues/410 - time.Sleep(misc.SetTimeout(120 * time.Second)) + time.Sleep(tools.SetTimeout(120 * time.Second)) _, err := kubectl.Run("scale", "deployment/fleet-agent", "-n", "cattle-fleet-local-system", "--replicas=0") Expect(err).To(Not(HaveOccurred())) _, err = kubectl.Run("scale", "deployment/fleet-controller", "-n", "cattle-fleet-system", "--replicas=0") @@ -306,11 +310,12 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() { "--namespace", "cattle-elemental-system", "--create-namespace", ) - }, misc.SetTimeout(1*time.Minute), 10*time.Second).Should(BeNil()) + }, tools.SetTimeout(1*time.Minute), 10*time.Second).Should(BeNil()) } // Wait for pod to be started - misc.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + err := rancher.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + Expect(err).To(Not(HaveOccurred())) }) }) }) diff --git a/tests/e2e/logs_test.go b/tests/e2e/logs_test.go index 9926c1a22..8ded65eeb 100644 --- a/tests/e2e/logs_test.go +++ b/tests/e2e/logs_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher-sandbox/ele-testhelpers/tools" ) func checkRC(err error) { @@ -61,7 +61,7 @@ var _ = Describe("E2E - Getting logs node", Label("logs"), func() { for _, b := range []binary{elemental, logCollector} { Eventually(func() error { return exec.Command("curl", "-L", b.Url, "-o", b.Name).Run() - }, misc.SetTimeout(1*time.Minute), 5*time.Second).Should(BeNil()) + }, tools.SetTimeout(1*time.Minute), 5*time.Second).Should(BeNil()) err := exec.Command("chmod", "+x", b.Name).Run() checkRC(err) diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 998462bd1..72ad63f4f 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -25,17 +25,19 @@ import ( . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher/elemental/tests/e2e/helpers/elemental" + "github.com/rancher/elemental/tests/e2e/helpers/network" ) const ( appYaml = "../assets/hello-world_app.yaml" - clusterYaml = "../assets/cluster.yaml" backupYaml = "../assets/backup.yaml" - emulateTPMYaml = "../assets/emulateTPM.yaml" + clusterYaml = "../assets/cluster.yaml" ciTokenYaml = "../assets/local-kubeconfig-token-skel.yaml" configPrivateCAScript = "../scripts/config-private-ca" dumbRegistrationYaml = "../assets/dumb_machineRegistration.yaml" + emulateTPMYaml = "../assets/emulateTPM.yaml" + httpSrv = "http://192.168.122.1:8000" installConfigYaml = "../../install-config.yaml" installHardenedScript = "../scripts/config-hardened" installVMScript = "../scripts/install-vm" @@ -96,7 +98,7 @@ func CheckClusterState(ns, cluster string) { "--namespace", ns, cluster, "-o", "jsonpath={.status.conditions[?(@.type==\"Ready\")].status}") return clusterStatus - }, misc.SetTimeout(2*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(Equal("True")) + }, tools.SetTimeout(2*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(Equal("True")) // Wait a little bit for the cluster to be in a stable state // Because if we do the next test too quickly it can be a false positive! @@ -109,7 +111,7 @@ func CheckClusterState(ns, cluster string) { "--namespace", ns, cluster, "-o", "jsonpath={.status.conditions[*].reason}") return reason - }, misc.SetTimeout(3*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(BeEmpty()) + }, tools.SetTimeout(3*time.Duration(usedNodes)*time.Minute), 10*time.Second).Should(BeEmpty()) } func GetNodeInfo(hostName string) (*tools.Client, string) { @@ -174,7 +176,7 @@ var _ = BeforeSuite(func() { Expect(err).To(Not(HaveOccurred())) // Set default hostname - vmName = misc.SetHostname(vmNameRoot, vmIndex) + vmName = elemental.SetHostname(vmNameRoot, vmIndex) } else { // Default value for vmIndex vmIndex = 0 @@ -225,5 +227,5 @@ var _ = BeforeSuite(func() { } // Start HTTP server - misc.FileShare("../..", ":8000") + network.HttpShare("../..", ":8000") }) diff --git a/tests/e2e/ui_test.go b/tests/e2e/ui_test.go index a008cd5ff..ba88b2c75 100644 --- a/tests/e2e/ui_test.go +++ b/tests/e2e/ui_test.go @@ -22,8 +22,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "github.com/rancher-sandbox/ele-testhelpers/rancher" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher/elemental/tests/e2e/helpers/elemental" + "github.com/rancher/elemental/tests/e2e/helpers/network" ) var _ = Describe("E2E - Bootstrap node for UI", Label("ui"), func() { @@ -57,7 +59,7 @@ var _ = Describe("E2E - Bootstrap node for UI", Label("ui"), func() { if isoBoot != "true" { By("Configuring iPXE boot script for network installation", func() { - numberOfFile, err := misc.ConfigureiPXE() + numberOfFile, err := network.ConfigureiPXE(httpSrv) Expect(err).To(Not(HaveOccurred())) Expect(numberOfFile).To(BeNumerically(">=", 1)) }) @@ -66,7 +68,7 @@ var _ = Describe("E2E - Bootstrap node for UI", Label("ui"), func() { By("Adding VM in default network", func() { // Add node in network configuration if needed if macAdrs == "" { - err := misc.AddNode(netDefaultFileName, vmName, vmIndex) + err := rancher.AddNode(netDefaultFileName, vmName, vmIndex) Expect(err).To(Not(HaveOccurred())) } @@ -90,7 +92,7 @@ var _ = Describe("E2E - Bootstrap node for UI", Label("ui"), func() { }) By("Checking that the VM is available in Rancher", func() { - id, err := misc.GetServerId(clusterNS, vmIndex) + id, err := elemental.GetServerId(clusterNS, vmIndex) Expect(err).To(Not(HaveOccurred())) Expect(id).To(Not(BeEmpty())) }) @@ -101,7 +103,7 @@ var _ = Describe("E2E - Bootstrap node for UI", Label("ui"), func() { }) By("Checking VM connection", func() { - id, err := misc.GetServerId(clusterNS, vmIndex) + id, err := elemental.GetServerId(clusterNS, vmIndex) Expect(err).To(Not(HaveOccurred())) Expect(id).To(Not(BeEmpty())) @@ -110,7 +112,7 @@ var _ = Describe("E2E - Bootstrap node for UI", Label("ui"), func() { out, _ := client.RunSSH("uname -n") out = strings.Trim(out, "\n") return out - }, misc.SetTimeout(2*time.Minute), 5*time.Second) + }, tools.SetTimeout(2*time.Minute), 5*time.Second) }) }) }) diff --git a/tests/e2e/uninstall-operator_test.go b/tests/e2e/uninstall-operator_test.go index 57ceb93a4..67a847236 100644 --- a/tests/e2e/uninstall-operator_test.go +++ b/tests/e2e/uninstall-operator_test.go @@ -22,7 +22,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher-sandbox/ele-testhelpers/rancher" + "github.com/rancher-sandbox/ele-testhelpers/tools" ) func testClusterAvailability(ns, cluster string) { @@ -31,7 +32,7 @@ func testClusterAvailability(ns, cluster string) { "--namespace", ns, cluster, "-o", "jsonpath={.metadata.name}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(clusterName)) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(clusterName)) } var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator"), func() { @@ -39,7 +40,7 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator // Default timeout is too small, so New() cannot be used k := &kubectl.Kubectl{ Namespace: "", - PollTimeout: misc.SetTimeout(300 * time.Second), + PollTimeout: tools.SetTimeout(300 * time.Second), PollInterval: 500 * time.Millisecond, } @@ -81,7 +82,7 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator "--namespace", clusterNS, "-o", "jsonpath={.items[*].metadata.name}") return out - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("NotFound")) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("NotFound")) }) By("Deleting cluster resource", func() { @@ -119,7 +120,8 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator } // Wait for pod to be started - misc.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + err := rancher.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + Expect(err).To(Not(HaveOccurred())) }) By("Creating a dumb MachineRegistration", func() { diff --git a/tests/e2e/upgrade_test.go b/tests/e2e/upgrade_test.go index bcba86260..886de0cc3 100644 --- a/tests/e2e/upgrade_test.go +++ b/tests/e2e/upgrade_test.go @@ -24,9 +24,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/rancher-sandbox/ele-testhelpers/kubectl" + "github.com/rancher-sandbox/ele-testhelpers/rancher" "github.com/rancher-sandbox/ele-testhelpers/tools" - "github.com/rancher/elemental/tests/e2e/helpers/install" - "github.com/rancher/elemental/tests/e2e/helpers/misc" + "github.com/rancher/elemental/tests/e2e/helpers/elemental" ) var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator"), func() { @@ -34,7 +34,7 @@ var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator") // Default timeout is too small, so New() cannot be used k := &kubectl.Kubectl{ Namespace: "", - PollTimeout: misc.SetTimeout(300 * time.Second), + PollTimeout: tools.SetTimeout(300 * time.Second), PollInterval: 500 * time.Millisecond, } @@ -63,10 +63,11 @@ var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator") } // Delay few seconds before checking, needed because we may have 2 pods at the same time - time.Sleep(misc.SetTimeout(30 * time.Second)) + time.Sleep(tools.SetTimeout(30 * time.Second)) // Wait for all pods to be started - misc.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + err = rancher.CheckPod(k, [][]string{{"cattle-elemental-system", "app=elemental-operator"}}) + Expect(err).To(Not(HaveOccurred())) }) }) @@ -75,7 +76,7 @@ var _ = Describe("E2E - Upgrading Rancher Manager", Label("upgrade-rancher-manag // Default timeout is too small, so New() cannot be used k := &kubectl.Kubectl{ Namespace: "", - PollTimeout: misc.SetTimeout(300 * time.Second), + PollTimeout: tools.SetTimeout(300 * time.Second), PollInterval: 500 * time.Millisecond, } @@ -91,21 +92,28 @@ var _ = Describe("E2E - Upgrading Rancher Manager", Label("upgrade-rancher-manag Expect(err).To(Not(HaveOccurred())) // Upgrade Rancher Manager - install.DeployRancherManager(rancherHostname, rancherUpgradeChannel, rancherUpgradeVersion, caType, proxy) + err = rancher.DeployRancherManager(rancherHostname, rancherUpgradeChannel, rancherUpgradeVersion, caType, proxy) + Expect(err).To(Not(HaveOccurred())) - // Wait for Rancher Manager to be running + // Wait for Rancher Manager to be restarted + status, err := kubectl.Run("rollout", "status", "deployment/rancher") + Expect(err).To(Not(HaveOccurred())) + Expect(status).To(ContainSubstring("successfully rolled out")) + + // Check that all Rancher Manager pods are running checkList := [][]string{ {"cattle-system", "app=rancher"}, {"cattle-fleet-local-system", "app=fleet-agent"}, {"cattle-system", "app=rancher-webhook"}, } - misc.CheckPod(k, checkList) + err = rancher.CheckPod(k, checkList) + Expect(err).To(Not(HaveOccurred())) // Check that all pods are using the same version Eventually(func() int { out, _ := kubectl.Run(getImageVersion...) return len(strings.Fields(out)) - }, misc.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(1)) + }, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(Equal(1)) // Get after-upgrade Rancher Manager version // and check that it's different to the before-upgrade version @@ -129,7 +137,7 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Get node information @@ -155,13 +163,13 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { By("Triggering Upgrade in Rancher with "+upgradeType, func() { // Set temporary file - upgradeTmp, err := misc.CreateTemp("upgrade") + upgradeTmp, err := tools.CreateTemp("upgrade") Expect(err).To(Not(HaveOccurred())) defer os.Remove(upgradeTmp) if upgradeType == "managedOSVersionName" { // Get elemental-operator version - operatorVersion, err := misc.GetOperatorVersion() + operatorVersion, err := elemental.GetOperatorVersion() Expect(err).To(Not(HaveOccurred())) operatorVersionShort := strings.Split(operatorVersion, ".") @@ -184,7 +192,7 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { out, _ := kubectl.Run("get", "ManagedOSVersion", "--namespace", clusterNS, upgradeOsChannel) return out - }, misc.SetTimeout(2*time.Minute), 10*time.Second).Should(Not(ContainSubstring("Error"))) + }, tools.SetTimeout(2*time.Minute), 10*time.Second).Should(Not(ContainSubstring("Error"))) // Set OS image to use for upgrade value = upgradeOsChannel @@ -194,19 +202,19 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { "--namespace", clusterNS, upgradeOsChannel, "-o", "jsonpath={.spec.metadata.upgradeImage}") Expect(err).To(Not(HaveOccurred())) - valueToCheck = misc.TrimStringFromChar(out, ":") + valueToCheck = tools.TrimStringFromChar(out, ":") } else if upgradeType == "osImage" { // Set OS image to use for upgrade value = upgradeImage // Extract the value to check after the upgrade - valueToCheck = misc.TrimStringFromChar(upgradeImage, ":") + valueToCheck = tools.TrimStringFromChar(upgradeImage, ":") } // Add a nodeSelector if needed if usedNodes == 1 { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, vmIndex) + hostName := elemental.SetHostname(vmNameRoot, vmIndex) Expect(hostName).To(Not(BeNil())) // Get node information @@ -219,15 +227,15 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { hostname = strings.Trim(hostname, "\n") label := "kubernetes.io/hostname" - selector, err := misc.AddSelector(label, hostname) + selector, err := elemental.AddSelector(label, hostname) Expect(err).To(Not(HaveOccurred()), selector) // Create new file for this specific upgrade - err = misc.ConcateFiles(upgradeSkelYaml, upgradeTmp, selector) + err = tools.AddDataToFile(upgradeSkelYaml, upgradeTmp, selector) Expect(err).To(Not(HaveOccurred())) } else { // Use original file as-is - misc.CopyFile(upgradeSkelYaml, upgradeTmp) + tools.CopyFile(upgradeSkelYaml, upgradeTmp) } // Set values @@ -245,7 +253,7 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { for index := vmIndex; index <= numberOfVMs; index++ { // Set node hostname - hostName := misc.SetHostname(vmNameRoot, index) + hostName := elemental.SetHostname(vmNameRoot, index) Expect(hostName).To(Not(BeNil())) // Get node information @@ -265,8 +273,8 @@ var _ = Describe("E2E - Upgrading node", Label("upgrade-node"), func() { // This remove the version and keep only the repo, as in the file // we have the exact version and we don't know it before the upgrade - return misc.TrimStringFromChar(strings.Trim(out, "\n"), ":") - }, misc.SetTimeout(5*time.Minute), 30*time.Second).Should(Equal(valueToCheck)) + return tools.TrimStringFromChar(strings.Trim(out, "\n"), ":") + }, tools.SetTimeout(5*time.Minute), 30*time.Second).Should(Equal(valueToCheck)) }) By("Checking OS version on "+h+" after upgrade", func() {