add snapshot and compare cluster functions to e2e tests (#1018)

Makefile

@@ -102,7 +102,7 @@ test: generate fmt vet manifests bin/setup-envtest
 test-e2e:
 	go test github.com/banzaicloud/koperator/tests/e2e \
 		-v \
-		-timeout 15m \
+		-timeout 20m \
 		-tags e2e \
 		--ginkgo.show-node-events \
 		--ginkgo.trace \

tests/e2e/const.go

@@ -31,13 +31,12 @@ const (
 	kubectlArgGoTemplateExternalListenersName             = `-o=go-template='{{range $key,$value := .status.listenerStatuses.externalListeners}}{{$key}}{{"\n"}}{{end}}`
 	kubectlArgGoTemplateExternalListenerAddressesTemplate = `-o=go-template='{{range .status.listenerStatuses.externalListeners.%s}}{{.address}}{{"\n"}}{{end}}`
 
-
-	crdKind                    = "customresourcedefinitions.apiextensions.k8s.io"
-	kafkaKind                  = "kafkaclusters.kafka.banzaicloud.io"
-	kafkaTopicKind             = "kafkatopics.kafka.banzaicloud.io"
-	kafkaClusterName           = "kafka"
-	testExternalTopicName      = "topic-test-external"
-	testInternalTopicName      = "topic-test-internal"
+	crdKind               = "customresourcedefinitions.apiextensions.k8s.io"
+	kafkaKind             = "kafkaclusters.kafka.banzaicloud.io"
+	kafkaTopicKind        = "kafkatopics.kafka.banzaicloud.io"
+	kafkaClusterName      = "kafka"
+	testExternalTopicName = "topic-test-external"
+	testInternalTopicName = "topic-test-internal"
 
 	kcatPodName                = "kcat"
 	zookeeperKind              = "zookeeperclusters.zookeeper.pravega.io"

tests/e2e/koperator_suite_test.go

@@ -54,6 +54,8 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = When("Testing e2e test altogether", Ordered, func() {
+	var snapshottedInfo = &clusterSnapshot{}
+	snapshotCluster(snapshottedInfo)
 	testInstall()
 	testInstallZookeeperCluster()
 	testInstallKafkaCluster("../../config/samples/simplekafkacluster.yaml")
@@ -62,4 +64,5 @@ var _ = When("Testing e2e test altogether", Ordered, func() {
 	testUninstallKafkaCluster()
 	testUninstallZookeeperCluster()
 	testUninstall()
+	snapshotClusterAndCompare(snapshottedInfo)
 })

tests/e2e/test_snapshot.go

@@ -0,0 +1,196 @@
+// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type clusterSnapshot struct {
+	resources []metav1.PartialObjectMetadata
+}
+
+func (s *clusterSnapshot) Resources() []metav1.PartialObjectMetadata {
+	return s.resources
+}
+
+// ResourcesAsComparisonType returns a slice of a helper type that makes comparisons easier
+func (s *clusterSnapshot) ResourcesAsComparisonType() []localComparisonPartialObjectMetadataType {
+	var localList []localComparisonPartialObjectMetadataType
+	for _, r := range s.resources {
+		localList = append(localList, localComparisonPartialObjectMetadataType{
+			GVK:       r.GroupVersionKind(),
+			Namespace: r.GetNamespace(),
+			Name:      r.GetName(),
+		})
+	}
+	return localList
+}
+
+// localComparisonPartialObjectMetadataType holds a version of the minimal information required
+// to compare k8s.io/apimachinery/pkg/apis/meta/v1.PartialObjectMetadata instances
+type localComparisonPartialObjectMetadataType struct {
+	GVK       schema.GroupVersionKind
+	Namespace string
+	Name      string
+}
+
+// snapshotCluster takes a clusterSnapshot of a K8s cluster and
+// stores it into the snapshotCluster instance referenced as input
+func snapshotCluster(snapshottedInfo *clusterSnapshot) bool {
+	return When("Get cluster resources state", Ordered, func() {
+		var kubectlOptions k8s.KubectlOptions
+		var err error
+
+		BeforeAll(func() {
+			By("Acquiring K8s config and context")
+			kubectlOptions, err = kubectlOptionsForCurrentContext()
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		var clusterResourceNames []string
+		var namespacedResourceNames []string
+
+		When("Get api-resources names", func() {
+			It("Get cluster-scoped api-resources names", func() {
+				clusterResourceNames, err = listK8sResourceKinds(kubectlOptions, "", "--namespaced=false")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(clusterResourceNames).NotTo(BeNil())
+				clusterResourceNames = pruneUnnecessaryClusterResourceNames(clusterResourceNames)
+			})
+			It("Get namespaced api-resources names", func() {
+				namespacedResourceNames, err = listK8sResourceKinds(kubectlOptions, "", "--namespaced=true")
+				Expect(err).NotTo(HaveOccurred())
+				Expect(namespacedResourceNames).NotTo(BeNil())
+				namespacedResourceNames = pruneUnnecessaryNamespacedResourceNames(namespacedResourceNames)
+			})
+		})
+
+		var resources []metav1.PartialObjectMetadata
+
+		var namespacesForNamespacedResources = []string{"default"}
+
+		When("Snapshotting objects", func() {
+			It("Recording cluster-scoped resource objects", func() {
+				By(fmt.Sprintf("Getting cluster-scoped resources %v as json", clusterResourceNames))
+				output, err := getK8sResources(kubectlOptions, clusterResourceNames, "", "", "--output=json")
+				Expect(err).NotTo(HaveOccurred())
+
+				By(fmt.Sprintf("Unmarshalling cluster-scoped resources %v from json", clusterResourceNames))
+				var resourceList metav1.PartialObjectMetadataList
+				err = json.Unmarshal([]byte(strings.Join(output, "\n")), &resourceList)
+				Expect(err).NotTo(HaveOccurred())
+
+				resources = append(resources, resourceList.Items...)
+			})
+			It("Recording namespaced resource objects", func() {
+				initialNS := kubectlOptions.Namespace
+				for _, ns := range namespacesForNamespacedResources {
+					kubectlOptions.Namespace = ns
+
+					By(fmt.Sprintf("Getting namespaced resources %v as json for namespace %s", namespacedResourceNames, ns))
+					output, err := getK8sResources(kubectlOptions, namespacedResourceNames, "", "", "--output=json")
+					Expect(err).NotTo(HaveOccurred())
+
+					By(fmt.Sprintf("Unmarshalling namespaced resources %v from json for namespace %s", namespacedResourceNames, ns))
+					var resourceList metav1.PartialObjectMetadataList
+					err = json.Unmarshal([]byte(strings.Join(output, "\n")), &resourceList)
+					Expect(err).NotTo(HaveOccurred())
+
+					resources = append(resources, resourceList.Items...)
+				}
+				kubectlOptions.Namespace = initialNS
+			})
+		})
+
+		AfterAll(func() {
+			By("Storing recorded objects into the input snapshot object")
+			snapshottedInfo.resources = resources
+		})
+
+	})
+}
+
+// snapshotClusterAndCompare takes a current snapshot of the K8s cluster and
+// compares it against a snapshot provided as input
+func snapshotClusterAndCompare(snapshottedInitialInfo *clusterSnapshot) bool {
+	return When("Verifying cluster resources state", Ordered, func() {
+		var snapshottedCurrentInfo = &clusterSnapshot{}
+		snapshotCluster(snapshottedCurrentInfo)
+
+		It("Checking resources list", func() {
+			// Temporarily increase maximum output length (default 4000) to fit more objects in the printed diff.
+			// Only doing this here because other assertions typically don't run against objects with this many elements.
+			initialMaxLength := format.MaxLength
+			defer func() { format.MaxLength = initialMaxLength }()
+			format.MaxLength = 9000
+
+			Expect(snapshottedCurrentInfo.ResourcesAsComparisonType()).To(ConsistOf(snapshottedInitialInfo.ResourcesAsComparisonType()))
+		})
+	})
+}
+
+func pruneUnnecessaryClusterResourceNames(resourceNameList []string) []string {
+	var updatedList []string
+	for _, name := range resourceNameList {
+		// Avoid failing because the number of K8s workers changed during the test. (e.g. PKE)
+		if name == "nodes" {
+			continue
+		}
+		// When the number of nodes changes we also get CSRs for signers kubernetes.io/kubelet-serving and kubernetes.io/kube-apiserver-client-kubelet
+		// TODO: in time, we want to be able to compare CSRs, too, or be able to ignore particular CSR list differences.
+		if name == "certificatesigningrequests.certificates.k8s.io" {
+			continue
+		}
+		// Ignore CSI elements from storage.k8s.io
+		// Additionally, these resources don't mesh well with computing differences for clusters with a variable number of workers.
+		if name == "csidrivers.storage.k8s.io" || name == "csinodes.storage.k8s.io" || name == "csistoragecapacities.storage.k8s.io" {
+			continue
+		}
+		// We never need to snapshot Cilium-related resources (namespaced or not).
+		if strings.HasPrefix(name, "cilium") {
+			continue
+		}
+		updatedList = append(updatedList, name)
+	}
+	return updatedList
+}
+
+func pruneUnnecessaryNamespacedResourceNames(resourceNameList []string) []string {
+	var updatedList []string
+	for _, name := range resourceNameList {
+		// The list of K8s Events is rarely unchanged over time. It is not fit for comparison.
+		// Additionally, at the very least, KafkaCluster installs create PVs which generate events by themselves.
+		if name == "events" || name == "events.events.k8s.io" {
+			continue
+		}
+		// We never need to snapshot Cilium-related resources (namespaced or not).
+		if strings.HasPrefix(name, "cilium") {
+			continue
+		}
+		updatedList = append(updatedList, name)
+	}
+	return updatedList
+}