diff --git a/pkg/common/objectmeta/objectmeta.go b/pkg/common/objectmeta/objectmeta.go index 47e24dc9..9b79a10e 100644 --- a/pkg/common/objectmeta/objectmeta.go +++ b/pkg/common/objectmeta/objectmeta.go @@ -20,6 +20,10 @@ const ( // TrafficManagerProfileFinalizer a finalizer added by the TrafficManagerProfile controller to all trafficManagerProfiles, // to make sure that the controller can react to profile deletions if necessary. TrafficManagerProfileFinalizer = fleetNetworkingPrefix + "traffic-manager-profile-cleanup" + + // TrafficManagerBackendFinalizer a finalizer added by the TrafficManagerBackend controller to all trafficManagerBackends, + // to make sure that the controller can react to backend deletions if necessary. + TrafficManagerBackendFinalizer = fleetNetworkingPrefix + "traffic-manager-backend-cleanup" ) // Labels diff --git a/pkg/controllers/hub/trafficmanagerbackend/controller.go b/pkg/controllers/hub/trafficmanagerbackend/controller.go new file mode 100644 index 00000000..4c39429c --- /dev/null +++ b/pkg/controllers/hub/trafficmanagerbackend/controller.go @@ -0,0 +1,289 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +// Package trafficmanagerbackend features the TrafficManagerBackend controller to reconcile TrafficManagerBackend CRs. +package trafficmanagerbackend + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager" + "golang.org/x/sync/errgroup" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "go.goms.io/fleet/pkg/utils/controller" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/pkg/common/azureerrors" + "go.goms.io/fleet-networking/pkg/common/objectmeta" + "go.goms.io/fleet-networking/pkg/controllers/hub/trafficmanagerprofile" +) + +const ( + trafficManagerBackendProfileFieldKey = ".spec.profile.name" + trafficManagerBackendBackendFieldKey = ".spec.backend.name" + + // AzureResourceEndpointNamePrefix is the prefix format of the Azure Traffic Manager Endpoint created by the fleet controller. + // The naming convention of a Traffic Manager Endpoint is fleet-{TrafficManagerBackendUUID}#. + // Using the UUID of the backend here in case to support cross namespace TrafficManagerBackend in the future. + AzureResourceEndpointNamePrefix = "fleet-%s#" + + // AzureResourceEndpointNameFormat is the name format of the Azure Traffic Manager Endpoint created by the fleet controller. + // The naming convention of a Traffic Manager Endpoint is fleet-{TrafficManagerBackendUUID}#{ServiceImportName}#{ClusterName}. + // All the object name length should be restricted to <= 63 characters. + // The endpoint name must contain no more than 260 characters, excluding the following characters "< > * % $ : \ ? + /". + AzureResourceEndpointNameFormat = AzureResourceEndpointNamePrefix + "%s#%s" +) + +var ( + // create the func as a variable so that the integration test can use a customized function. + generateAzureTrafficManagerProfileNameFunc = func(profile *fleetnetv1alpha1.TrafficManagerProfile) string { + return trafficmanagerprofile.GenerateAzureTrafficManagerProfileName(profile) + } + generateAzureTrafficManagerEndpointNamePrefixFunc = func(backend *fleetnetv1alpha1.TrafficManagerBackend) string { + return fmt.Sprintf(AzureResourceEndpointNamePrefix, backend.UID) + } +) + +// Reconciler reconciles a trafficManagerBackend object. +type Reconciler struct { + client.Client + + ProfilesClient *armtrafficmanager.ProfilesClient + EndpointsClient *armtrafficmanager.EndpointsClient + ResourceGroupName string // default resource group name to create azure traffic manager resources +} + +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=trafficmanagerbackends,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=trafficmanagerbackends/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=trafficmanagerbackends/finalizers,verbs=get;update +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=trafficmanagerprofiles,verbs=get;list;watch +//+kubebuilder:rbac:groups=networking.fleet.azure.com,resources=serviceimports,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile triggers a single reconcile round. +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + name := req.NamespacedName + backendKRef := klog.KRef(name.Namespace, name.Name) + + startTime := time.Now() + klog.V(2).InfoS("Reconciliation starts", "trafficManagerBackend", backendKRef) + defer func() { + latency := time.Since(startTime).Milliseconds() + klog.V(2).InfoS("Reconciliation ends", "trafficManagerBackend", backendKRef, "latency", latency) + }() + + backend := &fleetnetv1alpha1.TrafficManagerBackend{} + if err := r.Client.Get(ctx, name, backend); err != nil { + if apierrors.IsNotFound(err) { + klog.V(4).InfoS("Ignoring NotFound trafficManagerBackend", "trafficManagerBackend", backendKRef) + return ctrl.Result{}, nil + } + klog.ErrorS(err, "Failed to get trafficManagerBackend", "trafficManagerBackend", backendKRef) + return ctrl.Result{}, controller.NewAPIServerError(true, err) + } + + if !backend.ObjectMeta.DeletionTimestamp.IsZero() { + return r.handleDelete(ctx, backend) + } + + // register finalizer + if !controllerutil.ContainsFinalizer(backend, objectmeta.TrafficManagerBackendFinalizer) { + controllerutil.AddFinalizer(backend, objectmeta.TrafficManagerBackendFinalizer) + if err := r.Update(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to add finalizer to trafficManagerBackend", "trafficManagerBackend", backend) + return ctrl.Result{}, controller.NewUpdateIgnoreConflictError(err) + } + } + return r.handleUpdate(ctx, backend) +} + +func (r *Reconciler) handleDelete(ctx context.Context, backend *fleetnetv1alpha1.TrafficManagerBackend) (ctrl.Result, error) { + backendKObj := klog.KObj(backend) + // The backend is being deleted + if !controllerutil.ContainsFinalizer(backend, objectmeta.TrafficManagerBackendFinalizer) { + klog.V(4).InfoS("TrafficManagerBackend is being deleted", "trafficManagerBackend", backendKObj) + return ctrl.Result{}, nil + } + + if err := r.deleteAzureTrafficManagerEndpoints(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to delete Azure Traffic Manager endpoints", "trafficManagerBackend", backendKObj) + return ctrl.Result{}, err + } + + controllerutil.RemoveFinalizer(backend, objectmeta.TrafficManagerBackendFinalizer) + if err := r.Client.Update(ctx, backend); err != nil { + klog.ErrorS(err, "Failed to remove trafficManagerBackend finalizer", "trafficManagerBackend", backendKObj) + return ctrl.Result{}, controller.NewUpdateIgnoreConflictError(err) + } + klog.V(2).InfoS("Removed trafficManagerBackend finalizer", "trafficManagerBackend", backendKObj) + return ctrl.Result{}, nil +} + +func (r *Reconciler) deleteAzureTrafficManagerEndpoints(ctx context.Context, backend *fleetnetv1alpha1.TrafficManagerBackend) error { + backendKObj := klog.KObj(backend) + profile := &fleetnetv1alpha1.TrafficManagerProfile{} + if err := r.Client.Get(ctx, types.NamespacedName{Name: backend.Spec.Profile.Name, Namespace: backend.Namespace}, profile); err != nil { + if apierrors.IsNotFound(err) { + klog.V(2).InfoS("NotFound trafficManagerProfile and Azure resources should be deleted ", "trafficManagerBackend", backendKObj, "trafficManagerProfile", backend.Spec.Profile.Name) + return nil + } + klog.ErrorS(err, "Failed to get trafficManagerProfile", "trafficManagerBackend", backendKObj, "trafficManagerProfile", backend.Spec.Profile.Name) + return controller.NewAPIServerError(true, err) + } + + profileKObj := klog.KObj(profile) + azureProfileName := generateAzureTrafficManagerProfileNameFunc(profile) + getRes, getErr := r.ProfilesClient.Get(ctx, r.ResourceGroupName, azureProfileName, nil) + if getErr != nil { + if !azureerrors.IsNotFound(getErr) { + klog.ErrorS(getErr, "Failed to get the profile", "trafficManagerBackend", backendKObj, "trafficManagerProfile", profileKObj, "azureProfileName", azureProfileName) + return getErr + } + klog.V(2).InfoS("Azure Traffic Manager profile does not exist", "trafficManagerBackend", backendKObj, "trafficManagerProfile", profileKObj, "azureProfileName", azureProfileName) + return nil // skip handling endpoints deletion + } + if getRes.Profile.Properties == nil { + klog.V(2).InfoS("Azure Traffic Manager profile has nil properties and skipping handling endpoints deletion", "trafficManagerBackend", backendKObj, "trafficManagerProfile", profileKObj, "azureProfileName", azureProfileName) + return nil + } + + klog.V(2).InfoS("Deleting Azure Traffic Manager endpoints", "trafficManagerBackend", backendKObj, "trafficManagerProfile", backend.Spec.Profile.Name) + errs, cctx := errgroup.WithContext(ctx) + for i := range getRes.Profile.Properties.Endpoints { + endpoint := getRes.Profile.Properties.Endpoints[i] + if endpoint.Name == nil { + err := controller.NewUnexpectedBehaviorError(errors.New("azure Traffic Manager endpoint name is nil")) + klog.ErrorS(err, "Invalid Traffic Manager endpoint", "azureEndpoint", endpoint) + continue + } + // Traffic manager endpoint name is case-insensitive. + if !strings.HasPrefix(strings.ToLower(*endpoint.Name), generateAzureTrafficManagerEndpointNamePrefixFunc(backend)) { + continue // skipping deleting the endpoints which are not created by this backend + } + errs.Go(func() error { + if _, err := r.EndpointsClient.Delete(cctx, r.ResourceGroupName, azureProfileName, armtrafficmanager.EndpointTypeAzureEndpoints, *endpoint.Name, nil); err != nil { + if azureerrors.IsNotFound(getErr) { + klog.V(2).InfoS("Ignoring NotFound Azure Traffic Manager endpoint", "trafficManagerBackend", backendKObj, "azureProfileName", azureProfileName, "azureEndpointName", *endpoint.Name) + return nil + } + klog.ErrorS(err, "Failed to delete the endpoint", "trafficManagerBackend", backendKObj, "azureProfileName", azureProfileName, "azureEndpointName", *endpoint.Name) + return err + } + klog.V(2).InfoS("Deleted Azure Traffic Manager endpoint", "trafficManagerBackend", backendKObj, "azureProfileName", azureProfileName, "azureEndpointName", *endpoint.Name) + return nil + }) + } + return errs.Wait() +} + +func (r *Reconciler) handleUpdate(_ context.Context, _ *fleetnetv1alpha1.TrafficManagerBackend) (ctrl.Result, error) { + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + // set up an index for efficient trafficManagerBackend lookup + profileIndexerFunc := func(o client.Object) []string { + tmb, ok := o.(*fleetnetv1alpha1.TrafficManagerBackend) + if !ok { + return []string{} + } + return []string{tmb.Spec.Profile.Name} + } + if err := mgr.GetFieldIndexer().IndexField(ctx, &fleetnetv1alpha1.TrafficManagerBackend{}, trafficManagerBackendProfileFieldKey, profileIndexerFunc); err != nil { + klog.ErrorS(err, "Failed to setup profile field indexer for TrafficManagerBackend") + return err + } + + backendIndexerFunc := func(o client.Object) []string { + tmb, ok := o.(*fleetnetv1alpha1.TrafficManagerBackend) + if !ok { + return []string{} + } + return []string{tmb.Spec.Backend.Name} + } + if err := mgr.GetFieldIndexer().IndexField(ctx, &fleetnetv1alpha1.TrafficManagerBackend{}, trafficManagerBackendBackendFieldKey, backendIndexerFunc); err != nil { + klog.ErrorS(err, "Failed to setup backend field indexer for TrafficManagerBackend") + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&fleetnetv1alpha1.TrafficManagerBackend{}). + Watches( + &fleetnetv1alpha1.TrafficManagerProfile{}, + handler.EnqueueRequestsFromMapFunc(r.trafficManagerProfileEventHandler()), + ). + Watches( + &fleetnetv1alpha1.ServiceImport{}, + handler.EnqueueRequestsFromMapFunc(r.serviceImportEventHandler()), + ). + Complete(r) +} + +func (r *Reconciler) trafficManagerProfileEventHandler() handler.MapFunc { + return func(ctx context.Context, object client.Object) []reconcile.Request { + trafficManagerBackendList := &fleetnetv1alpha1.TrafficManagerBackendList{} + fieldMatcher := client.MatchingFields{ + trafficManagerBackendProfileFieldKey: object.GetName(), + } + // For now, we only support the backend and profile in the same namespace. + if err := r.Client.List(ctx, trafficManagerBackendList, client.InNamespace(object.GetNamespace()), fieldMatcher); err != nil { + klog.ErrorS(err, + "Failed to list trafficManagerBackends for the profile", + "trafficManagerProfile", klog.KObj(object)) + return []reconcile.Request{} + } + + res := make([]reconcile.Request, 0, len(trafficManagerBackendList.Items)) + for _, backend := range trafficManagerBackendList.Items { + res = append(res, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: backend.Namespace, + Name: backend.Name, + }, + }) + } + return res + } +} + +func (r *Reconciler) serviceImportEventHandler() handler.MapFunc { + return func(ctx context.Context, object client.Object) []reconcile.Request { + trafficManagerBackendList := &fleetnetv1alpha1.TrafficManagerBackendList{} + fieldMatcher := client.MatchingFields{ + trafficManagerBackendBackendFieldKey: object.GetName(), + } + // ServiceImport and TrafficManagerBackend should be in the same namespace. + if err := r.Client.List(ctx, trafficManagerBackendList, client.InNamespace(object.GetNamespace()), fieldMatcher); err != nil { + klog.ErrorS(err, + "Failed to list trafficManagerBackends for the serviceImport", + "serviceImport", klog.KObj(object)) + return []reconcile.Request{} + } + + res := make([]reconcile.Request, 0, len(trafficManagerBackendList.Items)) + for _, backend := range trafficManagerBackendList.Items { + res = append(res, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: backend.Namespace, + Name: backend.Name, + }, + }) + } + return res + } +} diff --git a/pkg/controllers/hub/trafficmanagerbackend/controller_integration_test.go b/pkg/controllers/hub/trafficmanagerbackend/controller_integration_test.go new file mode 100644 index 00000000..d49ccbd3 --- /dev/null +++ b/pkg/controllers/hub/trafficmanagerbackend/controller_integration_test.go @@ -0,0 +1,114 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package trafficmanagerbackend + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/test/common/trafficmanager/fakeprovider" + "go.goms.io/fleet-networking/test/common/trafficmanager/validator" +) + +func trafficManagerBackendForTest(name, profileName, serviceImportName string) *fleetnetv1alpha1.TrafficManagerBackend { + return &fleetnetv1alpha1.TrafficManagerBackend{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + }, + Spec: fleetnetv1alpha1.TrafficManagerBackendSpec{ + Profile: fleetnetv1alpha1.TrafficManagerProfileRef{ + Name: profileName, + }, + Backend: fleetnetv1alpha1.TrafficManagerBackendRef{ + Name: serviceImportName, + }, + Weight: ptr.To(int64(10)), + }, + } +} + +func trafficManagerProfileForTest(name string) *fleetnetv1alpha1.TrafficManagerProfile { + return &fleetnetv1alpha1.TrafficManagerProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + }, + Spec: fleetnetv1alpha1.TrafficManagerProfileSpec{}, + } +} + +var _ = Describe("Test TrafficManagerBackend Controller", func() { + Context("When creating trafficManagerBackend with invalid profile", Ordered, func() { + name := fakeprovider.ValidBackendName + namespacedName := types.NamespacedName{Namespace: testNamespace, Name: name} + var backend *fleetnetv1alpha1.TrafficManagerBackend + + It("Creating TrafficManagerBackend", func() { + backend = trafficManagerBackendForTest(name, "not-exist", "not-exist") + Expect(k8sClient.Create(ctx, backend)).Should(Succeed()) + }) + + It("Validating trafficManagerBackend", func() { + validator.IsTrafficManagerBackendFinalizerAdded(ctx, k8sClient, namespacedName) + }) + + It("Deleting trafficManagerBackend", func() { + err := k8sClient.Delete(ctx, backend) + Expect(err).Should(Succeed(), "failed to delete trafficManagerBackend") + }) + + It("Validating trafficManagerBackend is deleted", func() { + validator.IsTrafficManagerBackendDeleted(ctx, k8sClient, namespacedName) + }) + }) + + Context("When creating trafficManagerBackend with valid profile", Ordered, func() { + profileName := fakeprovider.ValidProfileWithEndpointsName + profileNamespacedName := types.NamespacedName{Namespace: testNamespace, Name: profileName} + var profile *fleetnetv1alpha1.TrafficManagerProfile + backendName := fakeprovider.ValidBackendName + backendNamespacedName := types.NamespacedName{Namespace: testNamespace, Name: backendName} + var backend *fleetnetv1alpha1.TrafficManagerBackend + + It("Creating a new TrafficManagerProfile", func() { + By("By creating a new TrafficManagerProfile") + profile = trafficManagerProfileForTest(profileName) + Expect(k8sClient.Create(ctx, profile)).Should(Succeed()) + }) + + It("Creating TrafficManagerBackend", func() { + backend = trafficManagerBackendForTest(backendName, profileName, "not-exist") + Expect(k8sClient.Create(ctx, backend)).Should(Succeed()) + }) + + It("Validating trafficManagerBackend", func() { + validator.IsTrafficManagerBackendFinalizerAdded(ctx, k8sClient, backendNamespacedName) + }) + + It("Deleting trafficManagerBackend", func() { + err := k8sClient.Delete(ctx, backend) + Expect(err).Should(Succeed(), "failed to delete trafficManagerBackend") + }) + + It("Validating trafficManagerBackend is deleted", func() { + validator.IsTrafficManagerBackendDeleted(ctx, k8sClient, backendNamespacedName) + }) + + It("Deleting trafficManagerProfile", func() { + err := k8sClient.Delete(ctx, profile) + Expect(err).Should(Succeed(), "failed to delete trafficManagerProfile") + }) + + It("Validating trafficManagerProfile is deleted", func() { + validator.IsTrafficManagerProfileDeleted(ctx, k8sClient, profileNamespacedName) + }) + }) +}) diff --git a/pkg/controllers/hub/trafficmanagerbackend/suite_test.go b/pkg/controllers/hub/trafficmanagerbackend/suite_test.go new file mode 100644 index 00000000..c2bf080d --- /dev/null +++ b/pkg/controllers/hub/trafficmanagerbackend/suite_test.go @@ -0,0 +1,148 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package trafficmanagerbackend + +import ( + "context" + "flag" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/test/common/trafficmanager/fakeprovider" +) + +var ( + cfg *rest.Config + mgr manager.Manager + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc +) + +const ( + testNamespace = "backend-ns" +) + +var ( + originalGenerateAzureTrafficManagerProfileNameFunc = generateAzureTrafficManagerProfileNameFunc + originalGenerateAzureTrafficManagerEndpointNamePrefixFunc = generateAzureTrafficManagerEndpointNamePrefixFunc +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "TrafficManagerBackend Controller Suite") +} + +var _ = BeforeSuite(func() { + klog.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("../../../../", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = fleetnetv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + By("construct the k8s client") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + By("starting the controller manager") + klog.InitFlags(flag.CommandLine) + flag.Parse() + + mgr, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + }) + Expect(err).NotTo(HaveOccurred()) + + profileClient, err := fakeprovider.NewProfileClient("default-sub") + Expect(err).Should(Succeed(), "failed to create the fake profile client") + + endpointClient, err := fakeprovider.NewEndpointsClient("default-sub") + Expect(err).Should(Succeed(), "failed to create the fake endpoint client") + + generateAzureTrafficManagerProfileNameFunc = func(profile *fleetnetv1alpha1.TrafficManagerProfile) string { + return profile.Name + } + generateAzureTrafficManagerEndpointNamePrefixFunc = func(backend *fleetnetv1alpha1.TrafficManagerBackend) string { + return backend.Name + } + + ctx, cancel = context.WithCancel(context.TODO()) + err = (&Reconciler{ + Client: mgr.GetClient(), + ProfilesClient: profileClient, + EndpointsClient: endpointClient, + ResourceGroupName: fakeprovider.DefaultResourceGroupName, + }).SetupWithManager(ctx, mgr) + Expect(err).ToNot(HaveOccurred()) + + By("Create profile namespace") + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + } + Expect(k8sClient.Create(ctx, &ns)).Should(Succeed()) + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + defer klog.Flush() + + By("delete profile namespace") + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNamespace, + }, + } + Expect(k8sClient.Delete(ctx, &ns)).Should(Succeed()) + + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) + + generateAzureTrafficManagerProfileNameFunc = originalGenerateAzureTrafficManagerProfileNameFunc + generateAzureTrafficManagerEndpointNamePrefixFunc = originalGenerateAzureTrafficManagerEndpointNamePrefixFunc +}) diff --git a/pkg/controllers/hub/trafficmanagerprofile/controller.go b/pkg/controllers/hub/trafficmanagerprofile/controller.go index da64b53a..7d585a33 100644 --- a/pkg/controllers/hub/trafficmanagerprofile/controller.go +++ b/pkg/controllers/hub/trafficmanagerprofile/controller.go @@ -43,10 +43,15 @@ const ( var ( // create the func as a variable so that the integration test can use a customized function. generateAzureTrafficManagerProfileNameFunc = func(profile *fleetnetv1alpha1.TrafficManagerProfile) string { - return fmt.Sprintf(AzureResourceProfileNameFormat, profile.UID) + return GenerateAzureTrafficManagerProfileName(profile) } ) +// GenerateAzureTrafficManagerProfileName generates the Azure Traffic Manager profile name based on the profile. +func GenerateAzureTrafficManagerProfileName(profile *fleetnetv1alpha1.TrafficManagerProfile) string { + return fmt.Sprintf(AzureResourceProfileNameFormat, profile.UID) +} + // Reconciler reconciles a TrafficManagerProfile object. type Reconciler struct { client.Client @@ -117,11 +122,12 @@ func (r *Reconciler) handleDelete(ctx context.Context, profile *fleetnetv1alpha1 return ctrl.Result{}, err } } + klog.V(2).InfoS("Deleted Azure Traffic Manager profile", "trafficManagerProfile", profileKObj, "azureProfileName", azureProfileName) controllerutil.RemoveFinalizer(profile, objectmeta.TrafficManagerProfileFinalizer) if err := r.Client.Update(ctx, profile); err != nil { klog.ErrorS(err, "Failed to remove trafficManagerProfile finalizer", "trafficManagerProfile", profileKObj) - return ctrl.Result{}, err + return ctrl.Result{}, controller.NewUpdateIgnoreConflictError(err) } klog.V(2).InfoS("Removed trafficManagerProfile finalizer", "trafficManagerProfile", profileKObj) return ctrl.Result{}, nil @@ -157,6 +163,7 @@ func (r *Reconciler) handleUpdate(ctx context.Context, profile *fleetnetv1alpha1 "azureProfileName", azureProfileName, "errorCode", responseError.ErrorCode, "statusCode", responseError.StatusCode) } + klog.V(2).InfoS("Created or updated Azure Traffic Manager Profile", "trafficManagerProfile", profileKObj, "azureProfileName", azureProfileName) return r.updateProfileStatus(ctx, profile, res.Profile, updateErr) } diff --git a/pkg/controllers/hub/trafficmanagerprofile/controller_test.go b/pkg/controllers/hub/trafficmanagerprofile/controller_test.go index bec9ebcb..27408fce 100644 --- a/pkg/controllers/hub/trafficmanagerprofile/controller_test.go +++ b/pkg/controllers/hub/trafficmanagerprofile/controller_test.go @@ -10,11 +10,25 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager" "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" ) +func TestGenerateAzureTrafficManagerProfileName(t *testing.T) { + profile := &fleetnetv1alpha1.TrafficManagerProfile{ + ObjectMeta: metav1.ObjectMeta{ + UID: "abc", + }, + } + want := "fleet-abc" + got := GenerateAzureTrafficManagerProfileName(profile) + if want != got { + t.Errorf("GenerateAzureTrafficManagerProfileName() = %s, want %s", got, want) + } +} + func TestConvertToTrafficManagerProfileSpec(t *testing.T) { tests := []struct { name string diff --git a/pkg/controllers/hub/trafficmanagerprofile/suite_test.go b/pkg/controllers/hub/trafficmanagerprofile/suite_test.go index d9a9d68a..20db6ca5 100644 --- a/pkg/controllers/hub/trafficmanagerprofile/suite_test.go +++ b/pkg/controllers/hub/trafficmanagerprofile/suite_test.go @@ -89,7 +89,7 @@ var _ = BeforeSuite(func() { }) Expect(err).NotTo(HaveOccurred()) - profileClient, err := fakeprovider.NewProfileClient(ctx, "default-sub") + profileClient, err := fakeprovider.NewProfileClient("default-sub") Expect(err).Should(Succeed(), "failed to create the fake profile client") generateAzureTrafficManagerProfileNameFunc = func(profile *fleetnetv1alpha1.TrafficManagerProfile) string { diff --git a/test/common/trafficmanager/fakeprovider/endpoint.go b/test/common/trafficmanager/fakeprovider/endpoint.go new file mode 100644 index 00000000..1ff42aca --- /dev/null +++ b/test/common/trafficmanager/fakeprovider/endpoint.go @@ -0,0 +1,55 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package fakeprovider + +import ( + "context" + "net/http" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + azcorefake "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/trafficmanager/armtrafficmanager/fake" +) + +// NewEndpointsClient creates a client which talks to a fake endpoint server. +func NewEndpointsClient(subscriptionID string) (*armtrafficmanager.EndpointsClient, error) { + fakeServer := fake.EndpointsServer{ + Delete: EndpointDelete, + } + clientFactory, err := armtrafficmanager.NewClientFactory(subscriptionID, &azcorefake.TokenCredential{}, + &arm.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: fake.NewEndpointsServerTransport(&fakeServer), + }, + }) + if err != nil { + return nil, err + } + return clientFactory.NewEndpointsClient(), nil +} + +// EndpointDelete returns the http status code based on the profileName and endpointName. +func EndpointDelete(_ context.Context, resourceGroupName string, profileName string, endpointType armtrafficmanager.EndpointType, endpointName string, _ *armtrafficmanager.EndpointsClientDeleteOptions) (resp azcorefake.Responder[armtrafficmanager.EndpointsClientDeleteResponse], errResp azcorefake.ErrorResponder) { + if resourceGroupName != DefaultResourceGroupName { + errResp.SetResponseError(http.StatusNotFound, "ResourceGroupNotFound") + return resp, errResp + } + if strings.HasPrefix(profileName, ValidProfileName) && endpointType == armtrafficmanager.EndpointTypeAzureEndpoints && strings.HasPrefix(strings.ToLower(endpointName), ValidBackendName+"#") { + endpointResp := armtrafficmanager.EndpointsClientDeleteResponse{} + resp.SetResponse(http.StatusOK, endpointResp, nil) + } else { + if endpointType != armtrafficmanager.EndpointTypeAzureEndpoints { + // controller should not send other endpoint types. + errResp.SetResponseError(http.StatusBadRequest, "InvalidEndpointType") + } else { + errResp.SetResponseError(http.StatusNotFound, "NotFound") + } + } + return resp, errResp +} diff --git a/test/common/trafficmanager/fakeprovider/profile.go b/test/common/trafficmanager/fakeprovider/profile.go index 4a2a8b2b..de74cb0e 100644 --- a/test/common/trafficmanager/fakeprovider/profile.go +++ b/test/common/trafficmanager/fakeprovider/profile.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "net/http" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" @@ -23,20 +24,29 @@ import ( const ( DefaultResourceGroupName = "default-resource-group-name" - ValidProfileName = "valid-profile" - ConflictErrProfileName = "conflict-err-profile" - InternalServerErrProfileName = "internal-server-err-profile" - ThrottledErrProfileName = "throttled-err-profile" + ValidProfileName = "valid-profile" + ValidProfileWithEndpointsName = "valid-profile-with-endpoints" + ConflictErrProfileName = "conflict-err-profile" + InternalServerErrProfileName = "internal-server-err-profile" + ThrottledErrProfileName = "throttled-err-profile" + + ValidBackendName = "valid-backend" + ServiceImportName = "test-import" + ClusterName = "member-1" ProfileDNSNameFormat = "%s.trafficmanager.net" ) +var ( + ValidEndpointName = fmt.Sprintf("%s#%s#%s", ValidBackendName, ServiceImportName, ClusterName) +) + // NewProfileClient creates a client which talks to a fake profile server. -func NewProfileClient(_ context.Context, subscriptionID string) (*armtrafficmanager.ProfilesClient, error) { +func NewProfileClient(subscriptionID string) (*armtrafficmanager.ProfilesClient, error) { fakeServer := fake.ProfilesServer{ - CreateOrUpdate: CreateOrUpdate, - Delete: Delete, - Get: Get, + CreateOrUpdate: ProfileCreateOrUpdate, + Delete: ProfileDelete, + Get: ProfileGet, } clientFactory, err := armtrafficmanager.NewClientFactory(subscriptionID, &azcorefake.TokenCredential{}, &arm.ClientOptions{ @@ -50,14 +60,14 @@ func NewProfileClient(_ context.Context, subscriptionID string) (*armtrafficmana return clientFactory.NewProfilesClient(), nil } -// Get returns the http status code based on the profileName. -func Get(_ context.Context, resourceGroupName string, profileName string, _ *armtrafficmanager.ProfilesClientGetOptions) (resp azcorefake.Responder[armtrafficmanager.ProfilesClientGetResponse], errResp azcorefake.ErrorResponder) { +// ProfileGet returns the http status code based on the profileName. +func ProfileGet(_ context.Context, resourceGroupName string, profileName string, _ *armtrafficmanager.ProfilesClientGetOptions) (resp azcorefake.Responder[armtrafficmanager.ProfilesClientGetResponse], errResp azcorefake.ErrorResponder) { if resourceGroupName != DefaultResourceGroupName { errResp.SetResponseError(http.StatusNotFound, "ResourceGroupNotFound") return resp, errResp } switch profileName { - case ValidProfileName: + case ValidProfileName, ValidProfileWithEndpointsName: profileResp := armtrafficmanager.ProfilesClientGetResponse{ Profile: armtrafficmanager.Profile{ Name: ptr.To(profileName), @@ -82,6 +92,16 @@ func Get(_ context.Context, resourceGroupName string, profileName string, _ *arm TrafficViewEnrollmentStatus: ptr.To(armtrafficmanager.TrafficViewEnrollmentStatusDisabled), }, }} + if profileName == ValidProfileWithEndpointsName { + profileResp.Profile.Properties.Endpoints = []*armtrafficmanager.Endpoint{ + { + Name: ptr.To(strings.ToUpper(ValidEndpointName)), // test case-insensitive + }, + { + Name: ptr.To("other-endpoint"), + }, + } + } resp.SetResponse(http.StatusOK, profileResp, nil) default: errResp.SetResponseError(http.StatusNotFound, "NotFoundError") @@ -89,8 +109,8 @@ func Get(_ context.Context, resourceGroupName string, profileName string, _ *arm return resp, errResp } -// CreateOrUpdate returns the http status code based on the profileName. -func CreateOrUpdate(_ context.Context, resourceGroupName string, profileName string, parameters armtrafficmanager.Profile, _ *armtrafficmanager.ProfilesClientCreateOrUpdateOptions) (resp azcorefake.Responder[armtrafficmanager.ProfilesClientCreateOrUpdateResponse], errResp azcorefake.ErrorResponder) { +// ProfileCreateOrUpdate returns the http status code based on the profileName. +func ProfileCreateOrUpdate(_ context.Context, resourceGroupName string, profileName string, parameters armtrafficmanager.Profile, _ *armtrafficmanager.ProfilesClientCreateOrUpdateOptions) (resp azcorefake.Responder[armtrafficmanager.ProfilesClientCreateOrUpdateResponse], errResp azcorefake.ErrorResponder) { if resourceGroupName != DefaultResourceGroupName { errResp.SetResponseError(http.StatusNotFound, "ResourceGroupNotFound") return resp, errResp @@ -133,8 +153,8 @@ func CreateOrUpdate(_ context.Context, resourceGroupName string, profileName str return resp, errResp } -// Delete returns the http status code based on the profileName. -func Delete(_ context.Context, resourceGroupName string, profileName string, _ *armtrafficmanager.ProfilesClientDeleteOptions) (resp azcorefake.Responder[armtrafficmanager.ProfilesClientDeleteResponse], errResp azcorefake.ErrorResponder) { +// ProfileDelete returns the http status code based on the profileName. +func ProfileDelete(_ context.Context, resourceGroupName string, profileName string, _ *armtrafficmanager.ProfilesClientDeleteOptions) (resp azcorefake.Responder[armtrafficmanager.ProfilesClientDeleteResponse], errResp azcorefake.ErrorResponder) { if resourceGroupName != DefaultResourceGroupName { errResp.SetResponseError(http.StatusNotFound, "ResourceGroupNotFound") return resp, errResp diff --git a/test/common/trafficmanager/validator/backend.go b/test/common/trafficmanager/validator/backend.go new file mode 100644 index 00000000..a36fd33a --- /dev/null +++ b/test/common/trafficmanager/validator/backend.go @@ -0,0 +1,45 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package validator + +import ( + "context" + "fmt" + + "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1" + "go.goms.io/fleet-networking/pkg/common/objectmeta" +) + +// IsTrafficManagerBackendFinalizerAdded validates whether the backend is created with the finalizer or not. +func IsTrafficManagerBackendFinalizerAdded(ctx context.Context, k8sClient client.Client, name types.NamespacedName) { + gomega.Eventually(func() error { + backend := &fleetnetv1alpha1.TrafficManagerBackend{} + if err := k8sClient.Get(ctx, name, backend); err != nil { + return fmt.Errorf("failed to get trafficManagerBackend %s: %w", name, err) + } + if !controllerutil.ContainsFinalizer(backend, objectmeta.TrafficManagerBackendFinalizer) { + return fmt.Errorf("trafficManagerBackend %s finalizer not added", name) + } + return nil + }, timeout, interval).Should(gomega.Succeed(), "Failed to add finalizer to trafficManagerBackend %s", name) +} + +// IsTrafficManagerBackendDeleted validates whether the backend is deleted or not. +func IsTrafficManagerBackendDeleted(ctx context.Context, k8sClient client.Client, name types.NamespacedName) { + gomega.Eventually(func() error { + backend := &fleetnetv1alpha1.TrafficManagerBackend{} + if err := k8sClient.Get(ctx, name, backend); !errors.IsNotFound(err) { + return fmt.Errorf("trafficManagerBackend %s still exists or an unexpected error occurred: %w", name, err) + } + return nil + }, timeout, interval).Should(gomega.Succeed(), "Failed to remove trafficManagerBackend %s ", name) +}