From afc4d49ad41c7ee9262e32cfb54a46197c83ed5c Mon Sep 17 00:00:00 2001 From: tharindu1st Date: Fri, 16 Feb 2024 10:20:23 +0530 Subject: [PATCH] fix httproute vhost issue --- .../internal/eventhub/dataloader.go | 34 ++++++-- .../internal/k8sClient/k8s_client.go | 77 +++++++++---------- apim-apk-agent/internal/mapper/cr_mapper.go | 18 ++--- apim-apk-agent/internal/utils/apis_fetcher.go | 26 ++++--- .../pkg/transformer/k8s_artifacts.go | 18 ++--- apim-apk-agent/pkg/transformer/transformer.go | 20 ++--- 6 files changed, 107 insertions(+), 86 deletions(-) diff --git a/apim-apk-agent/internal/eventhub/dataloader.go b/apim-apk-agent/internal/eventhub/dataloader.go index 95f4874a..43807297 100644 --- a/apim-apk-agent/internal/eventhub/dataloader.go +++ b/apim-apk-agent/internal/eventhub/dataloader.go @@ -27,15 +27,16 @@ import ( "time" loggers "github.com/sirupsen/logrus" + dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" "github.com/wso2/product-apim-tooling/apim-apk-agent/config" + internalk8sClient "github.com/wso2/product-apim-tooling/apim-apk-agent/internal/k8sClient" internalutils "github.com/wso2/product-apim-tooling/apim-apk-agent/internal/utils" - "sigs.k8s.io/controller-runtime/pkg/client" - pkgAuth "github.com/wso2/product-apim-tooling/apim-apk-agent/pkg/auth" "github.com/wso2/product-apim-tooling/apim-apk-agent/pkg/eventhub/types" logger "github.com/wso2/product-apim-tooling/apim-apk-agent/pkg/loggers" "github.com/wso2/product-apim-tooling/apim-apk-agent/pkg/tlsutils" "github.com/wso2/product-apim-tooling/apim-apk-agent/pkg/utils" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -50,8 +51,6 @@ const ( GatewayLabelParam string = "gatewayLabel" // APIUUIDParam is required to call /apis endpoint APIUUIDParam string = "apiId" - // ApisEndpoint is the resource path of /apis endpoint - ApisEndpoint string = "apis" ) var ( @@ -232,5 +231,30 @@ func retrieveDataFromResponseChannel(response response) { // FetchAPIsOnStartUp APIs from control plane during the server start up and push them // to the router and enforcer components. func FetchAPIsOnStartUp(conf *config.Config, k8sClient client.Client) { - internalutils.FetchAPIsOnEvent(conf, nil, k8sClient) + k8sAPIS, _, err := internalk8sClient.RetrieveAllAPISFromK8s(k8sClient, "") + if err != nil { + logger.LoggerSubscription.Errorf("Error occurred while fetching APIs from K8s %v", err) + } + apis, err := internalutils.FetchAPIsOnEvent(conf, nil, k8sClient) + if err != nil { + logger.LoggerSubscription.Errorf("Error occurred while fetching APIs from control plane %v", err) + } + removeApis := make([]dpv1alpha2.API, 0) + for _, k8sAPI := range k8sAPIS { + found := false + for _, api := range *apis { + if k8sAPI.Name == api { + found = true + break + } + } + if !found { + removeApis = append(removeApis, k8sAPI) + } + } + for _, removeAPI := range removeApis { + if !removeAPI.Spec.SystemAPI { + internalk8sClient.UndeployAPICR(removeAPI.Name, k8sClient) + } + } } diff --git a/apim-apk-agent/internal/k8sClient/k8s_client.go b/apim-apk-agent/internal/k8sClient/k8s_client.go index 4bf26b3a..e96a416c 100644 --- a/apim-apk-agent/internal/k8sClient/k8s_client.go +++ b/apim-apk-agent/internal/k8sClient/k8s_client.go @@ -25,13 +25,16 @@ import ( dpv1alpha1 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha1" dpv1alpha2 "github.com/wso2/apk/common-go-libs/apis/dp/v1alpha2" + "github.com/wso2/apk/common-go-libs/utils" "github.com/wso2/product-apim-tooling/apim-apk-agent/config" "github.com/wso2/product-apim-tooling/apim-apk-agent/internal/loggers" + "github.com/wso2/product-apim-tooling/apim-apk-agent/internal/logging" eventhubTypes "github.com/wso2/product-apim-tooling/apim-apk-agent/pkg/eventhub/types" corev1 "k8s.io/api/core/v1" k8error "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/gateway-api/apis/v1alpha2" gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -61,52 +64,19 @@ func DeployAPICR(api *dpv1alpha2.API, k8sClient client.Client) { // UndeployAPICR removes the API Custom Resource from the Kubernetes cluster based on API ID label. func UndeployAPICR(apiID string, k8sClient client.Client) { - // Define a list to hold API CRs with matching label - apiList := &dpv1alpha2.APIList{} - - // Retrieve all API CRs from the Kubernetes cluster - if err := k8sClient.List(context.Background(), apiList); err != nil { - loggers.LoggerXds.Errorf("Unable to list API CRs: %v", err) - } - - // Iterate over each API CR and check if it has the matching label - for _, api := range apiList.Items { - labels := api.GetLabels() - if labels["apiUUID"] == apiID { - // Found the API CR with the matching label, delete it - if err := k8sClient.Delete(context.Background(), &api); err != nil { - loggers.LoggerXds.Errorf("Unable to delete API CR: %v", err) - } - loggers.LoggerXds.Infof("Deleted API CR: %s", api.Name) - } + conf, errReadConfig := config.ReadConfigs() + if errReadConfig != nil { + loggers.LoggerXds.Errorf("Error reading configurations: %v", errReadConfig) } -} - -// UndeployAPICRs removes the API Custom Resources from the Kubernetes cluster -// that are not included in the provided apiUUID list. -func UndeployAPICRs(apiUUIDs []string, k8sClient client.Client) { - // Define a list to hold API CRs with matching label - apiList := &dpv1alpha2.APIList{} - + api := &dpv1alpha2.API{} // Retrieve all API CRs from the Kubernetes cluster - if err := k8sClient.List(context.Background(), apiList); err != nil { + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: apiID, Namespace: conf.DataPlane.Namespace}, api); err != nil { loggers.LoggerXds.Errorf("Unable to list API CRs: %v", err) } - - // Iterate over each API CR and check if it has a matching label - for _, api := range apiList.Items { - if !api.Spec.SystemAPI { - // Check if the API CR's UUID is not in the provided list - if !contains(apiUUIDs, api.GetLabels()["apiUUID"]) { - // Delete the API CR - if err := k8sClient.Delete(context.Background(), &api); err != nil { - loggers.LoggerXds.Errorf("Unable to delete API CR: %v", err) - } else { - loggers.LoggerXds.Infof("Deleted API CR: %s", api.Name) - } - } - } + if err := k8sClient.Delete(context.Background(), api); err != nil { + loggers.LoggerXds.Errorf("Unable to delete API CR: %v", err) } + loggers.LoggerXds.Infof("Deleted API CR: %s", api.Name) } // contains checks if a string is present in a slice of strings @@ -464,3 +434,28 @@ func getSha1Value(input string) string { hashBytes := hasher.Sum(nil) return hex.EncodeToString(hashBytes) } + +// RetrieveAllAPISFromK8s retrieves all the API CRs from the Kubernetes cluster +func RetrieveAllAPISFromK8s(k8sClient client.Client, nextToken string) ([]dpv1alpha2.API, string, error) { + apiList := dpv1alpha2.APIList{} + resolvedAPIList := make([]dpv1alpha2.API, 0) + var err error + if nextToken == "" { + err = k8sClient.List(context.Background(), &apiList, &client.ListOptions{Namespace: utils.GetOperatorPodNamespace()}) + } else { + err = k8sClient.List(context.Background(), &apiList, &client.ListOptions{Namespace: utils.GetOperatorPodNamespace(), Continue: nextToken}) + } + if err != nil { + loggers.LoggerSync.ErrorC(logging.PrintError(logging.Error1102, logging.CRITICAL, "Failed to get application from k8s %v", err.Error())) + return nil, "", err + } + resolvedAPIList = append(resolvedAPIList, apiList.Items...) + if apiList.Continue != "" { + tempAPIList, _, err := RetrieveAllAPISFromK8s(k8sClient, apiList.Continue) + if err != nil { + return nil, "", err + } + resolvedAPIList = append(resolvedAPIList, tempAPIList...) + } + return resolvedAPIList, apiList.Continue, nil +} diff --git a/apim-apk-agent/internal/mapper/cr_mapper.go b/apim-apk-agent/internal/mapper/cr_mapper.go index ee6f57f9..2486bee0 100644 --- a/apim-apk-agent/internal/mapper/cr_mapper.go +++ b/apim-apk-agent/internal/mapper/cr_mapper.go @@ -42,27 +42,27 @@ func MapAndCreateCR(k8sArtifact transformer.K8sArtifacts, k8sClient client.Clien internalk8sClient.DeployAPICR(&k8sArtifact.API, k8sClient) for _, apiPolicies := range k8sArtifact.APIPolicies { apiPolicies.Namespace = namespace - internalk8sClient.DeployAPIPolicyCR(&apiPolicies, k8sClient) + internalk8sClient.DeployAPIPolicyCR(apiPolicies, k8sClient) } for _, httpRoutes := range k8sArtifact.HTTPRoutes { httpRoutes.Namespace = namespace - internalk8sClient.DeployHTTPRouteCR(&httpRoutes, k8sClient) + internalk8sClient.DeployHTTPRouteCR(httpRoutes, k8sClient) } for _, backends := range k8sArtifact.Backends { backends.Namespace = namespace - internalk8sClient.DeployBackendCR(&backends, k8sClient) + internalk8sClient.DeployBackendCR(backends, k8sClient) } for _, configMaps := range k8sArtifact.ConfigMaps { configMaps.Namespace = namespace - internalk8sClient.DeployConfigMapCR(&configMaps, k8sClient) + internalk8sClient.DeployConfigMapCR(configMaps, k8sClient) } for _, authPolicies := range k8sArtifact.Authentication { authPolicies.Namespace = namespace - internalk8sClient.DeployAuthenticationCR(&authPolicies, k8sClient) + internalk8sClient.DeployAuthenticationCR(authPolicies, k8sClient) } for _, interceptorServices := range k8sArtifact.InterceptorServices { interceptorServices.Namespace = namespace - internalk8sClient.DeployInterceptorServicesCR(&interceptorServices, k8sClient) + internalk8sClient.DeployInterceptorServicesCR(interceptorServices, k8sClient) } if k8sArtifact.BackendJWT != nil { k8sArtifact.BackendJWT.Namespace = namespace @@ -70,15 +70,15 @@ func MapAndCreateCR(k8sArtifact transformer.K8sArtifacts, k8sClient client.Clien } for _, scopes := range k8sArtifact.Scopes { scopes.Namespace = namespace - internalk8sClient.DeployScopeCR(&scopes, k8sClient) + internalk8sClient.DeployScopeCR(scopes, k8sClient) } for _, rateLimitPolicy := range k8sArtifact.RateLimitPolicies { rateLimitPolicy.Namespace = namespace - internalk8sClient.DeployRateLimitPolicyCR(&rateLimitPolicy, k8sClient) + internalk8sClient.DeployRateLimitPolicyCR(rateLimitPolicy, k8sClient) } for _, secrets := range k8sArtifact.Secrets { secrets.Namespace = namespace - internalk8sClient.DeploySecretCR(&secrets, k8sClient) + internalk8sClient.DeploySecretCR(secrets, k8sClient) } return nil } diff --git a/apim-apk-agent/internal/utils/apis_fetcher.go b/apim-apk-agent/internal/utils/apis_fetcher.go index e86f027d..5dd3d096 100644 --- a/apim-apk-agent/internal/utils/apis_fetcher.go +++ b/apim-apk-agent/internal/utils/apis_fetcher.go @@ -54,8 +54,9 @@ func init() { } // FetchAPIsOnEvent will fetch API from control plane during the API Notification Event -func FetchAPIsOnEvent(conf *config.Config, apiUUID *string, k8sClient client.Client) { +func FetchAPIsOnEvent(conf *config.Config, apiUUID *string, k8sClient client.Client) (*[]string, error) { // Populate data from config. + apis := make([]string, 0) envs := conf.ControlPlane.EnvironmentLabels // Create a channel for the byte slice (response from the APIs from control plane) @@ -84,22 +85,22 @@ func FetchAPIsOnEvent(conf *config.Config, apiUUID *string, k8sClient client.Cli if err != nil { logger.LoggerSync.Errorf("Error while reading zip: %v", err) - return + return nil, err } deploymentJSON, exists := apiFiles["deployments.json"] if !exists { logger.LoggerSync.Errorf("deployments.json not found") - return + return nil, err } deploymentJSONBytes, err := transformer.ReadContent(deploymentJSON) if err != nil { logger.LoggerSync.Errorf("Error while decoding the API Project Artifact: %v", err) - return + return nil, err } deploymentDescriptor, err := transformer.ProcessDeploymentDescriptor(deploymentJSONBytes) if err != nil { logger.LoggerSync.Errorf("Error while decoding the API Project Artifact: %v", err) - return + return nil, err } apiDeployments := deploymentDescriptor.Data.Deployments if apiDeployments != nil { @@ -109,38 +110,38 @@ func FetchAPIsOnEvent(conf *config.Config, apiUUID *string, k8sClient client.Cli artifact, decodingError := transformer.DecodeAPIArtifact(apiZip) if decodingError != nil { logger.LoggerSync.Errorf("Error while decoding the API Project Artifact: %v", decodingError) - return + return nil, err } apkConf, apiUUID, revisionID, apkErr := transformer.GenerateAPKConf(artifact.APIJson, artifact.ClientCerts) if apkErr != nil { logger.LoggerSync.Errorf("Error while generating APK-Conf: %v", apkErr) - return + return nil, err } k8ResourceEndpoint := conf.DataPlane.K8ResourceEndpoint crResponse, err := transformer.GenerateCRs(apkConf, artifact.Swagger, k8ResourceEndpoint) if err != nil { logger.LoggerSync.Errorf("Error occured in receiving the updated CRDs: %v", err) - return + return nil, err } transformer.UpdateCRS(crResponse, apiDeployment.Environments, apiDeployment.OrganizationID, apiUUID, fmt.Sprint(revisionID), "namespace") mapperUtil.MapAndCreateCR(*crResponse, k8sClient) + apis = append(apis, apiUUID) logger.LoggerMsg.Info("API applied successfully.\n") } } + return &apis, nil } - } else if data.ErrorCode == 204 { logger.LoggerMsg.Infof("No API Artifacts are available in the control plane for the envionments :%s", strings.Join(envs, ", ")) - //health.SetControlPlaneRestAPIStatus(true) + return &[]string{}, nil } else if data.ErrorCode >= 400 && data.ErrorCode < 500 { logger.LoggerMsg.ErrorC(logging.ErrorDetails{ Message: fmt.Sprintf("Error occurred when retrieving APIs from control plane(unrecoverable error): %v", data.Err.Error()), Severity: logging.CRITICAL, ErrorCode: 1106, }) - //isNoAPIArtifacts := data.ErrorCode == 404 && strings.Contains(data.Err.Error(), "No Api artifacts found") - //health.SetControlPlaneRestAPIStatus(isNoAPIArtifacts) + return nil, data.Err } else { // Keep the iteration still until all the envrionment response properly. logger.LoggerMsg.ErrorC(logging.ErrorDetails{ @@ -152,6 +153,7 @@ func FetchAPIsOnEvent(conf *config.Config, apiUUID *string, k8sClient client.Cli sync.RetryFetchingAPIs(c, data, sync.RuntimeArtifactEndpoint, true) } logger.LoggerMsg.Info("Fetching API for an event is completed...") + return nil, nil } // GetAPI function calls the FetchAPIs() with relevant environment labels defined in the config. diff --git a/apim-apk-agent/pkg/transformer/k8s_artifacts.go b/apim-apk-agent/pkg/transformer/k8s_artifacts.go index 7b172053..39f9c24f 100644 --- a/apim-apk-agent/pkg/transformer/k8s_artifacts.go +++ b/apim-apk-agent/pkg/transformer/k8s_artifacts.go @@ -10,14 +10,14 @@ import ( // K8sArtifacts k8s artifact representation of API type K8sArtifacts struct { API dpv1alpha2.API - HTTPRoutes map[string]gwapiv1b1.HTTPRoute - Backends map[string]v1alpha1.Backend - Scopes map[string]v1alpha1.Scope - Authentication map[string]dpv1alpha2.Authentication - APIPolicies map[string]dpv1alpha2.APIPolicy - InterceptorServices map[string]v1alpha1.InterceptorService - ConfigMaps map[string]corev1.ConfigMap - Secrets map[string]corev1.Secret + HTTPRoutes map[string]*gwapiv1b1.HTTPRoute + Backends map[string]*v1alpha1.Backend + Scopes map[string]*v1alpha1.Scope + Authentication map[string]*dpv1alpha2.Authentication + APIPolicies map[string]*dpv1alpha2.APIPolicy + InterceptorServices map[string]*v1alpha1.InterceptorService + ConfigMaps map[string]*corev1.ConfigMap + Secrets map[string]*corev1.Secret BackendJWT *v1alpha1.BackendJWT - RateLimitPolicies map[string]v1alpha1.RateLimitPolicy + RateLimitPolicies map[string]*v1alpha1.RateLimitPolicy } diff --git a/apim-apk-agent/pkg/transformer/transformer.go b/apim-apk-agent/pkg/transformer/transformer.go index 17884ec9..21b7fbf4 100644 --- a/apim-apk-agent/pkg/transformer/transformer.go +++ b/apim-apk-agent/pkg/transformer/transformer.go @@ -238,7 +238,7 @@ func mapAuthConfigs(authHeader string, secSchemes []string, certAvailable bool, // GenerateCRs takes the .apk-conf, api definition, vHost and the organization for a particular API and then generate and returns // the relavant CRD set as a zip func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint string) (*K8sArtifacts, error) { - k8sArtifact := K8sArtifacts{HTTPRoutes: make(map[string]gwapiv1b1.HTTPRoute), Backends: make(map[string]dpv1alpha1.Backend), Scopes: make(map[string]dpv1alpha1.Scope), Authentication: make(map[string]dpv1alpha2.Authentication), APIPolicies: make(map[string]dpv1alpha2.APIPolicy), InterceptorServices: make(map[string]dpv1alpha1.InterceptorService), ConfigMaps: make(map[string]corev1.ConfigMap), Secrets: make(map[string]corev1.Secret), RateLimitPolicies: make(map[string]dpv1alpha1.RateLimitPolicy)} + k8sArtifact := K8sArtifacts{HTTPRoutes: make(map[string]*gwapiv1b1.HTTPRoute), Backends: make(map[string]*dpv1alpha1.Backend), Scopes: make(map[string]*dpv1alpha1.Scope), Authentication: make(map[string]*dpv1alpha2.Authentication), APIPolicies: make(map[string]*dpv1alpha2.APIPolicy), InterceptorServices: make(map[string]*dpv1alpha1.InterceptorService), ConfigMaps: make(map[string]*corev1.ConfigMap), Secrets: make(map[string]*corev1.Secret), RateLimitPolicies: make(map[string]*dpv1alpha1.RateLimitPolicy)} if apkConf == "" { logger.LoggerTransformer.Error("Empty apk-conf parameter provided. Unable to generate CRDs.") return nil, errors.New("Error: APK-Conf can't be empty") @@ -345,7 +345,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling APIPolicy YAML: %v", err) continue } - k8sArtifact.APIPolicies[apiPolicy.ObjectMeta.Name] = apiPolicy + k8sArtifact.APIPolicies[apiPolicy.ObjectMeta.Name] = &apiPolicy case "HTTPRoute": var httpRoute gwapiv1b1.HTTPRoute err = k8Yaml.Unmarshal(yamlData, &httpRoute) @@ -353,7 +353,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling HTTPRoute YAML: %v", err) continue } - k8sArtifact.HTTPRoutes[httpRoute.ObjectMeta.Name] = httpRoute + k8sArtifact.HTTPRoutes[httpRoute.ObjectMeta.Name] = &httpRoute case "Backend": var backend dpv1alpha1.Backend @@ -362,7 +362,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling Backend YAML: %v", err) continue } - k8sArtifact.Backends[backend.ObjectMeta.Name] = backend + k8sArtifact.Backends[backend.ObjectMeta.Name] = &backend case "ConfigMap": var configMap corev1.ConfigMap @@ -371,7 +371,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling ConfigMap YAML: %v", err) continue } - k8sArtifact.ConfigMaps[configMap.ObjectMeta.Name] = configMap + k8sArtifact.ConfigMaps[configMap.ObjectMeta.Name] = &configMap case "Authentication": var authPolicy dpv1alpha2.Authentication err = k8Yaml.Unmarshal(yamlData, &authPolicy) @@ -379,7 +379,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling Authentication YAML: %v", err) continue } - k8sArtifact.Authentication[authPolicy.ObjectMeta.Name] = authPolicy + k8sArtifact.Authentication[authPolicy.ObjectMeta.Name] = &authPolicy case "API": var api dpv1alpha2.API @@ -396,7 +396,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling InterceptorService YAML: %v", err) continue } - k8sArtifact.InterceptorServices[interceptorService.Name] = interceptorService + k8sArtifact.InterceptorServices[interceptorService.Name] = &interceptorService case "BackendJWT": var backendJWT *dpv1alpha1.BackendJWT err = k8Yaml.Unmarshal(yamlData, &backendJWT) @@ -412,7 +412,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling Scope YAML: %v", err) continue } - k8sArtifact.Scopes[scope.Name] = scope + k8sArtifact.Scopes[scope.Name] = &scope case "RateLimitPolicy": var rateLimitPolicy dpv1alpha1.RateLimitPolicy err = k8Yaml.Unmarshal(yamlData, &rateLimitPolicy) @@ -420,7 +420,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling RateLimitPolicy YAML: %v", err) continue } - k8sArtifact.RateLimitPolicies[rateLimitPolicy.Name] = rateLimitPolicy + k8sArtifact.RateLimitPolicies[rateLimitPolicy.Name] = &rateLimitPolicy case "Secret": var secret corev1.Secret err = k8Yaml.Unmarshal(yamlData, &secret) @@ -428,7 +428,7 @@ func GenerateCRs(apkConf string, apiDefinition string, k8ResourceGenEndpoint str logger.LoggerSync.Errorf("Error unmarshaling Secret YAML: %v", err) continue } - k8sArtifact.Secrets[secret.Name] = secret + k8sArtifact.Secrets[secret.Name] = &secret default: logger.LoggerSync.Errorf("[!]Unknown Kind parsed from the YAML File: %v", kind) }