From 417205d211d4bebfc461490df8426d3e1456f18f Mon Sep 17 00:00:00 2001 From: Adrian Stobbe Date: Tue, 1 Aug 2023 09:23:47 +0200 Subject: [PATCH] fixup! clean helm code from bootstrapper --- cli/internal/helm/BUILD.bazel | 11 +- cli/internal/helm/ciliumhelper.go | 72 +++++++++ cli/internal/helm/helm.go | 22 +++ .../helm/{release_test.go => helm_test.go} | 2 +- cli/internal/helm/helminstaller.go | 138 ------------------ cli/internal/helm/install.go | 2 +- cli/internal/helm/kubernetes_custom.go | 104 ------------- cli/internal/helm/release.go | 22 --- .../helm/{setup.go => suite_install.go} | 121 ++++++++++++++- cli/internal/helm/{client.go => upgrade.go} | 2 +- .../helm/{client_test.go => upgrade_test.go} | 0 11 files changed, 221 insertions(+), 275 deletions(-) create mode 100644 cli/internal/helm/ciliumhelper.go rename cli/internal/helm/{release_test.go => helm_test.go} (97%) delete mode 100644 cli/internal/helm/helminstaller.go delete mode 100644 cli/internal/helm/kubernetes_custom.go rename cli/internal/helm/{setup.go => suite_install.go} (54%) rename cli/internal/helm/{client.go => upgrade.go} (99%) rename cli/internal/helm/{client_test.go => upgrade_test.go} (100%) diff --git a/cli/internal/helm/BUILD.bazel b/cli/internal/helm/BUILD.bazel index a52e1accf5..60bcf7be3c 100644 --- a/cli/internal/helm/BUILD.bazel +++ b/cli/internal/helm/BUILD.bazel @@ -5,15 +5,14 @@ go_library( name = "helm", srcs = [ "backup.go", - "client.go", + "ciliumhelper.go", "helm.go", - "helminstaller.go", "install.go", - "kubernetes_custom.go", "loader.go", "release.go", "serviceversion.go", - "setup.go", + "suite_install.go", + "upgrade.go", "values.go", ], embedsrcs = [ @@ -460,9 +459,9 @@ go_test( name = "helm_test", srcs = [ "backup_test.go", - "client_test.go", + "helm_test.go", "loader_test.go", - "release_test.go", + "upgrade_test.go", ], data = glob(["testdata/**"]), embed = [":helm"], diff --git a/cli/internal/helm/ciliumhelper.go b/cli/internal/helm/ciliumhelper.go new file mode 100644 index 0000000000..3997a3e908 --- /dev/null +++ b/cli/internal/helm/ciliumhelper.go @@ -0,0 +1,72 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package helm + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type k8sDsClient struct { + clientset *kubernetes.Clientset +} + +func newK8sHelmHelper(kubeconfigPath string) (*k8sDsClient, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + return &k8sDsClient{clientset: clientset}, nil +} + +// WaitForDS waits for a DaemonSet to become ready. +func (h *k8sDsClient) WaitForDS(ctx context.Context, namespace, name string, log debugLog) error { + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context expired before DaemonSet %q became ready", name) + default: + ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, v1.GetOptions{}) + if err != nil { + return err + } + + if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled { + log.Debugf("DaemonSet %s is ready\n", name) + return nil + } + + log.Debugf("Waiting for DaemonSet %s to become ready...\n", name) + time.Sleep(10 * time.Second) + } + } +} + +// RestartDS restarts all pods of a DaemonSet by updating its template. +func (h *k8sDsClient) RestartDS(namespace, name string) error { + ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(context.Background(), name, v1.GetOptions{}) + if err != nil { + return err + } + + ds.Spec.Template.ObjectMeta.Annotations["restartTimestamp"] = fmt.Sprintf("%d", time.Now().Unix()) + _, err = h.clientset.AppsV1().DaemonSets(namespace).Update(context.Background(), ds, v1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} diff --git a/cli/internal/helm/helm.go b/cli/internal/helm/helm.go index dd64875402..51ec7fcffc 100644 --- a/cli/internal/helm/helm.go +++ b/cli/internal/helm/helm.go @@ -15,3 +15,25 @@ It is used by the CLI to: - create local backups before running service upgrades */ package helm + +// mergeMaps returns a new map that is the merger of it's inputs. +// Key collisions are resolved by taking the value of the second argument (map b). +// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108. +func mergeMaps(a, b map[string]any) map[string]any { + out := make(map[string]any, len(a)) + for k, v := range a { + out[k] = v + } + for k, v := range b { + if v, ok := v.(map[string]any); ok { + if bv, ok := out[k]; ok { + if bv, ok := bv.(map[string]any); ok { + out[k] = mergeMaps(bv, v) + continue + } + } + } + out[k] = v + } + return out +} diff --git a/cli/internal/helm/release_test.go b/cli/internal/helm/helm_test.go similarity index 97% rename from cli/internal/helm/release_test.go rename to cli/internal/helm/helm_test.go index 78576d5a55..34164ec542 100644 --- a/cli/internal/helm/release_test.go +++ b/cli/internal/helm/helm_test.go @@ -102,7 +102,7 @@ func TestMergeMaps(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { assert := assert.New(t) - newVals := MergeMaps(tc.vals, tc.extraVals) + newVals := mergeMaps(tc.vals, tc.extraVals) assert.Equal(tc.expected, newVals) }) } diff --git a/cli/internal/helm/helminstaller.go b/cli/internal/helm/helminstaller.go deleted file mode 100644 index fb8cd1d902..0000000000 --- a/cli/internal/helm/helminstaller.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package helm - -import ( - "context" - "fmt" - "time" - - "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" - "github.com/edgelesssys/constellation/v2/cli/internal/terraform" - "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" - "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" - "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/kms/uri" -) - -// SuiteInstaller installs all Helm charts required for a constellation cluster. -type SuiteInstaller interface { - Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, - idFile clusterid.File, - serviceAccURI string, releases *Releases, - ) error -} - -type helmInstallationClient struct { - log debugLog - installer helmInstaller -} - -// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster. -func NewInstallationClient(log debugLog) (SuiteInstaller, error) { - installer, err := NewInstaller(constants.AdminConfFilename, log) - if err != nil { - return nil, fmt.Errorf("creating Helm installer: %w", err) - } - return &helmInstallationClient{log: log, installer: installer}, nil -} - -func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, - idFile clusterid.File, - serviceAccURI string, releases *Releases, -) error { - tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) - if err != nil { - return fmt.Errorf("creating Terraform client: %w", err) - } - output, err := tfClient.ShowCluster(ctx, provider) - if err != nil { - return fmt.Errorf("getting Terraform output: %w", err) - } - - helper, err := newK8sHelmHelper(constants.AdminConfFilename) - if err != nil { - return fmt.Errorf("creating Kubernetes client: %w", err) - } - ciliumVals := setupCiliumVals(ctx, provider, helper, output) - //if err != nil { - // return fmt.Errorf("setting up Cilium values: %w", err) - //} - fmt.Println("ciliumVals: ", ciliumVals) - if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil { - return fmt.Errorf("installing Cilium: %w", err) - } - h.log.Debugf("Waiting for Cilium to become healthy") - timeToStartWaiting := time.Now() - // TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once - // saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN). - if err := helper.WaitForDS(ctx, "kube-system", "cilium", h.log); err != nil { - return fmt.Errorf("waiting for Cilium to become healthy: %w", err) - } - timeUntilFinishedWaiting := time.Since(timeToStartWaiting) - h.log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String()) - - h.log.Debugf("Restarting Cilium") - if err := helper.RestartDS("kube-system", "cilium"); err != nil { - return fmt.Errorf("restarting Cilium: %w", err) - } - - h.log.Debugf("Installing microservices") - serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output) - if err != nil { - return fmt.Errorf("setting up microservice values: %w", err) - } - if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil { - return fmt.Errorf("installing microservices: %w", err) - } - - h.log.Debugf("Installing cert-manager") - if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil { - return fmt.Errorf("installing cert-manager: %w", err) - } - - if releases.CSI != nil { - var csiVals map[string]any - if provider == cloudprovider.OpenStack { - creds, err := openstack.AccountKeyFromURI(serviceAccURI) - if err != nil { - return err - } - cinderIni := creds.CloudINI().CinderCSIConfiguration() - csiVals = map[string]any{ - "cinder-config": map[string]any{ - "secretData": cinderIni, - }, - } - } - - h.log.Debugf("Installing CSI deployments") - if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil { - return fmt.Errorf("installing CSI snapshot CRDs: %w", err) - } - } - - if releases.AWSLoadBalancerController != nil { - h.log.Debugf("Installing AWS Load Balancer Controller") - if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil { - return fmt.Errorf("installing AWS Load Balancer Controller: %w", err) - } - } - - h.log.Debugf("Installing constellation operators") - operatorVals := setupOperatorVals(ctx, idFile.UID) - if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil { - return fmt.Errorf("installing constellation operators: %w", err) - } - - // TODO(elchead): AB#3301 do cilium after version upgrade - return nil -} - -type helmInstaller interface { - InstallChart(context.Context, Release) error - InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error -} diff --git a/cli/internal/helm/install.go b/cli/internal/helm/install.go index 6e0558a264..14b91dadf3 100644 --- a/cli/internal/helm/install.go +++ b/cli/internal/helm/install.go @@ -68,7 +68,7 @@ func (h *Installer) InstallChart(ctx context.Context, release Release) error { // InstallChartWithValues is the generic install function for helm charts with custom values. func (h *Installer) InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error { - mergedVals := MergeMaps(release.Values, extraValues) + mergedVals := mergeMaps(release.Values, extraValues) h.ReleaseName = release.ReleaseName if err := h.SetWaitMode(release.WaitMode); err != nil { return err diff --git a/cli/internal/helm/kubernetes_custom.go b/cli/internal/helm/kubernetes_custom.go deleted file mode 100644 index 40627274d5..0000000000 --- a/cli/internal/helm/kubernetes_custom.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package helm - -import ( - "context" - "fmt" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -type k8sHelmHelper struct { - clientset *kubernetes.Clientset -} - -func newK8sHelmHelper(kubeconfigPath string) (*k8sHelmHelper, error) { - config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) - if err != nil { - return nil, err - } - - // Create a Kubernetes clientset - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, err - } - return &k8sHelmHelper{clientset: clientset}, nil -} - -// TODO(elchead): do not seem to need it anymore.. -// func (h *K8sHelmHelper) PatchNode(ctx context.Context, podCIDR string) error { -// selector := labels.Set{"node-role.kubernetes.io/control-plane": ""}.AsSelector() -// controlPlaneList, err := h.clientset.CoreV1().Nodes().List(ctx, v1.ListOptions{LabelSelector: selector.String()}) -// if err != nil { -// return err -// } -// if len(controlPlaneList.Items) != 1 { -// return fmt.Errorf("expected 1 control-plane node, got %d", len(controlPlaneList.Items)) -// } -// nodeName := controlPlaneList.Items[0].Name -// // Get the current node -// node, err := h.clientset.CoreV1().Nodes().Get(context.Background(), nodeName, v1.GetOptions{}) -// if err != nil { -// if errors.IsNotFound(err) { -// return fmt.Errorf("node %s not found", nodeName) -// } -// return err -// } -// fmt.Print("node.Spec.PodCIDR: ", node.Spec.PodCIDR) - -// // Update the node's spec -// //node.Spec.PodCIDR = podCIDR -// //_, err = h.clientset.CoreV1().Nodes().Patch(context.Background(), nodeName, types.MergePatchType, (types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"podCIDR":"%s"}}`, podCIDR)))) -// //if err != nil { -// // return err -// //} -// return nil -//} - -// WaitForDS waits for a DaemonSet to become ready. -func (h *k8sHelmHelper) WaitForDS(ctx context.Context, namespace, name string, log debugLog) error { - for { - select { - case <-ctx.Done(): - return fmt.Errorf("context expired before DaemonSet %q became ready", name) - default: - ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(ctx, name, v1.GetOptions{}) - if err != nil { - return err - } - - if ds.Status.NumberReady == ds.Status.DesiredNumberScheduled { - log.Debugf("DaemonSet %s is ready\n", name) - return nil - } - - log.Debugf("Waiting for DaemonSet %s to become ready...\n", name) - time.Sleep(10 * time.Second) - } - } -} - -// RestartDS restarts all pods of a DaemonSet by updating its template. -func (h *k8sHelmHelper) RestartDS(namespace, name string) error { - ds, err := h.clientset.AppsV1().DaemonSets(namespace).Get(context.Background(), name, v1.GetOptions{}) - if err != nil { - return err - } - - ds.Spec.Template.ObjectMeta.Annotations["restartTimestamp"] = fmt.Sprintf("%d", time.Now().Unix()) - _, err = h.clientset.AppsV1().DaemonSets(namespace).Update(context.Background(), ds, v1.UpdateOptions{}) - if err != nil { - return err - } - - return nil -} diff --git a/cli/internal/helm/release.go b/cli/internal/helm/release.go index dc2a09f38e..ccbe7646d4 100644 --- a/cli/internal/helm/release.go +++ b/cli/internal/helm/release.go @@ -25,28 +25,6 @@ type Releases struct { ConstellationServices Release } -// MergeMaps returns a new map that is the merger of it's inputs. -// Key collisions are resolved by taking the value of the second argument (map b). -// Taken from: https://github.com/helm/helm/blob/dbc6d8e20fe1d58d50e6ed30f09a04a77e4c68db/pkg/cli/values/options.go#L91-L108. -func MergeMaps(a, b map[string]any) map[string]any { - out := make(map[string]any, len(a)) - for k, v := range a { - out[k] = v - } - for k, v := range b { - if v, ok := v.(map[string]any); ok { - if bv, ok := out[k]; ok { - if bv, ok := bv.(map[string]any); ok { - out[k] = MergeMaps(bv, v) - continue - } - } - } - out[k] = v - } - return out -} - // WaitMode specifies the wait mode for a helm release. type WaitMode string diff --git a/cli/internal/helm/setup.go b/cli/internal/helm/suite_install.go similarity index 54% rename from cli/internal/helm/setup.go rename to cli/internal/helm/suite_install.go index 7ee95c56cf..0c6dd7459d 100644 --- a/cli/internal/helm/setup.go +++ b/cli/internal/helm/suite_install.go @@ -1,6 +1,5 @@ /* Copyright (c) Edgeless Systems GmbH - SPDX-License-Identifier: AGPL-3.0-only */ @@ -11,18 +10,136 @@ import ( "encoding/base64" "encoding/json" "fmt" + "time" + "github.com/edgelesssys/constellation/v2/cli/internal/clusterid" "github.com/edgelesssys/constellation/v2/cli/internal/terraform" "github.com/edgelesssys/constellation/v2/internal/cloud/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/gcpshared" + "github.com/edgelesssys/constellation/v2/internal/cloud/openstack" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/kms/uri" ) +// SuiteInstaller installs all Helm charts required for a constellation cluster. +type SuiteInstaller interface { + Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, + idFile clusterid.File, + serviceAccURI string, releases *Releases, + ) error +} + +type helmInstallationClient struct { + log debugLog + installer helmInstaller +} + +// NewInstallationClient creates a new Helm installation client to install all Helm charts required for a constellation cluster. +func NewInstallationClient(log debugLog) (SuiteInstaller, error) { + installer, err := NewInstaller(constants.AdminConfFilename, log) + if err != nil { + return nil, fmt.Errorf("creating Helm installer: %w", err) + } + return &helmInstallationClient{log: log, installer: installer}, nil +} + +func (h helmInstallationClient) Install(ctx context.Context, provider cloudprovider.Provider, masterSecret uri.MasterSecret, + idFile clusterid.File, + serviceAccURI string, releases *Releases, +) error { + tfClient, err := terraform.New(ctx, constants.TerraformWorkingDir) + if err != nil { + return fmt.Errorf("creating Terraform client: %w", err) + } + output, err := tfClient.ShowCluster(ctx, provider) + if err != nil { + return fmt.Errorf("getting Terraform output: %w", err) + } + + helper, err := newK8sHelmHelper(constants.AdminConfFilename) + if err != nil { + return fmt.Errorf("creating Kubernetes client: %w", err) + } + ciliumVals := setupCiliumVals(ctx, provider, helper, output) + if err := h.installer.InstallChartWithValues(ctx, releases.Cilium, ciliumVals); err != nil { + return fmt.Errorf("installing Cilium: %w", err) + } + h.log.Debugf("Waiting for Cilium to become healthy") + timeToStartWaiting := time.Now() + // TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once + // saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN). + if err := helper.WaitForDS(ctx, "kube-system", "cilium", h.log); err != nil { + return fmt.Errorf("waiting for Cilium to become healthy: %w", err) + } + timeUntilFinishedWaiting := time.Since(timeToStartWaiting) + h.log.Debugf("Cilium became healthy after %s", timeUntilFinishedWaiting.String()) + + h.log.Debugf("Restarting Cilium") + if err := helper.RestartDS("kube-system", "cilium"); err != nil { + return fmt.Errorf("restarting Cilium: %w", err) + } + + h.log.Debugf("Installing microservices") + serviceVals, err := setupMicroserviceVals(provider, masterSecret.Salt, idFile.UID, serviceAccURI, output) + if err != nil { + return fmt.Errorf("setting up microservice values: %w", err) + } + if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationServices, serviceVals); err != nil { + return fmt.Errorf("installing microservices: %w", err) + } + + h.log.Debugf("Installing cert-manager") + if err := h.installer.InstallChart(ctx, releases.CertManager); err != nil { + return fmt.Errorf("installing cert-manager: %w", err) + } + + if releases.CSI != nil { + var csiVals map[string]any + if provider == cloudprovider.OpenStack { + creds, err := openstack.AccountKeyFromURI(serviceAccURI) + if err != nil { + return err + } + cinderIni := creds.CloudINI().CinderCSIConfiguration() + csiVals = map[string]any{ + "cinder-config": map[string]any{ + "secretData": cinderIni, + }, + } + } + + h.log.Debugf("Installing CSI deployments") + if err := h.installer.InstallChartWithValues(ctx, *releases.CSI, csiVals); err != nil { + return fmt.Errorf("installing CSI snapshot CRDs: %w", err) + } + } + + if releases.AWSLoadBalancerController != nil { + h.log.Debugf("Installing AWS Load Balancer Controller") + if err := h.installer.InstallChart(ctx, *releases.AWSLoadBalancerController); err != nil { + return fmt.Errorf("installing AWS Load Balancer Controller: %w", err) + } + } + + h.log.Debugf("Installing constellation operators") + operatorVals := setupOperatorVals(ctx, idFile.UID) + if err := h.installer.InstallChartWithValues(ctx, releases.ConstellationOperators, operatorVals); err != nil { + return fmt.Errorf("installing constellation operators: %w", err) + } + return nil +} + +type helmInstaller interface { + InstallChart(context.Context, Release) error + InstallChartWithValues(ctx context.Context, release Release, extraValues map[string]any) error +} + // lb_port: everywhere hardcoded in TF 6443, for host use ip // TODO(malt3): switch over to DNS name on AWS and Azure // soon as every apiserver certificate of every control-plane node // has the dns endpoint in its SAN list. -func setupCiliumVals(_ context.Context, provider cloudprovider.Provider, _ *k8sHelmHelper, output terraform.ApplyOutput) map[string]any { +func setupCiliumVals(_ context.Context, provider cloudprovider.Provider, _ *k8sDsClient, output terraform.ApplyOutput) map[string]any { vals := map[string]any{ "k8sServiceHost": output.IP, "k8sServicePort": 6443, // TODO take from tf? diff --git a/cli/internal/helm/client.go b/cli/internal/helm/upgrade.go similarity index 99% rename from cli/internal/helm/client.go rename to cli/internal/helm/upgrade.go index c66c0c213c..3d8607658a 100644 --- a/cli/internal/helm/client.go +++ b/cli/internal/helm/upgrade.go @@ -423,7 +423,7 @@ func (c *Client) mergeClusterValues(localValues map[string]any, releaseName stri return nil, fmt.Errorf("getting values for %s: %w", releaseName, err) } - return MergeMaps(clusterValues, localValues), nil + return mergeMaps(clusterValues, localValues), nil } // GetValues queries the cluster for the values of the given release. diff --git a/cli/internal/helm/client_test.go b/cli/internal/helm/upgrade_test.go similarity index 100% rename from cli/internal/helm/client_test.go rename to cli/internal/helm/upgrade_test.go