Skip to content

Commit

Permalink
clean helm code from bootstrapper
Browse files Browse the repository at this point in the history
  • Loading branch information
elchead committed Aug 1, 2023
1 parent 3583dd8 commit eb7d476
Show file tree
Hide file tree
Showing 22 changed files with 44 additions and 175 deletions.
1 change: 0 additions & 1 deletion bootstrapper/cmd/bootstrapper/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ go_library(
"//internal/cloud/openstack",
"//internal/cloud/qemu",
"//internal/constants",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/dialer",
"//internal/kubernetes/kubectl",
Expand Down
16 changes: 5 additions & 11 deletions bootstrapper/cmd/bootstrapper/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
openstackcloud "github.com/edgelesssys/constellation/v2/internal/cloud/openstack"
qemucloud "github.com/edgelesssys/constellation/v2/internal/cloud/qemu"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/logger"
Expand Down Expand Up @@ -67,11 +66,6 @@ func main() {
var openDevice vtpm.TPMOpenFunc
var fs afero.Fs

helmClient, err := helm.NewInstaller(constants.ControlPlaneAdminConfFilename, log)
if err != nil {
log.With(zap.Error(err)).Fatalf("Helm client could not be initialized")
}

attestVariant, err := variant.FromString(os.Getenv(constants.AttestationVariant))
if err != nil {
log.With(zap.Error(err)).Fatalf("Failed to parse attestation variant")
Expand All @@ -96,7 +90,7 @@ func main() {

clusterInitJoiner = kubernetes.New(
"aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
openDevice = vtpm.OpenVTPM
fs = afero.NewOsFs()
Expand All @@ -116,7 +110,7 @@ func main() {
metadataAPI = metadata
clusterInitJoiner = kubernetes.New(
"gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
openDevice = vtpm.OpenVTPM
fs = afero.NewOsFs()
Expand All @@ -134,7 +128,7 @@ func main() {
metadataAPI = metadata
clusterInitJoiner = kubernetes.New(
"azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)

openDevice = vtpm.OpenVTPM
Expand All @@ -145,7 +139,7 @@ func main() {
metadata := qemucloud.New()
clusterInitJoiner = kubernetes.New(
"qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
metadataAPI = metadata

Expand All @@ -168,7 +162,7 @@ func main() {
}
clusterInitJoiner = kubernetes.New(
"openstack", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(),
metadata, helmClient, &kubewaiter.CloudKubeAPIWaiter{},
metadata, &kubewaiter.CloudKubeAPIWaiter{},
)
metadataAPI = metadata
openDevice = vtpm.OpenVTPM
Expand Down
2 changes: 1 addition & 1 deletion bootstrapper/cmd/bootstrapper/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger,
bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil
}
Expand Down
2 changes: 0 additions & 2 deletions bootstrapper/internal/initserver/initserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,6 @@ func (s *Server) Init(req *initproto.InitRequest, stream initproto.API_InitServe
kubeconfig, err := s.initializer.InitCluster(stream.Context(),
req.KubernetesVersion,
clusterName,
req.HelmDeployments,
req.ConformanceMode,
components.NewComponentsFromInitProto(req.KubernetesComponents),
req.ApiserverCertSans,
Expand Down Expand Up @@ -342,7 +341,6 @@ type ClusterInitializer interface {
ctx context.Context,
k8sVersion string,
clusterName string,
helmDeployments []byte,
conformanceMode bool,
kubernetesComponents components.Components,
apiServerCertSANs []string,
Expand Down
2 changes: 1 addition & 1 deletion bootstrapper/internal/initserver/initserver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ type stubClusterInitializer struct {

func (i *stubClusterInitializer) InitCluster(
context.Context, string, string,
[]byte, bool, components.Components, []string, *logger.Logger,
bool, components.Components, []string, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}
Expand Down
2 changes: 0 additions & 2 deletions bootstrapper/internal/kubernetes/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ go_library(
"//internal/cloud/cloudprovider",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",
Expand All @@ -37,7 +36,6 @@ go_test(
"//bootstrapper/internal/kubernetes/kubewaiter",
"//internal/cloud/metadata",
"//internal/constants",
"//internal/deploy/helm",
"//internal/kubernetes",
"//internal/logger",
"//internal/role",
Expand Down
8 changes: 0 additions & 8 deletions bootstrapper/internal/kubernetes/k8sutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"context"
"net"

"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
Expand All @@ -24,10 +23,3 @@ type clusterUtil interface {
FixCilium(ctx context.Context) error
StartKubelet() error
}

// helmClient bundles functions related to microservice deployment.
// Only microservices that can be deployed purely via Helm are deployed with this interface.
type helmClient interface {
InstallChart(context.Context, helm.Release) error
InstallChartWithValues(ctx context.Context, release helm.Release, extraValues map[string]any) error
}
58 changes: 2 additions & 56 deletions bootstrapper/internal/kubernetes/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package kubernetes

import (
"context"
"encoding/json"
"fmt"
"net"
"regexp"
Expand All @@ -20,7 +19,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
Expand All @@ -47,7 +45,6 @@ type kubeAPIWaiter interface {
type KubeWrapper struct {
cloudProvider string
clusterUtil clusterUtil
helmClient helmClient
kubeAPIWaiter kubeAPIWaiter
configProvider configurationProvider
client k8sapi.Client
Expand All @@ -57,12 +54,11 @@ type KubeWrapper struct {

// New creates a new KubeWrapper with real values.
func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client,
providerMetadata ProviderMetadata, helmClient helmClient, kubeAPIWaiter kubeAPIWaiter,
providerMetadata ProviderMetadata, kubeAPIWaiter kubeAPIWaiter,
) *KubeWrapper {
return &KubeWrapper{
cloudProvider: cloudProvider,
clusterUtil: clusterUtil,
helmClient: helmClient,
kubeAPIWaiter: kubeAPIWaiter,
configProvider: configProvider,
client: client,
Expand All @@ -73,15 +69,13 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura

// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, versionString, clusterName string,
helmReleasesRaw []byte, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
ctx context.Context, versionString, clusterName string, conformanceMode bool, kubernetesComponents components.Components, apiServerCertSANs []string, log *logger.Logger,
) ([]byte, error) {
log.With(zap.String("version", versionString)).Infof("Installing Kubernetes components")
if err := k.clusterUtil.InstallComponents(ctx, kubernetesComponents); err != nil {
return nil, err
}

var nodePodCIDR string
var validIPs []net.IP

// Step 1: retrieve cloud metadata for Kubernetes configuration
Expand All @@ -100,10 +94,6 @@ func (k *KubeWrapper) InitCluster(

nodeIP := instance.VPCIP
subnetworkPodCIDR := instance.SecondaryIPRange
if len(instance.AliasIPRanges) > 0 {
nodePodCIDR = instance.AliasIPRanges[0]
fmt.Println("nodePodCIDR: ", nodePodCIDR) // TODO(elchead): remove all commented code
}

// this is the endpoint in "kubeadm init --control-plane-endpoint=<IP/DNS>:<port>"
// TODO(malt3): switch over to DNS name on AWS and Azure
Expand Down Expand Up @@ -176,50 +166,6 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("annotating node with Kubernetes components hash: %w", err)
}

// Step 3: configure & start kubernetes controllers
log.Infof("Starting Kubernetes controllers and deployments")
//setupPodNetworkInput := k8sapi.SetupPodNetworkInput{
// CloudProvider: k.cloudProvider,
// NodeName: nodeName,
// FirstNodePodCIDR: nodePodCIDR,
// SubnetworkPodCIDR: subnetworkPodCIDR,
// LoadBalancerHost: controlPlaneHost,
// LoadBalancerPort: controlPlanePort,
//}

var helmReleases helm.Releases
if err := json.Unmarshal(helmReleasesRaw, &helmReleases); err != nil {
return nil, fmt.Errorf("unmarshalling helm releases: %w", err)
}

log.Infof("Installing Cilium")
//ciliumVals, err := k.setupCiliumVals(ctx, setupPodNetworkInput)
//if err != nil {
// return nil, fmt.Errorf("setting up cilium vals: %w", err)
//}
//log.Infof("ciliumVals: %+v\n", ciliumVals)
//if err := k.helmClient.InstallChartWithValues(ctx, helmReleases.Cilium, ciliumVals); err != nil {
// return nil, fmt.Errorf("installing cilium pod network: %w", err)
//}

//log.Infof("Waiting for Cilium to become healthy")
//timeToStartWaiting := time.Now()
//// TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once
//// saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN).
//waitCtx, cancel = context.WithTimeout(ctx, 20*time.Minute)
//defer cancel()
//if err := k.clusterUtil.WaitForCilium(waitCtx, log); err != nil {
// return nil, fmt.Errorf("waiting for Cilium to become healthy: %w", err)
//}
//timeUntilFinishedWaiting := time.Since(timeToStartWaiting)
//log.With(zap.Duration("duration", timeUntilFinishedWaiting)).Infof("Cilium became healthy")

//log.Infof("Restarting Cilium")
//if err := k.clusterUtil.FixCilium(ctx); err != nil {
// log.With(zap.Error(err)).Errorf("FixCilium failed")
// // Continue and don't throw an error here - things might be okay.
//}

log.Infof("Setting up internal-config ConfigMap")
if err := k.setupInternalConfigMap(ctx); err != nil {
return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
Expand Down
25 changes: 1 addition & 24 deletions bootstrapper/internal/kubernetes/kubernetes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
Expand All @@ -43,7 +42,6 @@ func TestInitCluster(t *testing.T) {

testCases := map[string]struct {
clusterUtil stubClusterUtil
helmClient stubHelmClient
kubectl stubKubectl
kubeAPIWaiter stubKubeAPIWaiter
providerMetadata ProviderMetadata
Expand Down Expand Up @@ -131,14 +129,12 @@ func TestInitCluster(t *testing.T) {
},
"kubeadm init fails when deploying cilium": {
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{ciliumError: assert.AnError},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up constellation-services chart": {
clusterUtil: stubClusterUtil{kubeconfig: []byte("someKubeconfig")},
helmClient: stubHelmClient{installChartError: assert.AnError},
kubeAPIWaiter: stubKubeAPIWaiter{},
providerMetadata: &stubProviderMetadata{},
wantErr: true,
Expand Down Expand Up @@ -182,7 +178,6 @@ func TestInitCluster(t *testing.T) {
kube := KubeWrapper{
cloudProvider: "aws", // provide a valid cloud provider for cilium installation
clusterUtil: &tc.clusterUtil,
helmClient: &tc.helmClient,
providerMetadata: tc.providerMetadata,
kubeAPIWaiter: &tc.kubeAPIWaiter,
configProvider: &stubConfigProvider{initConfig: k8sapi.KubeadmInitYAML{}},
Expand All @@ -192,7 +187,7 @@ func TestInitCluster(t *testing.T) {

_, err := kube.InitCluster(
context.Background(), string(tc.k8sVersion), "kubernetes",
[]byte("{}"), false, nil, nil, logger.NewTest(t),
false, nil, nil, logger.NewTest(t),
)

if tc.wantErr {
Expand Down Expand Up @@ -551,24 +546,6 @@ func (s *stubKubectl) EnforceCoreDNSSpread(_ context.Context) error {
return s.enforceCoreDNSSpreadErr
}

type stubHelmClient struct {
ciliumError error
installChartError error
}

func (s *stubHelmClient) InstallChart(ctx context.Context, release helm.Release) error {
return s.InstallChartWithValues(ctx, release, release.Values)
}

func (s *stubHelmClient) InstallChartWithValues(_ context.Context, release helm.Release, _ map[string]any) error {
switch release.ReleaseName {
case "cilium":
return s.ciliumError
default:
return s.installChartError
}
}

type stubKubeAPIWaiter struct {
waitErr error
}
Expand Down
3 changes: 1 addition & 2 deletions cli/internal/cmd/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ go_library(
"//internal/config/migration",
"//internal/constants",
"//internal/crypto",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/dialer",
"//internal/grpc/grpclog",
Expand Down Expand Up @@ -138,6 +137,7 @@ go_test(
"//bootstrapper/initproto",
"//cli/internal/cloudcmd",
"//cli/internal/clusterid",
"//cli/internal/helm",
"//cli/internal/iamid",
"//cli/internal/kubernetes",
"//cli/internal/terraform",
Expand All @@ -153,7 +153,6 @@ go_test(
"//internal/config",
"//internal/constants",
"//internal/crypto/testvector",
"//internal/deploy/helm",
"//internal/file",
"//internal/grpc/atlscredentials",
"//internal/grpc/dialer",
Expand Down
7 changes: 3 additions & 4 deletions cli/internal/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/crypto"
helmdeploy "github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/grpc/dialer"
"github.com/edgelesssys/constellation/v2/internal/grpc/grpclog"
Expand Down Expand Up @@ -441,9 +440,9 @@ func (i *initCmd) evalFlagArgs(cmd *cobra.Command) (initFlags, error) {
if err != nil {
return initFlags{}, fmt.Errorf("parsing skip-helm-wait flag: %w", err)
}
helmWaitMode := helmdeploy.WaitModeAtomic
helmWaitMode := helm.WaitModeAtomic
if skipHelmWait {
helmWaitMode = helmdeploy.WaitModeNone
helmWaitMode = helm.WaitModeNone
}
i.log.Debugf("Helm wait flag is %t", skipHelmWait)
configPath, err := cmd.Flags().GetString("config")
Expand Down Expand Up @@ -478,7 +477,7 @@ type initFlags struct {
configPath string
masterSecretPath string
conformance bool
helmWaitMode helmdeploy.WaitMode
helmWaitMode helm.WaitMode
force bool
mergeConfigs bool
}
Expand Down
Loading

0 comments on commit eb7d476

Please sign in to comment.