Merge pull request #1162 from rktidwell/master

Enable toggling of the CNI plugin used in the cluster

cmd/skuba/cluster/init.go

@@ -32,6 +32,7 @@ type initOptions struct {
 	KubernetesVersion string
 	CloudProvider     string
 	StrictCapDefaults bool
+	CniPlugin         string
 }
 
 // NewInitCmd creates a new `skuba cluster init` cobra command
@@ -47,7 +48,8 @@ func NewInitCmd() *cobra.Command {
 				initOptions.CloudProvider,
 				initOptions.ControlPlane,
 				initOptions.KubernetesVersion,
-				initOptions.StrictCapDefaults)
+				initOptions.StrictCapDefaults,
+				initOptions.CniPlugin)
 			if err != nil {
 				klog.Fatalf("init failed due to error: %s", err)
 			}
@@ -75,5 +77,7 @@ func NewInitCmd() *cobra.Command {
 
 	cmd.Flags().BoolVar(&initOptions.StrictCapDefaults, "strict-capability-defaults", false, "All the containers will start with CRI-O default capabilities")
 
+	cmd.Flags().StringVar(&initOptions.CniPlugin, "cni-plugin", "cilium", "Specify the CNI plugin to be used across the cluster. Valid values: cilium")
+
 	return cmd
 }

internal/pkg/skuba/addons/README.md

@@ -20,9 +20,21 @@ https://golang.org/doc/effective_go.html#init
 ## How to create a new Addon
 
 To create a new addon, check the rest of the addons source file to
-understand what is required. Again, as long as the addon is in this
-directory, it will get called.
-
-You should also express the version mapping with kubernetes in:
+understand what is required. You should express the version mapping
+with kubernetes in:
 
 https://github.com/SUSE/skuba/blob/master/internal/pkg/skuba/kubernetes/versions.go
+
+Each addon must also declare its `AddOnType`. Any number of addons of type
+`CniAddOn`can be created. However, only a single addon of type `CniAddOn`
+can be used in the cluster and as such only one addon providing a CNI
+plugin will be activated and deployed.
+
+The CNI addon that is activated defaults to cilium, but can be toggled by
+passing `--cni-plugin` to `skuba cluster init`.
+
+Example:
+
+```sh
+skuba cluster init --control-plane load-balancer.example.com --cni-plugin cilium company-cluster
+```

internal/pkg/skuba/addons/addons.go

@@ -74,15 +74,23 @@ patches:
 `
 )
 
+const (
+	CniAddOn     AddOnType = "CNI"
+	GenericAddOn AddOnType = "GENERIC"
+)
+
 var Addons = map[kubernetes.Addon]Addon{}
 
+type AddOnType string
+
 type Addon struct {
-	addon              kubernetes.Addon
+	Addon              kubernetes.Addon
 	templater          addonTemplater
 	preflightTemplater preflightAddonTemplater
 	callbacks          addonCallbacks
 	addonPriority      addonPriority
 	getImageCallbacks  []getImageCallback
+	AddOnType          AddOnType
 }
 
 type addonCallbacks interface {
@@ -110,7 +118,7 @@ func (renderContext renderContext) AnnotatedVersion() string {
 }
 
 func (renderContext renderContext) ManifestVersion() string {
-	addonVersion := kubernetes.AddonVersionForClusterVersion(renderContext.addon.addon, renderContext.config.ClusterVersion)
+	addonVersion := kubernetes.AddonVersionForClusterVersion(renderContext.addon.Addon, renderContext.config.ClusterVersion)
 	if addonVersion == nil {
 		return ""
 	}
@@ -119,14 +127,15 @@ func (renderContext renderContext) ManifestVersion() string {
 
 // registerAddon incorporates one addon information to the Addons map that keeps track of the
 // addons which will get deployed
-func registerAddon(addon kubernetes.Addon, addonTemplater addonTemplater, preflightAddonTemplater preflightAddonTemplater, callbacks addonCallbacks, addonPriority addonPriority, getImageCallbacks []getImageCallback) {
+func registerAddon(addon kubernetes.Addon, addonType AddOnType, addonTemplater addonTemplater, preflightAddonTemplater preflightAddonTemplater, callbacks addonCallbacks, addonPriority addonPriority, getImageCallbacks []getImageCallback) {
 	Addons[addon] = Addon{
-		addon:              addon,
+		Addon:              addon,
 		templater:          addonTemplater,
 		preflightTemplater: preflightAddonTemplater,
 		callbacks:          callbacks,
 		addonPriority:      addonPriority,
 		getImageCallbacks:  getImageCallbacks,
+		AddOnType:          addonType,
 	}
 }
 
@@ -153,7 +162,7 @@ func DeployAddons(client clientset.Interface, addonConfiguration AddonConfigurat
 		return err
 	}
 	for _, addon := range addonsByPriority() {
-		addonName := addon.addon
+		addonName := addon.Addon
 		if !addon.IsPresentForClusterVersion(addonConfiguration.ClusterVersion) {
 			// This registered addon is not available on the chosen Kubernetes version, skip it
 			continue
@@ -208,7 +217,7 @@ func (addon Addon) RenderPreflight(addonConfiguration AddonConfiguration) (strin
 
 // IsPresentForClusterVersion verifies if the Addon can be deployed with the current k8s version
 func (addon Addon) IsPresentForClusterVersion(clusterVersion *version.Version) bool {
-	return kubernetes.AddonVersionForClusterVersion(addon.addon, clusterVersion) != nil
+	return kubernetes.AddonVersionForClusterVersion(addon.Addon, clusterVersion) != nil
 }
 
 // HasToBeApplied decides if the Addon is deployed by checking its version with addonVersionLower
@@ -226,19 +235,26 @@ func (addon Addon) HasToBeApplied(addonConfiguration AddonConfiguration, skubaCo
 		//      now, just return that it doesn't have to be applied.
 		return false, nil
 	}
+	// Check whether this is a CNI addon and whether its base config has been rendered.
+	// If it's a CNI plugin and the base config has not been rendered, we can assume
+	// the user requested a different CNI plugin and this addon does not need to be
+	// applied.
+	if info, err := os.Stat(addon.addonDir()); addon.AddOnType == CniAddOn && (os.IsNotExist(err) || !info.IsDir()) {
+		return false, nil
+	}
 	if skubaConfiguration.AddonsVersion == nil {
 		return true, nil
 	}
-	currentAddonVersion, found := skubaConfiguration.AddonsVersion[addon.addon]
+	currentAddonVersion, found := skubaConfiguration.AddonsVersion[addon.Addon]
 	if !found {
 		return true, nil
 	}
-	addonVersion := kubernetes.AddonVersionForClusterVersion(addon.addon, addonConfiguration.ClusterVersion)
+	addonVersion := kubernetes.AddonVersionForClusterVersion(addon.Addon, addonConfiguration.ClusterVersion)
 	return addonVersionLower(currentAddonVersion, addonVersion), nil
 }
 
 func (addon Addon) addonDir() string {
-	return filepath.Join(skubaconstants.AddonsDir(), string(addon.addon))
+	return filepath.Join(skubaconstants.AddonsDir(), string(addon.Addon))
 }
 
 func (addon Addon) baseResourcesDir(rootDir string) string {
@@ -250,11 +266,11 @@ func (addon Addon) patchResourcesDir(rootDir string) string {
 }
 
 func (addon Addon) manifestFilename() string {
-	return fmt.Sprintf("%s.yaml", addon.addon)
+	return fmt.Sprintf("%s.yaml", addon.Addon)
 }
 
 func (addon Addon) preflightManifestFilename() string {
-	return fmt.Sprintf("%s-preflight.yaml", addon.addon)
+	return fmt.Sprintf("%s-preflight.yaml", addon.Addon)
 }
 
 func (addon Addon) manifestPath(rootDir string) string {
@@ -292,7 +308,7 @@ func (addon Addon) kustomizePath(rootDir string) string {
 func (addon Addon) Write(addonConfiguration AddonConfiguration) error {
 	addonManifest, err := addon.Render(addonConfiguration)
 	if err != nil {
-		return errors.Wrapf(err, "unable to render %s addon template", addon.addon)
+		return errors.Wrapf(err, "unable to render %s addon template", addon.Addon)
 	}
 	baseResourcesDir := addon.baseResourcesDir(addon.addonDir())
 	if err := os.MkdirAll(baseResourcesDir, 0700); err != nil {
@@ -303,7 +319,7 @@ func (addon Addon) Write(addonConfiguration AddonConfiguration) error {
 		return errors.Wrapf(err, "unable to create directory: %s", patchResourcesDir)
 	}
 	if err := ioutil.WriteFile(addon.manifestPath(addon.addonDir()), []byte(addonTemplateWarning+addonManifest), 0600); err != nil {
-		return errors.Wrapf(err, "unable to write %s addon rendered template", addon.addon)
+		return errors.Wrapf(err, "unable to write %s addon rendered template", addon.Addon)
 	}
 	return nil
 }
@@ -321,7 +337,7 @@ func (addon Addon) applyPreflight(addonConfiguration AddonConfiguration, sandbox
 	}
 	preflightManifestPath := addon.preflightManifestPath(sandboxDir)
 	if err := ioutil.WriteFile(preflightManifestPath, []byte(renderedPreflightManifest), 0600); err != nil {
-		return true, errors.Wrapf(err, "could not create %q addon manifests", addon.addon)
+		return true, errors.Wrapf(err, "could not create %q addon manifests", addon.Addon)
 	}
 	cmd := exec.Command("kubectl", "apply", "--kubeconfig", skubaconstants.KubeConfigAdminFile(), "-f", preflightManifestPath)
 	if combinedOutput, err := cmd.CombinedOutput(); err != nil {
@@ -344,10 +360,10 @@ func (addon Addon) deletePreflight(sandboxDir string) error {
 // Apply deploys the addon by calling kubectl apply and pointing to the generated addon
 // manifest
 func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonConfiguration, skubaConfiguration *skuba.SkubaConfiguration) error {
-	klog.V(1).Infof("applying %q addon", addon.addon)
+	klog.V(1).Infof("applying %q addon", addon.Addon)
 	sandboxDir, err := addon.createSandbox()
 	if err != nil {
-		return errors.Wrapf(err, "could not create %q addon sandbox", addon.addon)
+		return errors.Wrapf(err, "could not create %q addon sandbox", addon.Addon)
 	}
 	defer func() {
 		_ = addon.deleteSandbox(sandboxDir)
@@ -366,7 +382,7 @@ func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonCon
 	}
 	if addon.callbacks != nil {
 		if err := addon.callbacks.beforeApply(addonConfiguration, skubaConfiguration); err != nil {
-			klog.Errorf("failed on %q addon BeforeApply callback: %v", addon.addon, err)
+			klog.Errorf("failed on %q addon BeforeApply callback: %v", addon.Addon, err)
 			return err
 		}
 	}
@@ -387,22 +403,22 @@ func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonCon
 	}
 	// Write base resources
 	if err := ioutil.WriteFile(addon.manifestPath(sandboxDir), []byte(renderedManifest), 0600); err != nil {
-		return errors.Wrapf(err, "could not create %q addon manifests", addon.addon)
+		return errors.Wrapf(err, "could not create %q addon manifests", addon.Addon)
 	}
 	// Write patch resources
 	if err := addon.copyCustomizePatches(sandboxDir); err != nil {
-		return errors.Wrapf(err, "could not link %q addon patches", addon.addon)
+		return errors.Wrapf(err, "could not link %q addon patches", addon.Addon)
 	}
 	patchList, err := addon.listPatches()
 	if err != nil {
-		return errors.Wrapf(err, "could not list patches for %q addon", addon.addon)
+		return errors.Wrapf(err, "could not list patches for %q addon", addon.Addon)
 	}
 	kustomizeContents, err := addon.kustomizeContents([]string{addon.manifestFilename()}, patchList)
 	if err != nil {
 		return errors.Wrapf(err, "could not render kustomize file")
 	}
 	if err := ioutil.WriteFile(addon.kustomizePath(sandboxDir), []byte(kustomizeContents), 0600); err != nil {
-		return errors.Wrapf(err, "could not create %q kustomize file", addon.addon)
+		return errors.Wrapf(err, "could not create %q kustomize file", addon.Addon)
 	}
 	cmd := exec.Command("kubectl", "apply", "--kubeconfig", skubaconstants.KubeConfigAdminFile(), "-k", sandboxDir)
 	if combinedOutput, err := cmd.CombinedOutput(); err != nil {
@@ -412,11 +428,11 @@ func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonCon
 	if addon.callbacks != nil {
 		if err := addon.callbacks.afterApply(addonConfiguration, skubaConfiguration); err != nil {
 			// TODO: should we rollback here?
-			klog.Errorf("failed on %q addon AfterApply callback: %v", addon.addon, err)
+			klog.Errorf("failed on %q addon AfterApply callback: %v", addon.Addon, err)
 			return err
 		}
 	}
-	return updateSkubaConfigMapWithAddonVersion(client, addon.addon, addonConfiguration.ClusterVersion, skubaConfiguration)
+	return updateSkubaConfigMapWithAddonVersion(client, addon.Addon, addonConfiguration.ClusterVersion, skubaConfiguration)
 }
 
 // Images returns the images required for this Addon to properly function
@@ -441,7 +457,7 @@ func (addon Addon) listPatches() ([]string, error) {
 }
 
 func (addon Addon) createSandbox() (string, error) {
-	return ioutil.TempDir("", fmt.Sprintf("skuba-addon-%s", addon.addon))
+	return ioutil.TempDir("", fmt.Sprintf("skuba-addon-%s", addon.Addon))
 }
 
 func (addon Addon) deleteSandbox(sandboxDir string) error {

internal/pkg/skuba/addons/cilium.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 SUSE LLC.
+ * Copyright (c) 2020 SUSE LLC.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Cilium, renderCiliumTemplate, renderCiliumPreflightTemplate, ciliumCallbacks{}, normalPriority, []getImageCallback{GetCiliumInitImage, GetCiliumOperatorImage, GetCiliumImage})
+	registerAddon(kubernetes.Cilium, CniAddOn, renderCiliumTemplate, renderCiliumPreflightTemplate, ciliumCallbacks{}, normalPriority, []getImageCallback{GetCiliumInitImage, GetCiliumOperatorImage, GetCiliumImage})
 }
 
 func GetCiliumInitImage(imageTag string) string {

internal/pkg/skuba/addons/dex.go

@@ -28,7 +28,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Dex, renderDexTemplate, nil, dexCallbacks{}, normalPriority, []getImageCallback{GetDexImage})
+	registerAddon(kubernetes.Dex, GenericAddOn, renderDexTemplate, nil, dexCallbacks{}, normalPriority, []getImageCallback{GetDexImage})
 }
 
 func GetDexImage(imageTag string) string {

internal/pkg/skuba/addons/gangway.go

@@ -28,7 +28,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Gangway, renderGangwayTemplate, nil, gangwayCallbacks{}, normalPriority, []getImageCallback{GetGangwayImage})
+	registerAddon(kubernetes.Gangway, GenericAddOn, renderGangwayTemplate, nil, gangwayCallbacks{}, normalPriority, []getImageCallback{GetGangwayImage})
 }
 
 func GetGangwayImage(imageTag string) string {

internal/pkg/skuba/addons/kured.go

@@ -24,7 +24,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Kured, renderKuredTemplate, nil, nil, normalPriority, []getImageCallback{GetKuredImage})
+	registerAddon(kubernetes.Kured, GenericAddOn, renderKuredTemplate, nil, nil, normalPriority, []getImageCallback{GetKuredImage})
 }
 
 func GetKuredImage(imageTag string) string {

internal/pkg/skuba/addons/metricsserver.go

@@ -35,7 +35,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.MetricsServer, renderMetricsServerTemplate, nil, metricsServerCallbacks{}, normalPriority, []getImageCallback{GetMetricsServerImage})
+	registerAddon(kubernetes.MetricsServer, GenericAddOn, renderMetricsServerTemplate, nil, metricsServerCallbacks{}, normalPriority, []getImageCallback{GetMetricsServerImage})
 }
 
 func GetMetricsServerImage(imageTag string) string {

internal/pkg/skuba/addons/psp.go

@@ -20,7 +20,7 @@ package addons
 import "github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
 
 func init() {
-	registerAddon(kubernetes.PSP, renderPSPTemplate, nil, nil, highPriority, []getImageCallback{})
+	registerAddon(kubernetes.PSP, GenericAddOn, renderPSPTemplate, nil, nil, highPriority, []getImageCallback{})
 }
 
 func renderPSPTemplate(addonConfiguration AddonConfiguration) string {