Introduce special handling of CNI addons

This change introduces special handling of addons that introduce CNI
plugins. The change introduces the following new functionality:

- Introduce '--cni-plugin' flag for use with 'skuba cluster init'
- Addons now declare their AddOnType, allowing for custom logic based on
  AddOnType to be invoked when writing out initial cluster manifests
- The user-supplied CNI plugin request is checked against addons that provide
  CNI plugins
- If '--cni-plugin' is not specified, cilium selected as the default CNI
  addon. This maintains backward-compatible behavior for 'skuba cluster init'
- Rather than write config files for all CNI plugin addons,
  configuration files are only written for the selected CNI addon
- CNI addons that are not applicable will not be applied during
  upgrades or node actions such as bootstrap or join

Signed-off-by: Ryan Tidwell <[email protected]>

cmd/skuba/cluster/init.go

@@ -32,6 +32,7 @@ type initOptions struct {
 	KubernetesVersion string
 	CloudProvider     string
 	StrictCapDefaults bool
+	CniPlugin         string
 }
 
 // NewInitCmd creates a new `skuba cluster init` cobra command
@@ -47,7 +48,8 @@ func NewInitCmd() *cobra.Command {
 				initOptions.CloudProvider,
 				initOptions.ControlPlane,
 				initOptions.KubernetesVersion,
-				initOptions.StrictCapDefaults)
+				initOptions.StrictCapDefaults,
+				initOptions.CniPlugin)
 			if err != nil {
 				klog.Fatalf("init failed due to error: %s", err)
 			}
@@ -75,5 +77,7 @@ func NewInitCmd() *cobra.Command {
 
 	cmd.Flags().BoolVar(&initOptions.StrictCapDefaults, "strict-capability-defaults", false, "All the containers will start with CRI-O default capabilities")
 
+	cmd.Flags().StringVar(&initOptions.CniPlugin, "cni-plugin", "cilium", "Specify the CNI plugin to be used across the cluster. Valid values: cilium")
+
 	return cmd
 }

internal/pkg/skuba/addons/README.md

@@ -20,9 +20,21 @@ https://golang.org/doc/effective_go.html#init
 ## How to create a new Addon
 
 To create a new addon, check the rest of the addons source file to
-understand what is required. Again, as long as the addon is in this
-directory, it will get called.
-
-You should also express the version mapping with kubernetes in:
+understand what is required. You should express the version mapping
+with kubernetes in:
 
 https://github.com/SUSE/skuba/blob/master/internal/pkg/skuba/kubernetes/versions.go
+
+Each addon must also declare its `AddOnType`. Any number of addons of type
+`CniAddOn`can be created. However, only a single addon of type `CniAddOn`
+can be used in the cluster and as such only one addon providing a CNI
+plugin will be activated and deployed.
+
+The CNI addon that is activated defaults to cilium, but can be toggled by
+passing `--cni-plugin` to `skuba cluster init`.
+
+Example:
+
+```sh
+skuba cluster init --control-plane load-balancer.example.com --cni-plugin cilium company-cluster
+```

internal/pkg/skuba/addons/addons.go

@@ -74,15 +74,23 @@ patches:
 `
 )
 
+const (
+	CniAddOn     AddOnType = "CNI"
+	GenericAddOn AddOnType = "GENERIC"
+)
+
 var Addons = map[kubernetes.Addon]Addon{}
 
+type AddOnType string
+
 type Addon struct {
-	addon              kubernetes.Addon
+	Addon              kubernetes.Addon
 	templater          addonTemplater
 	preflightTemplater preflightAddonTemplater
 	callbacks          addonCallbacks
 	addonPriority      addonPriority
 	getImageCallbacks  []getImageCallback
+	AddOnType          AddOnType
 }
 
 type addonCallbacks interface {
@@ -110,7 +118,7 @@ func (renderContext renderContext) AnnotatedVersion() string {
 }
 
 func (renderContext renderContext) ManifestVersion() string {
-	addonVersion := kubernetes.AddonVersionForClusterVersion(renderContext.addon.addon, renderContext.config.ClusterVersion)
+	addonVersion := kubernetes.AddonVersionForClusterVersion(renderContext.addon.Addon, renderContext.config.ClusterVersion)
 	if addonVersion == nil {
 		return ""
 	}
@@ -119,14 +127,15 @@ func (renderContext renderContext) ManifestVersion() string {
 
 // registerAddon incorporates one addon information to the Addons map that keeps track of the
 // addons which will get deployed
-func registerAddon(addon kubernetes.Addon, addonTemplater addonTemplater, preflightAddonTemplater preflightAddonTemplater, callbacks addonCallbacks, addonPriority addonPriority, getImageCallbacks []getImageCallback) {
+func registerAddon(addon kubernetes.Addon, addonType AddOnType, addonTemplater addonTemplater, preflightAddonTemplater preflightAddonTemplater, callbacks addonCallbacks, addonPriority addonPriority, getImageCallbacks []getImageCallback) {
 	Addons[addon] = Addon{
-		addon:              addon,
+		Addon:              addon,
 		templater:          addonTemplater,
 		preflightTemplater: preflightAddonTemplater,
 		callbacks:          callbacks,
 		addonPriority:      addonPriority,
 		getImageCallbacks:  getImageCallbacks,
+		AddOnType:          addonType,
 	}
 }
 
@@ -153,7 +162,7 @@ func DeployAddons(client clientset.Interface, addonConfiguration AddonConfigurat
 		return err
 	}
 	for _, addon := range addonsByPriority() {
-		addonName := addon.addon
+		addonName := addon.Addon
 		if !addon.IsPresentForClusterVersion(addonConfiguration.ClusterVersion) {
 			// This registered addon is not available on the chosen Kubernetes version, skip it
 			continue
@@ -207,7 +216,7 @@ func (addon Addon) RenderPreflight(addonConfiguration AddonConfiguration) (strin
 
 // IsPresentForClusterVersion verifies if the Addon can be deployed with the current k8s version
 func (addon Addon) IsPresentForClusterVersion(clusterVersion *version.Version) bool {
-	return kubernetes.AddonVersionForClusterVersion(addon.addon, clusterVersion) != nil
+	return kubernetes.AddonVersionForClusterVersion(addon.Addon, clusterVersion) != nil
 }
 
 // HasToBeApplied decides if the Addon is deployed by checking its version with addonVersionLower
@@ -225,19 +234,26 @@ func (addon Addon) HasToBeApplied(addonConfiguration AddonConfiguration, skubaCo
 		//      now, just return that it doesn't have to be applied.
 		return false, nil
 	}
+	// Check whether this is a CNI addon and whether its base config has been rendered.
+	// If it's a CNI plugin and the base config has not been rendered, we can assume
+	// the user requested a different CNI plugin and this addon does not need to be
+	// applied.
+	if info, err := os.Stat(addon.addonDir()); addon.AddOnType == CniAddOn && (os.IsNotExist(err) || !info.IsDir()) {
+		return false, nil
+	}
 	if skubaConfiguration.AddonsVersion == nil {
 		return true, nil
 	}
-	currentAddonVersion, found := skubaConfiguration.AddonsVersion[addon.addon]
+	currentAddonVersion, found := skubaConfiguration.AddonsVersion[addon.Addon]
 	if !found {
 		return true, nil
 	}
-	addonVersion := kubernetes.AddonVersionForClusterVersion(addon.addon, addonConfiguration.ClusterVersion)
+	addonVersion := kubernetes.AddonVersionForClusterVersion(addon.Addon, addonConfiguration.ClusterVersion)
 	return addonVersionLower(currentAddonVersion, addonVersion), nil
 }
 
 func (addon Addon) addonDir() string {
-	return filepath.Join(skubaconstants.AddonsDir(), string(addon.addon))
+	return filepath.Join(skubaconstants.AddonsDir(), string(addon.Addon))
 }
 
 func (addon Addon) baseResourcesDir(rootDir string) string {
@@ -249,11 +265,11 @@ func (addon Addon) patchResourcesDir(rootDir string) string {
 }
 
 func (addon Addon) manifestFilename() string {
-	return fmt.Sprintf("%s.yaml", addon.addon)
+	return fmt.Sprintf("%s.yaml", addon.Addon)
 }
 
 func (addon Addon) preflightManifestFilename() string {
-	return fmt.Sprintf("%s-preflight.yaml", addon.addon)
+	return fmt.Sprintf("%s-preflight.yaml", addon.Addon)
 }
 
 func (addon Addon) manifestPath(rootDir string) string {
@@ -291,7 +307,7 @@ func (addon Addon) kustomizePath(rootDir string) string {
 func (addon Addon) Write(addonConfiguration AddonConfiguration) error {
 	addonManifest, err := addon.Render(addonConfiguration)
 	if err != nil {
-		return errors.Wrapf(err, "unable to render %s addon template", addon.addon)
+		return errors.Wrapf(err, "unable to render %s addon template", addon.Addon)
 	}
 	baseResourcesDir := addon.baseResourcesDir(addon.addonDir())
 	if err := os.MkdirAll(baseResourcesDir, 0700); err != nil {
@@ -302,7 +318,7 @@ func (addon Addon) Write(addonConfiguration AddonConfiguration) error {
 		return errors.Wrapf(err, "unable to create directory: %s", patchResourcesDir)
 	}
 	if err := ioutil.WriteFile(addon.manifestPath(addon.addonDir()), []byte(addonTemplateWarning+addonManifest), 0600); err != nil {
-		return errors.Wrapf(err, "unable to write %s addon rendered template", addon.addon)
+		return errors.Wrapf(err, "unable to write %s addon rendered template", addon.Addon)
 	}
 	return nil
 }
@@ -320,7 +336,7 @@ func (addon Addon) applyPreflight(addonConfiguration AddonConfiguration, sandbox
 	}
 	preflightManifestPath := addon.preflightManifestPath(sandboxDir)
 	if err := ioutil.WriteFile(preflightManifestPath, []byte(renderedPreflightManifest), 0600); err != nil {
-		return true, errors.Wrapf(err, "could not create %q addon manifests", addon.addon)
+		return true, errors.Wrapf(err, "could not create %q addon manifests", addon.Addon)
 	}
 	cmd := exec.Command("kubectl", "apply", "--kubeconfig", skubaconstants.KubeConfigAdminFile(), "-f", preflightManifestPath)
 	if combinedOutput, err := cmd.CombinedOutput(); err != nil {
@@ -343,10 +359,10 @@ func (addon Addon) deletePreflight(sandboxDir string) error {
 // Apply deploys the addon by calling kubectl apply and pointing to the generated addon
 // manifest
 func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonConfiguration, skubaConfiguration *skuba.SkubaConfiguration) error {
-	klog.V(1).Infof("applying %q addon", addon.addon)
+	klog.V(1).Infof("applying %q addon", addon.Addon)
 	sandboxDir, err := addon.createSandbox()
 	if err != nil {
-		return errors.Wrapf(err, "could not create %q addon sandbox", addon.addon)
+		return errors.Wrapf(err, "could not create %q addon sandbox", addon.Addon)
 	}
 	defer func() {
 		_ = addon.deleteSandbox(sandboxDir)
@@ -365,7 +381,7 @@ func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonCon
 	}
 	if addon.callbacks != nil {
 		if err := addon.callbacks.beforeApply(addonConfiguration, skubaConfiguration); err != nil {
-			klog.Errorf("failed on %q addon BeforeApply callback: %v", addon.addon, err)
+			klog.Errorf("failed on %q addon BeforeApply callback: %v", addon.Addon, err)
 			return err
 		}
 	}
@@ -386,22 +402,22 @@ func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonCon
 	}
 	// Write base resources
 	if err := ioutil.WriteFile(addon.manifestPath(sandboxDir), []byte(renderedManifest), 0600); err != nil {
-		return errors.Wrapf(err, "could not create %q addon manifests", addon.addon)
+		return errors.Wrapf(err, "could not create %q addon manifests", addon.Addon)
 	}
 	// Write patch resources
 	if err := addon.copyCustomizePatches(sandboxDir); err != nil {
-		return errors.Wrapf(err, "could not link %q addon patches", addon.addon)
+		return errors.Wrapf(err, "could not link %q addon patches", addon.Addon)
 	}
 	patchList, err := addon.listPatches()
 	if err != nil {
-		return errors.Wrapf(err, "could not list patches for %q addon", addon.addon)
+		return errors.Wrapf(err, "could not list patches for %q addon", addon.Addon)
 	}
 	kustomizeContents, err := addon.kustomizeContents([]string{addon.manifestFilename()}, patchList)
 	if err != nil {
 		return errors.Wrapf(err, "could not render kustomize file")
 	}
 	if err := ioutil.WriteFile(addon.kustomizePath(sandboxDir), []byte(kustomizeContents), 0600); err != nil {
-		return errors.Wrapf(err, "could not create %q kustomize file", addon.addon)
+		return errors.Wrapf(err, "could not create %q kustomize file", addon.Addon)
 	}
 	cmd := exec.Command("kubectl", "apply", "--kubeconfig", skubaconstants.KubeConfigAdminFile(), "-k", sandboxDir)
 	if combinedOutput, err := cmd.CombinedOutput(); err != nil {
@@ -411,11 +427,11 @@ func (addon Addon) Apply(client clientset.Interface, addonConfiguration AddonCon
 	if addon.callbacks != nil {
 		if err := addon.callbacks.afterApply(addonConfiguration, skubaConfiguration); err != nil {
 			// TODO: should we rollback here?
-			klog.Errorf("failed on %q addon AfterApply callback: %v", addon.addon, err)
+			klog.Errorf("failed on %q addon AfterApply callback: %v", addon.Addon, err)
 			return err
 		}
 	}
-	return updateSkubaConfigMapWithAddonVersion(client, addon.addon, addonConfiguration.ClusterVersion, skubaConfiguration)
+	return updateSkubaConfigMapWithAddonVersion(client, addon.Addon, addonConfiguration.ClusterVersion, skubaConfiguration)
 }
 
 // Images returns the images required for this Addon to properly function
@@ -440,7 +456,7 @@ func (addon Addon) listPatches() ([]string, error) {
 }
 
 func (addon Addon) createSandbox() (string, error) {
-	return ioutil.TempDir("", fmt.Sprintf("skuba-addon-%s", addon.addon))
+	return ioutil.TempDir("", fmt.Sprintf("skuba-addon-%s", addon.Addon))
 }
 
 func (addon Addon) deleteSandbox(sandboxDir string) error {

internal/pkg/skuba/addons/cilium.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019 SUSE LLC.
+ * Copyright (c) 2020 SUSE LLC.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Cilium, renderCiliumTemplate, renderCiliumPreflightTemplate, ciliumCallbacks{}, normalPriority, []getImageCallback{GetCiliumInitImage, GetCiliumOperatorImage, GetCiliumImage})
+	registerAddon(kubernetes.Cilium, CniAddOn, renderCiliumTemplate, renderCiliumPreflightTemplate, ciliumCallbacks{}, normalPriority, []getImageCallback{GetCiliumInitImage, GetCiliumOperatorImage, GetCiliumImage})
 }
 
 func GetCiliumInitImage(imageTag string) string {

internal/pkg/skuba/addons/dex.go

@@ -28,7 +28,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Dex, renderDexTemplate, nil, dexCallbacks{}, normalPriority, []getImageCallback{GetDexImage})
+	registerAddon(kubernetes.Dex, GenericAddOn, renderDexTemplate, nil, dexCallbacks{}, normalPriority, []getImageCallback{GetDexImage})
 }
 
 func GetDexImage(imageTag string) string {

internal/pkg/skuba/addons/gangway.go

@@ -28,7 +28,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Gangway, renderGangwayTemplate, nil, gangwayCallbacks{}, normalPriority, []getImageCallback{GetGangwayImage})
+	registerAddon(kubernetes.Gangway, GenericAddOn, renderGangwayTemplate, nil, gangwayCallbacks{}, normalPriority, []getImageCallback{GetGangwayImage})
 }
 
 func GetGangwayImage(imageTag string) string {

internal/pkg/skuba/addons/kured.go

@@ -24,7 +24,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.Kured, renderKuredTemplate, nil, nil, normalPriority, []getImageCallback{GetKuredImage})
+	registerAddon(kubernetes.Kured, GenericAddOn, renderKuredTemplate, nil, nil, normalPriority, []getImageCallback{GetKuredImage})
 }
 
 func GetKuredImage(imageTag string) string {

internal/pkg/skuba/addons/metricsserver.go

@@ -35,7 +35,7 @@ import (
 )
 
 func init() {
-	registerAddon(kubernetes.MetricsServer, renderMetricsServerTemplate, nil, metricsServerCallbacks{}, normalPriority, []getImageCallback{GetMetricsServerImage})
+	registerAddon(kubernetes.MetricsServer, GenericAddOn, renderMetricsServerTemplate, nil, metricsServerCallbacks{}, normalPriority, []getImageCallback{GetMetricsServerImage})
 }
 
 func GetMetricsServerImage(imageTag string) string {

internal/pkg/skuba/addons/psp.go

@@ -20,7 +20,7 @@ package addons
 import "github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
 
 func init() {
-	registerAddon(kubernetes.PSP, renderPSPTemplate, nil, nil, highPriority, []getImageCallback{})
+	registerAddon(kubernetes.PSP, GenericAddOn, renderPSPTemplate, nil, nil, highPriority, []getImageCallback{})
 }
 
 func renderPSPTemplate(addonConfiguration AddonConfiguration) string {

pkg/skuba/actions/cluster/init/init.go

@@ -55,6 +55,7 @@ type InitConfiguration struct {
 	// Note: UseHyperKube can be removed when we drop the support of
 	// provisioning clusters of version 1.17.
 	UseHyperKube bool
+	CniPlugin    kubernetes.Addon
 }
 
 func (initConfiguration InitConfiguration) ControlPlaneHost() string {
@@ -76,7 +77,7 @@ func (initConfiguration InitConfiguration) ControlPlaneHostAndPort() string {
 	return fmt.Sprintf("%s:%s", controlPlaneHost, controlPlanePort)
 }
 
-func NewInitConfiguration(clusterName, cloudProvider, controlPlane, kubernetesDesiredVersion string, strictCapDefaults bool) (InitConfiguration, error) {
+func NewInitConfiguration(clusterName, cloudProvider, controlPlane, kubernetesDesiredVersion string, strictCapDefaults bool, cniPlugin string) (InitConfiguration, error) {
 	kubernetesVersion := kubernetes.LatestVersion()
 	var err error
 	needsHyperKube := false
@@ -104,6 +105,7 @@ func NewInitConfiguration(clusterName, cloudProvider, controlPlane, kubernetesDe
 		CoreDNSImageTag:   kubernetes.ComponentVersionForClusterVersion(kubernetes.CoreDNS, kubernetesVersion),
 		StrictCapDefaults: strictCapDefaults,
 		UseHyperKube:      needsHyperKube,
+		CniPlugin:         kubernetes.Addon(cniPlugin),
 	}, nil
 }
 
@@ -114,6 +116,11 @@ func Init(initConfiguration InitConfiguration) error {
 	if _, err := os.Stat(initConfiguration.ClusterName); err == nil {
 		return errors.Errorf("cluster configuration directory %q already exists", initConfiguration.ClusterName)
 	}
+	if addon, found := addons.Addons[initConfiguration.CniPlugin]; !found || addon.AddOnType != addons.CniAddOn {
+		return fmt.Errorf("unknown CNI plugin provided: %s", initConfiguration.CniPlugin)
+	}
+
+	// write configuration files
 	if err := writeScaffoldFiles(initConfiguration); err != nil {
 		return err
 	}
@@ -134,6 +141,16 @@ func Init(initConfiguration InitConfiguration) error {
 	return nil
 }
 
+func isAddonRequired(addon addons.Addon, initConfiguration InitConfiguration) bool {
+	if !addon.IsPresentForClusterVersion(initConfiguration.KubernetesVersion) {
+		return false
+	}
+	if addon.AddOnType == addons.CniAddOn && initConfiguration.CniPlugin != addon.Addon {
+		return false
+	}
+	return true
+}
+
 func writeScaffoldFiles(initConfiguration InitConfiguration) error {
 	scaffoldFilesToWrite := criScaffoldFiles["criconfig"]
 	kubernetesVersion := initConfiguration.KubernetesVersion
@@ -209,7 +226,7 @@ func writeAddonConfigFiles(initConfiguration InitConfiguration) error {
 		ClusterName:    initConfiguration.ClusterName,
 	}
 	for addonName, addon := range addons.Addons {
-		if !addon.IsPresentForClusterVersion(initConfiguration.KubernetesVersion) {
+		if !isAddonRequired(addon, initConfiguration) {
 			continue
 		}
 		if err := addon.Write(addonConfiguration); err != nil {