Merge pull request #890 from ereslibre/check-upgrade-preconditions-on…

…-plan-and-apply-4.1.0

Check preconditions on worker nodes before upgrading

internal/pkg/skuba/upgrade/node/versions.go

@@ -18,6 +18,7 @@
 package node
 
 import (
+	"bytes"
 	"fmt"
 	"reflect"
 
@@ -29,6 +30,7 @@ import (
 
 	"github.com/SUSE/skuba/internal/pkg/skuba/kubeadm"
 	"github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
+	"github.com/SUSE/skuba/internal/pkg/skuba/upgrade/addon"
 	upgradecluster "github.com/SUSE/skuba/internal/pkg/skuba/upgrade/cluster"
 )
 
@@ -88,16 +90,74 @@ func (nviu NodeVersionInfoUpdate) IsFirstControlPlaneNodeToBeUpgraded(client cli
 	return isControlPlane && allControlPlanesMatchVersion && matchesClusterVersion, nil
 }
 
-func UpdateStatus(clientSet clientset.Interface, nodeName string) (NodeVersionInfoUpdate, error) {
-	currentClusterVersion, err := kubeadm.GetCurrentClusterVersion(clientSet)
+// NodeUpgradeableCheck returns whether a given node is upgradeable or not, taking global cluster restrictions into account.
+// If all preconditions are met for the given node, no error will be returned.
+func (nviu NodeVersionInfoUpdate) NodeUpgradeableCheck(client clientset.Interface, currentClusterVersion *version.Version) error {
+	errorMessages := []string{}
+	isFirstControlPlaneNodeToBeUpgraded, err := nviu.IsFirstControlPlaneNodeToBeUpgraded(client)
+	if err != nil {
+		return err
+	}
+	if isFirstControlPlaneNodeToBeUpgraded {
+		// First check if all schedulable workers will tolerate the version we are upgrading to. If they don't, they need to be upgraded first.
+		upgradeable, err := kubernetes.AllWorkerNodesTolerateVersion(client, nviu.Update.APIServerVersion)
+		if err != nil {
+			return err
+		}
+		if !upgradeable {
+			errorMessages = append(errorMessages, fmt.Sprintf("Make sure all schedulable worker nodes match the current cluster version: %s, and retry after upgrading them", currentClusterVersion))
+		}
+		// Then check if we have addon upgrades available that would need to be applied first.
+		updatedAddons, err := addon.UpdatedAddons(client, currentClusterVersion)
+		if err != nil {
+			return err
+		}
+		if addon.HasAddonUpdate(updatedAddons) {
+			errorMessages = append(errorMessages, fmt.Sprintf("There are addon upgrades available for the current cluster version (%s) that need to be applied first", currentClusterVersion))
+		}
+	} else {
+		// There is already at least one updated control plane node
+		if nviu.Current.IsControlPlane() {
+			// Secondary control plane, check if all schedulable worker nodes tolerate the current cluster version
+			upgradeable, err := kubernetes.AllWorkerNodesTolerateVersion(client, currentClusterVersion)
+			if err != nil {
+				return err
+			}
+			if !upgradeable {
+				errorMessages = append(errorMessages, fmt.Sprintf("Make sure all schedulable worker nodes match the current cluster version: %s, and retry after upgrading them", currentClusterVersion))
+			}
+		} else {
+			// Worker node, check if all control plane nodes match the current cluster version
+			upgradeable, err := kubernetes.AllControlPlanesMatchVersion(client, currentClusterVersion)
+			if err != nil {
+				return err
+			}
+			if !upgradeable {
+				errorMessages = append(errorMessages, fmt.Sprintf("Make sure all control plane nodes match the current cluster version: %s, and retry after upgrading them", currentClusterVersion))
+			}
+		}
+	}
+	if len(errorMessages) > 0 {
+		var errorMessage bytes.Buffer
+		errorMessage.WriteString(fmt.Sprintf("node %s cannot be upgraded yet. The following errors were detected:\n", nviu.Current.Nodename))
+		for _, error := range errorMessages {
+			errorMessage.WriteString(fmt.Sprintf(" - %s\n", error))
+		}
+		return errors.New(errorMessage.String())
+	}
+	return nil
+}
+
+func UpdateStatus(client clientset.Interface, nodeName string) (NodeVersionInfoUpdate, error) {
+	currentClusterVersion, err := kubeadm.GetCurrentClusterVersion(client)
 	if err != nil {
 		return NodeVersionInfoUpdate{}, err
 	}
-	allNodesVersioningInfo, err := kubernetes.AllNodesVersioningInfo(clientSet)
+	allNodesVersioningInfo, err := kubernetes.AllNodesVersioningInfo(client)
 	if err != nil {
 		return NodeVersionInfoUpdate{}, err
 	}
-	node, err := clientSet.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+	node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
 	if err != nil {
 		return NodeVersionInfoUpdate{}, errors.Wrapf(err, "could not find node %s", nodeName)
 	}

pkg/skuba/actions/node/upgrade/apply.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
@@ -33,6 +32,7 @@ import (
 	"github.com/SUSE/skuba/internal/pkg/skuba/kured"
 	"github.com/SUSE/skuba/internal/pkg/skuba/node"
 	upgradenode "github.com/SUSE/skuba/internal/pkg/skuba/upgrade/node"
+	"github.com/pkg/errors"
 )
 
 func Apply(clientSet clientset.Interface, target *deployments.Target) error {
@@ -46,20 +46,27 @@ func Apply(clientSet clientset.Interface, target *deployments.Target) error {
 	}
 	currentVersion := currentClusterVersion.String()
 	latestVersion := kubernetes.LatestVersion().String()
-	fmt.Printf("Current Kubernetes cluster version: %s\n", currentVersion)
-	fmt.Printf("Latest Kubernetes version: %s\n", latestVersion)
-	fmt.Println()
-
 	nodeVersionInfoUpdate, err := upgradenode.UpdateStatus(clientSet, target.Nodename)
 	if err != nil {
 		return err
 	}
 
+	fmt.Printf("Current Kubernetes cluster version: %s\n", currentVersion)
+	fmt.Printf("Latest Kubernetes version: %s\n", latestVersion)
+	fmt.Printf("Current Node version: %s\n", nodeVersionInfoUpdate.Current.KubeletVersion.String())
+	fmt.Println()
+
 	if nodeVersionInfoUpdate.IsUpdated() {
 		fmt.Printf("Node %s is up to date\n", target.Nodename)
 		return nil
 	}
 
+	// Check if the node is upgradeable (matches preconditions)
+	if err := nodeVersionInfoUpdate.NodeUpgradeableCheck(clientSet, currentClusterVersion); err != nil {
+		fmt.Println()
+		return err
+	}
+
 	// Check if skuba-update.timer is already disabled
 	skubaUpdateWasEnabled, err := target.IsServiceEnabled("skuba-update.timer")
 	if err != nil {
@@ -72,56 +79,33 @@ func Apply(clientSet clientset.Interface, target *deployments.Target) error {
 		return err
 	}
 
-	var upgradeable bool
 	var initCfgContents []byte
 
-	// Check if the target node is the first control plane to be updated
-	isFirstControlPlaneUpgrade, err := nodeVersionInfoUpdate.IsFirstControlPlaneNodeToBeUpgraded(clientSet)
+	// Check if it's the first control plane node to be upgraded
+	isFirstControlPlaneNodeToBeUpgraded, err := nodeVersionInfoUpdate.IsFirstControlPlaneNodeToBeUpgraded(clientSet)
 	if err != nil {
 		return err
 	}
-	if isFirstControlPlaneUpgrade {
-		var err error
-		upgradeable, err = kubernetes.AllWorkerNodesTolerateVersion(clientSet, nodeVersionInfoUpdate.Update.APIServerVersion)
+	if isFirstControlPlaneNodeToBeUpgraded {
+		fmt.Println("Fetching the cluster configuration...")
+
+		initCfg, err := kubeadm.GetClusterConfiguration(clientSet)
 		if err != nil {
 			return err
 		}
-		if upgradeable {
-			fmt.Println("Fetching the cluster configuration...")
-
-			initCfg, err := kubeadm.GetClusterConfiguration(clientSet)
-			if err != nil {
-				return err
-			}
-			if err := node.AddTargetInformationToInitConfigurationWithClusterVersion(target, initCfg, nodeVersionInfoUpdate.Update.APIServerVersion); err != nil {
-				return errors.Wrap(err, "error adding target information to init configuration")
-			}
-			kubeadm.UpdateClusterConfigurationWithClusterVersion(initCfg, nodeVersionInfoUpdate.Update.APIServerVersion)
-			initCfgContents, err = kubeadmconfigutil.MarshalInitConfigurationToBytes(initCfg, schema.GroupVersion{
-				Group:   "kubeadm.k8s.io",
-				Version: kubeadm.GetKubeadmApisVersion(nodeVersionInfoUpdate.Update.APIServerVersion),
-			})
-			if err != nil {
-				return err
-			}
+		if err := node.AddTargetInformationToInitConfigurationWithClusterVersion(target, initCfg, nodeVersionInfoUpdate.Update.APIServerVersion); err != nil {
+			return errors.Wrap(err, "error adding target information to init configuration")
 		}
-	} else {
-		// there is already at least one updated control plane node
-		if nodeVersionInfoUpdate.Current.IsControlPlane() {
-			upgradeable, err = kubernetes.AllWorkerNodesTolerateVersion(clientSet, currentClusterVersion)
-			if err != nil {
-				return err
-			}
-		} else {
-			// worker nodes have no preconditions, are always upgradeable
-			upgradeable = true
+		kubeadm.UpdateClusterConfigurationWithClusterVersion(initCfg, nodeVersionInfoUpdate.Update.APIServerVersion)
+		initCfgContents, err = kubeadmconfigutil.MarshalInitConfigurationToBytes(initCfg, schema.GroupVersion{
+			Group:   "kubeadm.k8s.io",
+			Version: kubeadm.GetKubeadmApisVersion(nodeVersionInfoUpdate.Update.APIServerVersion),
+		})
+		if err != nil {
+			return err
 		}
 	}
 
-	if !upgradeable {
-		return errors.Errorf("node %s cannot be upgraded yet", target.Nodename)
-	}
-
 	fmt.Printf("Performing node %s (%s) upgrade, please wait...\n", target.Nodename, target.Target)
 
 	if skubaUpdateWasEnabled {
@@ -144,7 +128,7 @@ func Apply(clientSet clientset.Interface, target *deployments.Target) error {
 			return err
 		}
 	}
-	if isFirstControlPlaneUpgrade {
+	if isFirstControlPlaneNodeToBeUpgraded {
 		err = target.Apply(deployments.UpgradeConfiguration{
 			KubeadmConfigContents: string(initCfgContents),
 		}, "kubeadm.upgrade.apply")

pkg/skuba/actions/node/upgrade/plan.go

@@ -38,7 +38,6 @@ func Plan(clientSet clientset.Interface, nodeName string) error {
 
 	fmt.Printf("Current Kubernetes cluster version: %s\n", currentClusterVersion.String())
 	fmt.Printf("Latest Kubernetes version: %s\n", kubernetes.LatestVersion().String())
-	fmt.Println()
 	fmt.Printf("Current Node version: %s\n", nodeVersionInfoUpdate.Current.KubeletVersion.String())
 	fmt.Println()
 
@@ -54,6 +53,12 @@ func Plan(clientSet clientset.Interface, nodeName string) error {
 		}
 		fmt.Printf("  - kubelet: %s -> %s\n", nodeVersionInfoUpdate.Current.KubeletVersion.String(), nodeVersionInfoUpdate.Update.KubeletVersion.String())
 		fmt.Printf("  - cri-o: %s -> %s\n", nodeVersionInfoUpdate.Current.ContainerRuntimeVersion.String(), nodeVersionInfoUpdate.Update.ContainerRuntimeVersion.String())
+
+		// Check if the node is upgradeable (matches preconditions)
+		if err := nodeVersionInfoUpdate.NodeUpgradeableCheck(clientSet, currentClusterVersion); err != nil {
+			fmt.Println()
+			return err
+		}
 	}
 
 	return nil