[6.1.x] Configure pod subnet size through cluster configuration resou…

…rce (#2305)

* Add pod subnet config to cluster configuration resource

* Bump planet to 6.1.43

Makefile

@@ -42,7 +42,7 @@ RELEASE_OUT ?=
 TELEPORT_TAG = 3.2.13
 # TELEPORT_REPOTAG adapts TELEPORT_TAG to the teleport tagging scheme
 TELEPORT_REPOTAG := v$(TELEPORT_TAG)
-PLANET_TAG := 6.1.42-$(K8S_VER_SUFFIX)
+PLANET_TAG := 6.1.43-$(K8S_VER_SUFFIX)
 PLANET_BRANCH := $(PLANET_TAG)
 K8S_APP_TAG := $(GRAVITY_TAG)
 TELEKUBE_APP_TAG := $(GRAVITY_TAG)

lib/ops/ops.go

@@ -1403,7 +1403,7 @@ func (r *CreateSiteInstallOperationRequest) CheckAndSetDefaults() error {
 	if r.Provisioner == schema.ProvisionerAWSTerraform {
 		r.Variables.AWS.SetDefaults()
 	}
-	err := validate.KubernetesSubnetsFromStrings(r.Variables.OnPrem.PodCIDR, r.Variables.OnPrem.ServiceCIDR)
+	err := validate.KubernetesSubnetsFromStrings(r.Variables.OnPrem.PodCIDR, r.Variables.OnPrem.ServiceCIDR, "")
 	if err != nil {
 		return trace.Wrap(err)
 	}

lib/ops/opsservice/configure.go

@@ -1397,6 +1397,9 @@ func (s *site) addClusterConfig(config clusterconfig.Interface, overrideArgs map
 	if globalConfig.PodCIDR != "" {
 		overrideArgs["pod-subnet"] = globalConfig.PodCIDR
 	}
+	if globalConfig.PodSubnetSize != "" {
+		overrideArgs["pod-subnet-size"] = globalConfig.PodSubnetSize
+	}
 	if globalConfig.ServiceNodePortRange != "" {
 		args = append(args,
 			fmt.Sprintf("--service-node-portrange=%v", globalConfig.ServiceNodePortRange),

lib/ops/resources/gravity/collection.go

@@ -667,6 +667,9 @@ func (r configCollection) WriteText(w io.Writer) error {
 	if len(config.PodCIDR) != 0 {
 		fmt.Fprintf(t, "Pod IP Range:\t%v\n", config.PodCIDR)
 	}
+	if len(config.PodSubnetSize) != 0 {
+		fmt.Fprintf(t, "Pod subnet size:\t%v\n", config.PodSubnetSize)
+	}
 	if len(config.ServiceCIDR) != 0 {
 		fmt.Fprintf(t, "Service IP Range:\t%v\n", config.ServiceCIDR)
 	}

lib/ops/resources/gravity/gravity.go

@@ -565,7 +565,7 @@ func Validate(resource storage.UnknownResource) (err error) {
 			return trace.Wrap(err)
 		}
 		globalConfig := config.GetGlobalConfig()
-		return validate.KubernetesSubnetsFromStrings(globalConfig.PodCIDR, globalConfig.ServiceCIDR)
+		return validate.KubernetesSubnetsFromStrings(globalConfig.PodCIDR, globalConfig.ServiceCIDR, globalConfig.PodSubnetSize)
 	default:
 		return trace.NotImplemented("unsupported resource %q, supported are: %v",
 			resource.Kind, modules.GetResources().SupportedResources())

lib/storage/clusterconfig/clusterconfig.go

@@ -92,7 +92,7 @@ func (r *Resource) SetExpiry(expires time.Time) {
 	r.Metadata.SetExpiry(expires)
 }
 
-// Expires returns expiration time
+// Expiry returns expiration time
 func (r *Resource) Expiry() time.Time {
 	return r.Metadata.Expiry()
 }
@@ -147,6 +147,9 @@ func (r Resource) Merge(other Resource) Resource {
 	if other.Spec.Global.ServiceCIDR != "" {
 		r.Spec.Global.ServiceCIDR = other.Spec.Global.ServiceCIDR
 	}
+	if other.Spec.Global.PodSubnetSize != "" {
+		r.Spec.Global.PodSubnetSize = other.Spec.Global.PodSubnetSize
+	}
 	if other.Spec.Global.CloudConfig != "" {
 		r.Spec.Global.CloudConfig = other.Spec.Global.CloudConfig
 	}
@@ -230,7 +233,7 @@ type Spec struct {
 	Global Global `json:"global,omitempty"`
 }
 
-// ComponentsConfigs groups component configurations
+// ComponentConfigs groups component configurations
 type ComponentConfigs struct {
 	// Kubelet defines kubelet configuration
 	Kubelet *Kubelet `json:"kubelet,omitempty"`
@@ -261,8 +264,12 @@ type ControlPlaneComponent struct {
 
 // IsEmpty determines whether this global configuration is empty.
 func (r Global) IsEmpty() bool {
-	return r.CloudConfig == "" && r.ServiceCIDR == "" && r.PodCIDR == "" &&
-		r.ServiceNodePortRange == "" && r.ProxyPortRange == "" &&
+	return r.CloudConfig == "" &&
+		r.ServiceCIDR == "" &&
+		r.PodCIDR == "" &&
+		r.PodSubnetSize == "" &&
+		r.ServiceNodePortRange == "" &&
+		r.ProxyPortRange == "" &&
 		len(r.FeatureGates) == 0
 }
 
@@ -284,6 +291,8 @@ type Global struct {
 	// PodCIDR defines the CIDR Range for Pods in cluster.
 	// Targets: controller manager, kubelet
 	PodCIDR string `json:"podCIDR,omitempty"`
+	// PodSubnetSize defines the size of the subnet allocated for each host.
+	PodSubnetSize string `json:"podSubnetSize,omitempty"`
 	// ProxyPortRange specifies the range of host ports (beginPort-endPort, single port or beginPort+offset, inclusive)
 	// that may be consumed in order to proxy service traffic.
 	// If (unspecified, 0, or 0-0) then ports will be randomly chosen.
@@ -333,6 +342,7 @@ const specSchemaTemplate = `{
             "serviceNodePortRange": {"type": "string"},
             "poxyPortRange": {"type": "string"},
             "podCIDR": {"type": "string"},
+            "podSubnetSize": {"type": "string"},
             "featureGates": {
               "type": "object",
               "patternProperties": {

lib/storage/clusterconfig/clusterconfig_test.go

@@ -168,7 +168,8 @@ spec:
   global:
     featureGates:
       FeatureA: true
-      FeatureB: false`,
+      FeatureB: false
+    podSubnetSize: "26"`,
 			resource: &Resource{
 				Kind:    storage.KindClusterConfiguration,
 				Version: "v1",
@@ -182,6 +183,7 @@ spec:
 							"FeatureA": true,
 							"FeatureB": false,
 						},
+						PodSubnetSize: "26",
 					},
 				},
 			},
@@ -232,8 +234,9 @@ func (*S) TestMergesClusterConfiguration(c *C) {
 						},
 					},
 					Global: Global{
-						PodCIDR:     "10.244.0.0/16",
-						ServiceCIDR: "100.10.0.0/16",
+						PodCIDR:       "10.244.0.0/16",
+						ServiceCIDR:   "100.10.0.0/16",
+						PodSubnetSize: "26",
 						FeatureGates: map[string]bool{
 							"feature1": true,
 							"feature2": false,
@@ -249,8 +252,9 @@ func (*S) TestMergesClusterConfiguration(c *C) {
 						},
 					},
 					Global: Global{
-						PodCIDR:     "10.244.0.0/16",
-						ServiceCIDR: "100.10.0.0/16",
+						PodCIDR:       "10.244.0.0/16",
+						ServiceCIDR:   "100.10.0.0/16",
+						PodSubnetSize: "26",
 						FeatureGates: map[string]bool{
 							"feature1": true,
 							"feature2": false,
@@ -276,6 +280,7 @@ address: 10.0.0.1
 					Global: Global{
 						PodCIDR:        "10.244.0.0/16",
 						ServiceCIDR:    "100.10.0.0/16",
+						PodSubnetSize:  "26",
 						ProxyPortRange: "8080-8081",
 						FeatureGates: map[string]bool{
 							"feature1": true,
@@ -321,6 +326,7 @@ address: 10.0.0.1
 					Global: Global{
 						PodCIDR:        "10.245.0.0/16",
 						ServiceCIDR:    "100.10.0.0/16",
+						PodSubnetSize:  "26",
 						ProxyPortRange: "8080-8081",
 						FeatureGates: map[string]bool{
 							"feature1": true,

lib/update/clusterconfig/plan.go

@@ -206,7 +206,7 @@ func shouldUpdateNodes(clusterConfig clusterconfig.Interface, numWorkerNodes int
 	var hasComponentUpdate, hasCIDRUpdate bool
 	config := clusterConfig.GetGlobalConfig()
 	hasComponentUpdate = len(config.FeatureGates) != 0
-	hasCIDRUpdate = len(config.PodCIDR) != 0 || len(config.ServiceCIDR) != 0
+	hasCIDRUpdate = len(config.PodCIDR) != 0 || len(config.ServiceCIDR) != 0 || config.PodSubnetSize != ""
 	return !clusterConfig.GetKubeletConfig().IsEmpty() || hasComponentUpdate || hasCIDRUpdate
 }
 

lib/validate/clusterconfig.go

@@ -51,6 +51,11 @@ func ClusterConfiguration(existing, update clusterconfig.Interface) error {
 	if globalConfig.CloudProvider == "" && newGlobalConfig.CloudConfig != "" {
 		return trace.BadParameter("cannot set cloud configuration: cluster does not have cloud provider configured")
 	}
+
+	podCIDRString := globalConfig.PodCIDR
+	serviceCIDRString := globalConfig.ServiceCIDR
+	podSubnetSizeString := globalConfig.PodSubnetSize
+
 	if newGlobalConfig.PodCIDR != "" {
 		_, podCIDR, err := net.ParseCIDR(newGlobalConfig.PodCIDR)
 		if err != nil {
@@ -60,18 +65,9 @@ func ClusterConfiguration(existing, update clusterconfig.Interface) error {
 			return trace.BadParameter("specified pod subnet (%v) is the same as existing pod subnet",
 				newGlobalConfig.PodCIDR)
 		}
-		serviceSubnet := newGlobalConfig.ServiceCIDR
-		if serviceSubnet == "" {
-			serviceSubnet = globalConfig.ServiceCIDR
-		}
-		_, serviceCIDR, err := net.ParseCIDR(serviceSubnet)
-		if err != nil {
-			return trace.Wrap(err, "invalid service subnet: %v", serviceSubnet)
-		}
-		if err := KubernetesSubnets(podCIDR, serviceCIDR); err != nil {
-			return trace.Wrap(err)
-		}
+		podCIDRString = newGlobalConfig.PodCIDR
 	}
+
 	if newGlobalConfig.ServiceCIDR != "" {
 		_, serviceCIDR, err := net.ParseCIDR(newGlobalConfig.ServiceCIDR)
 		if err != nil {
@@ -81,18 +77,17 @@ func ClusterConfiguration(existing, update clusterconfig.Interface) error {
 			return trace.BadParameter("specified service subnet (%v) is the same as existing service subnet",
 				newGlobalConfig.ServiceCIDR)
 		}
-		podSubnet := newGlobalConfig.PodCIDR
-		if podSubnet == "" {
-			podSubnet = globalConfig.PodCIDR
-		}
-		_, podCIDR, err := net.ParseCIDR(podSubnet)
-		if err != nil {
-			return trace.Wrap(err, "invalid pod subnet: %v", podSubnet)
-		}
-		if err := KubernetesSubnets(podCIDR, serviceCIDR); err != nil {
-			return trace.Wrap(err)
-		}
+		serviceCIDRString = newGlobalConfig.ServiceCIDR
+	}
+
+	if newGlobalConfig.PodSubnetSize != "" {
+		podSubnetSizeString = newGlobalConfig.PodSubnetSize
+	}
+
+	if err := KubernetesSubnetsFromStrings(podCIDRString, serviceCIDRString, podSubnetSizeString); err != nil {
+		return trace.Wrap(err)
 	}
+
 	return nil
 }
 

lib/validate/net.go

@@ -18,14 +18,16 @@ package validate
 
 import (
 	"net"
+	"strconv"
 
 	"github.com/gravitational/trace"
 )
 
 // KubernetesSubnetsFromStrings makes sure that the provided CIDR ranges are valid and can be used as
 // pod/service Kubernetes subnets
-func KubernetesSubnetsFromStrings(podCIDR, serviceCIDR string) error {
+func KubernetesSubnetsFromStrings(podCIDR, serviceCIDR, podSubnetSize string) error {
 	var podNet, serviceNet *net.IPNet
+	var subnetSize int
 	var err error
 
 	// make sure the pod subnet is valid
@@ -46,30 +48,55 @@ func KubernetesSubnetsFromStrings(podCIDR, serviceCIDR string) error {
 		}
 	}
 
+	// make sure podSubnetSize is valid
+	if podSubnetSize != "" {
+		subnetSize, err = strconv.Atoi(podSubnetSize)
+		if err != nil || subnetSize < 1 || subnetSize > 32 {
+			return trace.BadParameter("invalid pod subnet size: %q", podSubnetSize)
+		}
+
+		// The minimum subnet size accepted by flannel is /28:
+		// https://github.com/gravitational/flannel/blob/4485ecd41305e2401d58fca2c60e2b8873b9b255/subnet/config.go#L70-L74
+		if subnetSize > 28 {
+			return trace.BadParameter("pod subnet is too small. Minimum useful network prefix is /28: %q", podSubnetSize)
+		}
+	}
+
 	// make sure the subnets do not overlap
-	return KubernetesSubnets(podNet, serviceNet)
+	return KubernetesSubnets(podNet, serviceNet, subnetSize)
 }
 
 // KubernetesSubnets makes sure that the provided CIDR ranges can be used as
 // pod/service Kubernetes subnets
-func KubernetesSubnets(podNet, serviceNet *net.IPNet) (err error) {
-	if podNet != nil {
-		// make sure the pod subnet is valid
-		// the pod network should be /22 minimum so k8s can allocate /24 to each node (minimum 3 nodes)
-		ones, _ := podNet.Mask.Size()
-		if ones > 22 {
-			return trace.BadParameter(
-				"pod subnet should be a minimum of /22: %q", podNet.String())
-		}
+func KubernetesSubnets(podNet, serviceNet *net.IPNet, podSubnetSize int) (err error) {
+	if podNet == nil {
+		return nil
 	}
-	if podNet != nil && serviceNet != nil {
+
+	// make sure the pod subnet is valid
+	// the pod network should be /22 minimum so k8s can allocate /24 to each node (minimum 3 nodes)
+	ones, _ := podNet.Mask.Size()
+	if ones > 22 {
+		return trace.BadParameter(
+			"pod subnet should be a minimum of /22: %q", podNet.String())
+	}
+
+	if serviceNet != nil {
 		// make sure the subnets do not overlap
 		if podNet.Contains(serviceNet.IP) || serviceNet.Contains(podNet.IP) {
 			return trace.BadParameter(
 				"pod subnet %q and service subnet %q should not overlap",
 				podNet.String(), serviceNet.String())
 		}
 	}
+
+	if podSubnetSize != 0 {
+		// make sure the subnet size is smaller than the pod network CIDR range
+		if podSubnetSize < ones {
+			return trace.BadParameter("pod subnet size (%d) cannot be larger than the network CIDR range (%q)",
+				podSubnetSize, podNet.String())
+		}
+	}
 	return nil
 }
 

lib/validate/net_test.go

@@ -30,17 +30,19 @@ var _ = check.Suite(&S{})
 
 func (*S) TestValidateKubernetesSubnets(c *check.C) {
 	type testCase struct {
-		podCIDR     string
-		serviceCIDR string
-		ok          bool
-		description string
+		podCIDR       string
+		serviceCIDR   string
+		podSubnetSize string
+		ok            bool
+		description   string
 	}
 	testCases := []testCase{
 		{
-			podCIDR:     "10.244.0.0/16",
-			serviceCIDR: "10.100.0.0/16",
-			ok:          true,
-			description: "default subnets should validate",
+			podCIDR:       "10.244.0.0/16",
+			serviceCIDR:   "10.100.0.0/16",
+			podSubnetSize: "24",
+			ok:            true,
+			description:   "default subnets should validate",
 		},
 		{
 			podCIDR:     "10.244.0.0-10.244.255.0",
@@ -57,15 +59,31 @@ func (*S) TestValidateKubernetesSubnets(c *check.C) {
 			ok:          false,
 			description: "pod subnet is too small",
 		},
+		{
+			podSubnetSize: "33",
+			ok:            false,
+			description:   "pob subnet size is not valid; value cannot be > 32",
+		},
+		{
+			podSubnetSize: "0",
+			ok:            false,
+			description:   "pob subnet size is not valid; value cannot be < 1",
+		},
 		{
 			podCIDR:     "10.100.0.0/16",
 			serviceCIDR: "10.100.100.0/16",
 			ok:          false,
 			description: "pod and service subnets overlap",
 		},
+		{
+			podCIDR:       "10.100.0.0/16",
+			podSubnetSize: "14",
+			ok:            false,
+			description:   "pod subnet size is larger than the pod network CIDR range",
+		},
 	}
 	for _, tc := range testCases {
-		err := KubernetesSubnetsFromStrings(tc.podCIDR, tc.serviceCIDR)
+		err := KubernetesSubnetsFromStrings(tc.podCIDR, tc.serviceCIDR, tc.podSubnetSize)
 		if tc.ok {
 			c.Assert(err, check.IsNil, check.Commentf(tc.description))
 		} else {

tool/gravity/cli/config.go

@@ -317,7 +317,7 @@ func (i *InstallConfig) CheckAndSetDefaults(validator resources.Validator) (err
 	if err != nil {
 		return trace.Wrap(err)
 	}
-	err = validate.KubernetesSubnetsFromStrings(i.PodCIDR, i.ServiceCIDR)
+	err = validate.KubernetesSubnetsFromStrings(i.PodCIDR, i.ServiceCIDR, "")
 	if err != nil {
 		return trace.Wrap(err)
 	}