From ad876672fe7ac3dca0cd99f1c7abd6edd32f8fbf Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Thu, 5 Nov 2020 13:11:34 -0800 Subject: [PATCH] [6.1.x] Configure pod subnet size through cluster configuration resource (#2305) * Add pod subnet config to cluster configuration resource * Bump planet to 6.1.43 --- Makefile | 2 +- lib/ops/ops.go | 2 +- lib/ops/opsservice/configure.go | 3 ++ lib/ops/resources/gravity/collection.go | 3 ++ lib/ops/resources/gravity/gravity.go | 2 +- lib/storage/clusterconfig/clusterconfig.go | 18 +++++-- .../clusterconfig/clusterconfig_test.go | 16 ++++-- lib/update/clusterconfig/plan.go | 2 +- lib/validate/clusterconfig.go | 39 +++++++------- lib/validate/net.go | 51 ++++++++++++++----- lib/validate/net_test.go | 36 +++++++++---- tool/gravity/cli/config.go | 2 +- tool/gravity/cli/config_test.go | 2 + 13 files changed, 121 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index a5e1e4ea3e..45477dafbd 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ RELEASE_OUT ?= TELEPORT_TAG = 3.2.13 # TELEPORT_REPOTAG adapts TELEPORT_TAG to the teleport tagging scheme TELEPORT_REPOTAG := v$(TELEPORT_TAG) -PLANET_TAG := 6.1.42-$(K8S_VER_SUFFIX) +PLANET_TAG := 6.1.43-$(K8S_VER_SUFFIX) PLANET_BRANCH := $(PLANET_TAG) K8S_APP_TAG := $(GRAVITY_TAG) TELEKUBE_APP_TAG := $(GRAVITY_TAG) diff --git a/lib/ops/ops.go b/lib/ops/ops.go index 883ebff8f1..c92bb2dbc9 100644 --- a/lib/ops/ops.go +++ b/lib/ops/ops.go @@ -1403,7 +1403,7 @@ func (r *CreateSiteInstallOperationRequest) CheckAndSetDefaults() error { if r.Provisioner == schema.ProvisionerAWSTerraform { r.Variables.AWS.SetDefaults() } - err := validate.KubernetesSubnetsFromStrings(r.Variables.OnPrem.PodCIDR, r.Variables.OnPrem.ServiceCIDR) + err := validate.KubernetesSubnetsFromStrings(r.Variables.OnPrem.PodCIDR, r.Variables.OnPrem.ServiceCIDR, "") if err != nil { return trace.Wrap(err) } diff --git a/lib/ops/opsservice/configure.go b/lib/ops/opsservice/configure.go index b234a9a19a..87cd1286e9 100644 --- a/lib/ops/opsservice/configure.go +++ b/lib/ops/opsservice/configure.go @@ -1397,6 +1397,9 @@ func (s *site) addClusterConfig(config clusterconfig.Interface, overrideArgs map if globalConfig.PodCIDR != "" { overrideArgs["pod-subnet"] = globalConfig.PodCIDR } + if globalConfig.PodSubnetSize != "" { + overrideArgs["pod-subnet-size"] = globalConfig.PodSubnetSize + } if globalConfig.ServiceNodePortRange != "" { args = append(args, fmt.Sprintf("--service-node-portrange=%v", globalConfig.ServiceNodePortRange), diff --git a/lib/ops/resources/gravity/collection.go b/lib/ops/resources/gravity/collection.go index 01de1c6e68..3e8f1c61d0 100644 --- a/lib/ops/resources/gravity/collection.go +++ b/lib/ops/resources/gravity/collection.go @@ -667,6 +667,9 @@ func (r configCollection) WriteText(w io.Writer) error { if len(config.PodCIDR) != 0 { fmt.Fprintf(t, "Pod IP Range:\t%v\n", config.PodCIDR) } + if len(config.PodSubnetSize) != 0 { + fmt.Fprintf(t, "Pod subnet size:\t%v\n", config.PodSubnetSize) + } if len(config.ServiceCIDR) != 0 { fmt.Fprintf(t, "Service IP Range:\t%v\n", config.ServiceCIDR) } diff --git a/lib/ops/resources/gravity/gravity.go b/lib/ops/resources/gravity/gravity.go index b31ea4efc5..5156d5e30e 100644 --- a/lib/ops/resources/gravity/gravity.go +++ b/lib/ops/resources/gravity/gravity.go @@ -565,7 +565,7 @@ func Validate(resource storage.UnknownResource) (err error) { return trace.Wrap(err) } globalConfig := config.GetGlobalConfig() - return validate.KubernetesSubnetsFromStrings(globalConfig.PodCIDR, globalConfig.ServiceCIDR) + return validate.KubernetesSubnetsFromStrings(globalConfig.PodCIDR, globalConfig.ServiceCIDR, globalConfig.PodSubnetSize) default: return trace.NotImplemented("unsupported resource %q, supported are: %v", resource.Kind, modules.GetResources().SupportedResources()) diff --git a/lib/storage/clusterconfig/clusterconfig.go b/lib/storage/clusterconfig/clusterconfig.go index 04c1265177..8df54ea537 100644 --- a/lib/storage/clusterconfig/clusterconfig.go +++ b/lib/storage/clusterconfig/clusterconfig.go @@ -92,7 +92,7 @@ func (r *Resource) SetExpiry(expires time.Time) { r.Metadata.SetExpiry(expires) } -// Expires returns expiration time +// Expiry returns expiration time func (r *Resource) Expiry() time.Time { return r.Metadata.Expiry() } @@ -147,6 +147,9 @@ func (r Resource) Merge(other Resource) Resource { if other.Spec.Global.ServiceCIDR != "" { r.Spec.Global.ServiceCIDR = other.Spec.Global.ServiceCIDR } + if other.Spec.Global.PodSubnetSize != "" { + r.Spec.Global.PodSubnetSize = other.Spec.Global.PodSubnetSize + } if other.Spec.Global.CloudConfig != "" { r.Spec.Global.CloudConfig = other.Spec.Global.CloudConfig } @@ -230,7 +233,7 @@ type Spec struct { Global Global `json:"global,omitempty"` } -// ComponentsConfigs groups component configurations +// ComponentConfigs groups component configurations type ComponentConfigs struct { // Kubelet defines kubelet configuration Kubelet *Kubelet `json:"kubelet,omitempty"` @@ -261,8 +264,12 @@ type ControlPlaneComponent struct { // IsEmpty determines whether this global configuration is empty. func (r Global) IsEmpty() bool { - return r.CloudConfig == "" && r.ServiceCIDR == "" && r.PodCIDR == "" && - r.ServiceNodePortRange == "" && r.ProxyPortRange == "" && + return r.CloudConfig == "" && + r.ServiceCIDR == "" && + r.PodCIDR == "" && + r.PodSubnetSize == "" && + r.ServiceNodePortRange == "" && + r.ProxyPortRange == "" && len(r.FeatureGates) == 0 } @@ -284,6 +291,8 @@ type Global struct { // PodCIDR defines the CIDR Range for Pods in cluster. // Targets: controller manager, kubelet PodCIDR string `json:"podCIDR,omitempty"` + // PodSubnetSize defines the size of the subnet allocated for each host. + PodSubnetSize string `json:"podSubnetSize,omitempty"` // ProxyPortRange specifies the range of host ports (beginPort-endPort, single port or beginPort+offset, inclusive) // that may be consumed in order to proxy service traffic. // If (unspecified, 0, or 0-0) then ports will be randomly chosen. @@ -333,6 +342,7 @@ const specSchemaTemplate = `{ "serviceNodePortRange": {"type": "string"}, "poxyPortRange": {"type": "string"}, "podCIDR": {"type": "string"}, + "podSubnetSize": {"type": "string"}, "featureGates": { "type": "object", "patternProperties": { diff --git a/lib/storage/clusterconfig/clusterconfig_test.go b/lib/storage/clusterconfig/clusterconfig_test.go index 74973e2097..48614fb235 100644 --- a/lib/storage/clusterconfig/clusterconfig_test.go +++ b/lib/storage/clusterconfig/clusterconfig_test.go @@ -168,7 +168,8 @@ spec: global: featureGates: FeatureA: true - FeatureB: false`, + FeatureB: false + podSubnetSize: "26"`, resource: &Resource{ Kind: storage.KindClusterConfiguration, Version: "v1", @@ -182,6 +183,7 @@ spec: "FeatureA": true, "FeatureB": false, }, + PodSubnetSize: "26", }, }, }, @@ -232,8 +234,9 @@ func (*S) TestMergesClusterConfiguration(c *C) { }, }, Global: Global{ - PodCIDR: "10.244.0.0/16", - ServiceCIDR: "100.10.0.0/16", + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "100.10.0.0/16", + PodSubnetSize: "26", FeatureGates: map[string]bool{ "feature1": true, "feature2": false, @@ -249,8 +252,9 @@ func (*S) TestMergesClusterConfiguration(c *C) { }, }, Global: Global{ - PodCIDR: "10.244.0.0/16", - ServiceCIDR: "100.10.0.0/16", + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "100.10.0.0/16", + PodSubnetSize: "26", FeatureGates: map[string]bool{ "feature1": true, "feature2": false, @@ -276,6 +280,7 @@ address: 10.0.0.1 Global: Global{ PodCIDR: "10.244.0.0/16", ServiceCIDR: "100.10.0.0/16", + PodSubnetSize: "26", ProxyPortRange: "8080-8081", FeatureGates: map[string]bool{ "feature1": true, @@ -321,6 +326,7 @@ address: 10.0.0.1 Global: Global{ PodCIDR: "10.245.0.0/16", ServiceCIDR: "100.10.0.0/16", + PodSubnetSize: "26", ProxyPortRange: "8080-8081", FeatureGates: map[string]bool{ "feature1": true, diff --git a/lib/update/clusterconfig/plan.go b/lib/update/clusterconfig/plan.go index 5a89b1e33b..04de65b079 100644 --- a/lib/update/clusterconfig/plan.go +++ b/lib/update/clusterconfig/plan.go @@ -206,7 +206,7 @@ func shouldUpdateNodes(clusterConfig clusterconfig.Interface, numWorkerNodes int var hasComponentUpdate, hasCIDRUpdate bool config := clusterConfig.GetGlobalConfig() hasComponentUpdate = len(config.FeatureGates) != 0 - hasCIDRUpdate = len(config.PodCIDR) != 0 || len(config.ServiceCIDR) != 0 + hasCIDRUpdate = len(config.PodCIDR) != 0 || len(config.ServiceCIDR) != 0 || config.PodSubnetSize != "" return !clusterConfig.GetKubeletConfig().IsEmpty() || hasComponentUpdate || hasCIDRUpdate } diff --git a/lib/validate/clusterconfig.go b/lib/validate/clusterconfig.go index b36a1195d4..6ec97c840a 100644 --- a/lib/validate/clusterconfig.go +++ b/lib/validate/clusterconfig.go @@ -51,6 +51,11 @@ func ClusterConfiguration(existing, update clusterconfig.Interface) error { if globalConfig.CloudProvider == "" && newGlobalConfig.CloudConfig != "" { return trace.BadParameter("cannot set cloud configuration: cluster does not have cloud provider configured") } + + podCIDRString := globalConfig.PodCIDR + serviceCIDRString := globalConfig.ServiceCIDR + podSubnetSizeString := globalConfig.PodSubnetSize + if newGlobalConfig.PodCIDR != "" { _, podCIDR, err := net.ParseCIDR(newGlobalConfig.PodCIDR) if err != nil { @@ -60,18 +65,9 @@ func ClusterConfiguration(existing, update clusterconfig.Interface) error { return trace.BadParameter("specified pod subnet (%v) is the same as existing pod subnet", newGlobalConfig.PodCIDR) } - serviceSubnet := newGlobalConfig.ServiceCIDR - if serviceSubnet == "" { - serviceSubnet = globalConfig.ServiceCIDR - } - _, serviceCIDR, err := net.ParseCIDR(serviceSubnet) - if err != nil { - return trace.Wrap(err, "invalid service subnet: %v", serviceSubnet) - } - if err := KubernetesSubnets(podCIDR, serviceCIDR); err != nil { - return trace.Wrap(err) - } + podCIDRString = newGlobalConfig.PodCIDR } + if newGlobalConfig.ServiceCIDR != "" { _, serviceCIDR, err := net.ParseCIDR(newGlobalConfig.ServiceCIDR) if err != nil { @@ -81,18 +77,17 @@ func ClusterConfiguration(existing, update clusterconfig.Interface) error { return trace.BadParameter("specified service subnet (%v) is the same as existing service subnet", newGlobalConfig.ServiceCIDR) } - podSubnet := newGlobalConfig.PodCIDR - if podSubnet == "" { - podSubnet = globalConfig.PodCIDR - } - _, podCIDR, err := net.ParseCIDR(podSubnet) - if err != nil { - return trace.Wrap(err, "invalid pod subnet: %v", podSubnet) - } - if err := KubernetesSubnets(podCIDR, serviceCIDR); err != nil { - return trace.Wrap(err) - } + serviceCIDRString = newGlobalConfig.ServiceCIDR + } + + if newGlobalConfig.PodSubnetSize != "" { + podSubnetSizeString = newGlobalConfig.PodSubnetSize + } + + if err := KubernetesSubnetsFromStrings(podCIDRString, serviceCIDRString, podSubnetSizeString); err != nil { + return trace.Wrap(err) } + return nil } diff --git a/lib/validate/net.go b/lib/validate/net.go index e60a1996e1..29fb5e23d0 100644 --- a/lib/validate/net.go +++ b/lib/validate/net.go @@ -18,14 +18,16 @@ package validate import ( "net" + "strconv" "github.com/gravitational/trace" ) // KubernetesSubnetsFromStrings makes sure that the provided CIDR ranges are valid and can be used as // pod/service Kubernetes subnets -func KubernetesSubnetsFromStrings(podCIDR, serviceCIDR string) error { +func KubernetesSubnetsFromStrings(podCIDR, serviceCIDR, podSubnetSize string) error { var podNet, serviceNet *net.IPNet + var subnetSize int var err error // make sure the pod subnet is valid @@ -46,23 +48,40 @@ func KubernetesSubnetsFromStrings(podCIDR, serviceCIDR string) error { } } + // make sure podSubnetSize is valid + if podSubnetSize != "" { + subnetSize, err = strconv.Atoi(podSubnetSize) + if err != nil || subnetSize < 1 || subnetSize > 32 { + return trace.BadParameter("invalid pod subnet size: %q", podSubnetSize) + } + + // The minimum subnet size accepted by flannel is /28: + // https://github.com/gravitational/flannel/blob/4485ecd41305e2401d58fca2c60e2b8873b9b255/subnet/config.go#L70-L74 + if subnetSize > 28 { + return trace.BadParameter("pod subnet is too small. Minimum useful network prefix is /28: %q", podSubnetSize) + } + } + // make sure the subnets do not overlap - return KubernetesSubnets(podNet, serviceNet) + return KubernetesSubnets(podNet, serviceNet, subnetSize) } // KubernetesSubnets makes sure that the provided CIDR ranges can be used as // pod/service Kubernetes subnets -func KubernetesSubnets(podNet, serviceNet *net.IPNet) (err error) { - if podNet != nil { - // make sure the pod subnet is valid - // the pod network should be /22 minimum so k8s can allocate /24 to each node (minimum 3 nodes) - ones, _ := podNet.Mask.Size() - if ones > 22 { - return trace.BadParameter( - "pod subnet should be a minimum of /22: %q", podNet.String()) - } +func KubernetesSubnets(podNet, serviceNet *net.IPNet, podSubnetSize int) (err error) { + if podNet == nil { + return nil } - if podNet != nil && serviceNet != nil { + + // make sure the pod subnet is valid + // the pod network should be /22 minimum so k8s can allocate /24 to each node (minimum 3 nodes) + ones, _ := podNet.Mask.Size() + if ones > 22 { + return trace.BadParameter( + "pod subnet should be a minimum of /22: %q", podNet.String()) + } + + if serviceNet != nil { // make sure the subnets do not overlap if podNet.Contains(serviceNet.IP) || serviceNet.Contains(podNet.IP) { return trace.BadParameter( @@ -70,6 +89,14 @@ func KubernetesSubnets(podNet, serviceNet *net.IPNet) (err error) { podNet.String(), serviceNet.String()) } } + + if podSubnetSize != 0 { + // make sure the subnet size is smaller than the pod network CIDR range + if podSubnetSize < ones { + return trace.BadParameter("pod subnet size (%d) cannot be larger than the network CIDR range (%q)", + podSubnetSize, podNet.String()) + } + } return nil } diff --git a/lib/validate/net_test.go b/lib/validate/net_test.go index c1d3b7d208..1c19c9652e 100644 --- a/lib/validate/net_test.go +++ b/lib/validate/net_test.go @@ -30,17 +30,19 @@ var _ = check.Suite(&S{}) func (*S) TestValidateKubernetesSubnets(c *check.C) { type testCase struct { - podCIDR string - serviceCIDR string - ok bool - description string + podCIDR string + serviceCIDR string + podSubnetSize string + ok bool + description string } testCases := []testCase{ { - podCIDR: "10.244.0.0/16", - serviceCIDR: "10.100.0.0/16", - ok: true, - description: "default subnets should validate", + podCIDR: "10.244.0.0/16", + serviceCIDR: "10.100.0.0/16", + podSubnetSize: "24", + ok: true, + description: "default subnets should validate", }, { podCIDR: "10.244.0.0-10.244.255.0", @@ -57,15 +59,31 @@ func (*S) TestValidateKubernetesSubnets(c *check.C) { ok: false, description: "pod subnet is too small", }, + { + podSubnetSize: "33", + ok: false, + description: "pob subnet size is not valid; value cannot be > 32", + }, + { + podSubnetSize: "0", + ok: false, + description: "pob subnet size is not valid; value cannot be < 1", + }, { podCIDR: "10.100.0.0/16", serviceCIDR: "10.100.100.0/16", ok: false, description: "pod and service subnets overlap", }, + { + podCIDR: "10.100.0.0/16", + podSubnetSize: "14", + ok: false, + description: "pod subnet size is larger than the pod network CIDR range", + }, } for _, tc := range testCases { - err := KubernetesSubnetsFromStrings(tc.podCIDR, tc.serviceCIDR) + err := KubernetesSubnetsFromStrings(tc.podCIDR, tc.serviceCIDR, tc.podSubnetSize) if tc.ok { c.Assert(err, check.IsNil, check.Commentf(tc.description)) } else { diff --git a/tool/gravity/cli/config.go b/tool/gravity/cli/config.go index b2fbe9f51b..d1942423dc 100644 --- a/tool/gravity/cli/config.go +++ b/tool/gravity/cli/config.go @@ -317,7 +317,7 @@ func (i *InstallConfig) CheckAndSetDefaults(validator resources.Validator) (err if err != nil { return trace.Wrap(err) } - err = validate.KubernetesSubnetsFromStrings(i.PodCIDR, i.ServiceCIDR) + err = validate.KubernetesSubnetsFromStrings(i.PodCIDR, i.ServiceCIDR, "") if err != nil { return trace.Wrap(err) } diff --git a/tool/gravity/cli/config_test.go b/tool/gravity/cli/config_test.go index 6ae43217ad..77e9889ba7 100644 --- a/tool/gravity/cli/config_test.go +++ b/tool/gravity/cli/config_test.go @@ -48,6 +48,7 @@ spec: global: cloudProvider: gce serviceCIDR: "100.200.0.0/16" + podSubnetSize: "26" cloudConfig: | [global] node-tags=example-cluster @@ -77,6 +78,7 @@ spec: CloudProvider: "gce", ServiceCIDR: "100.200.0.0/16", PodCIDR: "100.96.0.0/16", + PodSubnetSize: "26", CloudConfig: `[global] node-tags=example-cluster multizone="true"