From 5657682609bc9ae52c1adf7164fa05d394d8f9ca Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Fri, 26 Mar 2021 10:41:41 +0100 Subject: [PATCH] Add dual stack support: APIs Enable ipv6 on nodes if dual-stack --- pkg/apis/config/v1alpha4/default.go | 14 +++- pkg/apis/config/v1alpha4/types.go | 2 + .../internal/create/actions/config/config.go | 7 +- pkg/cluster/internal/kubeadm/config.go | 15 ++-- .../internal/providers/docker/provision.go | 5 +- .../internal/providers/podman/provision.go | 5 +- pkg/internal/apis/config/default.go | 14 +++- pkg/internal/apis/config/types.go | 2 + pkg/internal/apis/config/validate.go | 73 ++++++++++++++++- pkg/internal/apis/config/validate_test.go | 81 +++++++++++++++++++ 10 files changed, 195 insertions(+), 23 deletions(-) diff --git a/pkg/apis/config/v1alpha4/default.go b/pkg/apis/config/v1alpha4/default.go index 3bd00f0be5..4626fdd41b 100644 --- a/pkg/apis/config/v1alpha4/default.go +++ b/pkg/apis/config/v1alpha4/default.go @@ -37,24 +37,27 @@ func SetDefaultsCluster(obj *Cluster) { SetDefaultsNode(a) } if obj.Networking.IPFamily == "" { - obj.Networking.IPFamily = "ipv4" + obj.Networking.IPFamily = IPv4Family } // default to listening on 127.0.0.1:randomPort on ipv4 // and [::1]:randomPort on ipv6 if obj.Networking.APIServerAddress == "" { obj.Networking.APIServerAddress = "127.0.0.1" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.APIServerAddress = "::1" } } // default the pod CIDR if obj.Networking.PodSubnet == "" { obj.Networking.PodSubnet = "10.244.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { // node-mask cidr default is /64 so we need a larger subnet, we use /56 following best practices // xref: https://www.ripe.net/publications/docs/ripe-690#4--size-of-end-user-prefix-assignment---48---56-or-something-else- obj.Networking.PodSubnet = "fd00:10:244::/56" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.PodSubnet = "10.244.0.0/16,fd00:10:244::/56" + } } // default the service CIDR using a different subnet than kubeadm default // https://github.com/kubernetes/kubernetes/blob/746404f82a28e55e0b76ffa7e40306fb88eb3317/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go#L32 @@ -62,9 +65,12 @@ func SetDefaultsCluster(obj *Cluster) { // we allocate a /16 subnet that allows 65535 services (current Kubernetes tested limit is O(10k) services) if obj.Networking.ServiceSubnet == "" { obj.Networking.ServiceSubnet = "10.96.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.ServiceSubnet = "fd00:10:96::/112" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.ServiceSubnet = "10.96.0.0/16,fd00:10:96::/112" + } } // default the KubeProxyMode using iptables as it's already the default if obj.Networking.KubeProxyMode == "" { diff --git a/pkg/apis/config/v1alpha4/types.go b/pkg/apis/config/v1alpha4/types.go index 1916b56250..eef673673f 100644 --- a/pkg/apis/config/v1alpha4/types.go +++ b/pkg/apis/config/v1alpha4/types.go @@ -199,6 +199,8 @@ const ( IPv4Family ClusterIPFamily = "ipv4" // IPv6Family sets ClusterIPFamily to ipv6 IPv6Family ClusterIPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual + DualStackFamily ClusterIPFamily = "dual" ) // ProxyMode defines a proxy mode for kube-proxy diff --git a/pkg/cluster/internal/create/actions/config/config.go b/pkg/cluster/internal/create/actions/config/config.go index 64beac53cc..a5ab77b200 100644 --- a/pkg/cluster/internal/create/actions/config/config.go +++ b/pkg/cluster/internal/create/actions/config/config.go @@ -78,7 +78,7 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { KubeProxyMode: string(ctx.Config.Networking.KubeProxyMode), ServiceSubnet: ctx.Config.Networking.ServiceSubnet, ControlPlane: true, - IPv6: ctx.Config.Networking.IPFamily == "ipv6", + IPv6: ctx.Config.Networking.IPFamily == config.IPv6Family, FeatureGates: ctx.Config.FeatureGates, RuntimeConfig: ctx.Config.RuntimeConfig, RootlessProvider: providerInfo.Rootless, @@ -237,11 +237,14 @@ func getKubeadmConfig(cfg *config.Cluster, data kubeadm.ConfigData, node nodes.N data.NodeAddress = nodeAddress // configure the right protocol addresses - if cfg.Networking.IPFamily == "ipv6" { + if cfg.Networking.IPFamily == config.IPv6Family || cfg.Networking.IPFamily == config.DualStackFamily { if ip := net.ParseIP(nodeAddressIPv6); ip.To16() == nil { return "", errors.Errorf("failed to get IPv6 address for node %s; is %s configured to use IPv6 correctly?", node.String(), provider) } data.NodeAddress = nodeAddressIPv6 + if cfg.Networking.IPFamily == config.DualStackFamily { + data.NodeAddress = fmt.Sprintf("%s,%s", nodeAddress, nodeAddressIPv6) + } } // generate the config contents diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index 82f97e8556..80c22586b5 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -46,7 +46,7 @@ type ConfigData struct { // ControlPlane flag specifies the node belongs to the control plane ControlPlane bool - // The main IP address of the node + // The IP address or comma separated list IP addresses of of the node NodeAddress string // The name for the node (not the address) NodeName string @@ -86,6 +86,8 @@ type ConfigData struct { // DerivedConfigData fields are automatically derived by // ConfigData.Derive if they are not specified / zero valued type DerivedConfigData struct { + // AdvertiseAddress is the first address in NodeAddress + AdvertiseAddress string // DockerStableTag is automatically derived from KubernetesVersion DockerStableTag string // SortedFeatureGateKeys allows us to iterate FeatureGates deterministically @@ -98,6 +100,9 @@ type DerivedConfigData struct { // Derive automatically derives DockerStableTag if not specified func (c *ConfigData) Derive() { + // get the first address to use it as the API advertised address + c.AdvertiseAddress = strings.Split(c.NodeAddress, ",")[0] + if c.DockerStableTag == "" { c.DockerStableTag = strings.Replace(c.KubernetesVersion, "+", "_", -1) } @@ -194,7 +199,7 @@ bootstrapTokens: # we use a well know port for making the API server discoverable inside docker network. # from the host machine such port will be accessible via a random local port instead. localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} nodeRegistration: criSocket: "/run/containerd/containerd.sock" @@ -211,7 +216,7 @@ metadata: {{ if .ControlPlane -}} controlPlane: localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} {{- end }} nodeRegistration: @@ -321,7 +326,7 @@ bootstrapTokens: # we use a well know port for making the API server discoverable inside docker network. # from the host machine such port will be accessible via a random local port instead. localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} nodeRegistration: criSocket: "unix:///run/containerd/containerd.sock" @@ -339,7 +344,7 @@ metadata: {{ if .ControlPlane -}} controlPlane: localAPIEndpoint: - advertiseAddress: "{{ .NodeAddress }}" + advertiseAddress: "{{ .AdvertiseAddress }}" bindPort: {{.APIBindPort}} {{- end }} nodeRegistration: diff --git a/pkg/cluster/internal/providers/docker/provision.go b/pkg/cluster/internal/providers/docker/provision.go index 3502cd0946..501618615e 100644 --- a/pkg/cluster/internal/providers/docker/provision.go +++ b/pkg/cluster/internal/providers/docker/provision.go @@ -63,7 +63,8 @@ func planCreation(cfg *config.Cluster, networkName string) (createContainerFuncs // For now remote docker + multi control plane is not supported apiServerPort = 0 // replaced with random ports apiServerAddress = "127.0.0.1" // only the LB needs to be non-local - if clusterIsIPv6(cfg) { + // only for IPv6 only clusters + if cfg.Networking.IPFamily == config.IPv6Family { apiServerAddress = "::1" // only the LB needs to be non-local } // plan loadbalancer node @@ -134,7 +135,7 @@ func createContainer(args []string) error { } func clusterIsIPv6(cfg *config.Cluster) bool { - return cfg.Networking.IPFamily == "ipv6" + return cfg.Networking.IPFamily == config.IPv6Family || cfg.Networking.IPFamily == config.DualStackFamily } func clusterHasImplicitLoadBalancer(cfg *config.Cluster) bool { diff --git a/pkg/cluster/internal/providers/podman/provision.go b/pkg/cluster/internal/providers/podman/provision.go index 97b9413a83..fc04fd92c2 100644 --- a/pkg/cluster/internal/providers/podman/provision.go +++ b/pkg/cluster/internal/providers/podman/provision.go @@ -51,7 +51,8 @@ func planCreation(cfg *config.Cluster, networkName string) (createContainerFuncs // For now remote podman + multi control plane is not supported apiServerPort = 0 // replaced with random ports apiServerAddress = "127.0.0.1" // only the LB needs to be non-local - if clusterIsIPv6(cfg) { + // only for IPv6 only clusters + if cfg.Networking.IPFamily == config.IPv6Family { apiServerAddress = "::1" // only the LB needs to be non-local } // plan loadbalancer node @@ -120,7 +121,7 @@ func createContainer(args []string) error { } func clusterIsIPv6(cfg *config.Cluster) bool { - return cfg.Networking.IPFamily == "ipv6" + return cfg.Networking.IPFamily == config.IPv6Family || cfg.Networking.IPFamily == config.DualStackFamily } func clusterHasImplicitLoadBalancer(cfg *config.Cluster) bool { diff --git a/pkg/internal/apis/config/default.go b/pkg/internal/apis/config/default.go index ad24ce4895..7a93df8022 100644 --- a/pkg/internal/apis/config/default.go +++ b/pkg/internal/apis/config/default.go @@ -48,14 +48,14 @@ func SetDefaultsCluster(obj *Cluster) { SetDefaultsNode(a) } if obj.Networking.IPFamily == "" { - obj.Networking.IPFamily = "ipv4" + obj.Networking.IPFamily = IPv4Family } // default to listening on 127.0.0.1:randomPort on ipv4 // and [::1]:randomPort on ipv6 if obj.Networking.APIServerAddress == "" { obj.Networking.APIServerAddress = "127.0.0.1" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.APIServerAddress = "::1" } } @@ -63,11 +63,14 @@ func SetDefaultsCluster(obj *Cluster) { // default the pod CIDR if obj.Networking.PodSubnet == "" { obj.Networking.PodSubnet = "10.244.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { // node-mask cidr default is /64 so we need a larger subnet, we use /56 following best practices // xref: https://www.ripe.net/publications/docs/ripe-690#4--size-of-end-user-prefix-assignment---48---56-or-something-else- obj.Networking.PodSubnet = "fd00:10:244::/56" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.PodSubnet = "10.244.0.0/16,fd00:10:244::/56" + } } // default the service CIDR using the kubeadm default @@ -76,9 +79,12 @@ func SetDefaultsCluster(obj *Cluster) { // we allocate a /16 subnet that allows 65535 services (current Kubernetes tested limit is O(10k) services) if obj.Networking.ServiceSubnet == "" { obj.Networking.ServiceSubnet = "10.96.0.0/16" - if obj.Networking.IPFamily == "ipv6" { + if obj.Networking.IPFamily == IPv6Family { obj.Networking.ServiceSubnet = "fd00:10:96::/112" } + if obj.Networking.IPFamily == DualStackFamily { + obj.Networking.ServiceSubnet = "10.96.0.0/16,fd00:10:96::/112" + } } // default the KubeProxyMode using iptables as it's already the default if obj.Networking.KubeProxyMode == "" { diff --git a/pkg/internal/apis/config/types.go b/pkg/internal/apis/config/types.go index 7c9786b1ab..36ef76eacd 100644 --- a/pkg/internal/apis/config/types.go +++ b/pkg/internal/apis/config/types.go @@ -160,6 +160,8 @@ const ( IPv4Family ClusterIPFamily = "ipv4" // IPv6Family sets ClusterIPFamily to ipv6 IPv6Family ClusterIPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual + DualStackFamily ClusterIPFamily = "dual" ) // ProxyMode defines a proxy mode for kube-proxy diff --git a/pkg/internal/apis/config/validate.go b/pkg/internal/apis/config/validate.go index ed9c4e2c84..59c8cf5cd4 100644 --- a/pkg/internal/apis/config/validate.go +++ b/pkg/internal/apis/config/validate.go @@ -17,8 +17,10 @@ limitations under the License. package config import ( + "fmt" "net" "regexp" + "strings" "sigs.k8s.io/kind/pkg/errors" ) @@ -49,13 +51,15 @@ func (c *Cluster) Validate() error { } } + isDualStack := c.Networking.IPFamily == DualStackFamily // podSubnet should be a valid CIDR - if _, _, err := net.ParseCIDR(c.Networking.PodSubnet); err != nil { - errs = append(errs, errors.Wrapf(err, "invalid podSubnet")) + if err := validateSubnets(c.Networking.PodSubnet, isDualStack); err != nil { + errs = append(errs, errors.Errorf("invalid pod subnet %v", err)) } + // serviceSubnet should be a valid CIDR - if _, _, err := net.ParseCIDR(c.Networking.ServiceSubnet); err != nil { - errs = append(errs, errors.Wrapf(err, "invalid serviceSubnet")) + if err := validateSubnets(c.Networking.ServiceSubnet, isDualStack); err != nil { + errs = append(errs, errors.Errorf("invalid service subnet %v", err)) } // KubeProxyMode should be iptables or ipvs @@ -135,3 +139,64 @@ func validatePort(port int32) error { } return nil } + +func validateSubnets(subnetStr string, dualstack bool) error { + allErrs := []error{} + + cidrsString := strings.Split(subnetStr, ",") + subnets := make([]*net.IPNet, 0, len(cidrsString)) + for _, cidrString := range cidrsString { + _, cidr, err := net.ParseCIDR(cidrString) + if err != nil { + return fmt.Errorf("failed to parse cidr value:%q with error: %v", cidrString, err) + } + subnets = append(subnets, cidr) + } + + switch { + // if DualStack only 2 CIDRs allowed + case dualstack && len(subnets) > 2: + allErrs = append(allErrs, errors.New("expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking")) + // if DualStack and there are 2 CIDRs validate if there is at least one of each IP family + case dualstack && len(subnets) == 2: + areDualStackCIDRs, err := isDualStackCIDRs(subnets) + if err != nil { + allErrs = append(allErrs, err) + } else if !areDualStackCIDRs { + allErrs = append(allErrs, errors.New("expected one (IPv4 or IPv6) CIDR or two CIDRs from each family for dual-stack networking")) + } + // if not DualStack only one CIDR allowed + case !dualstack && len(subnets) > 1: + allErrs = append(allErrs, errors.New("only one CIDR allowed for single-stack networking")) + } + + if len(allErrs) > 0 { + return errors.NewAggregate(allErrs) + } + return nil +} + +// isDualStackCIDRs returns if +// - all are valid cidrs +// - at least one cidr from each family (v4 or v6) +func isDualStackCIDRs(cidrs []*net.IPNet) (bool, error) { + v4Found := false + v6Found := false + for _, cidr := range cidrs { + if cidr == nil { + return false, fmt.Errorf("cidr %v is invalid", cidr) + } + + if v4Found && v6Found { + continue + } + + if cidr.IP != nil && cidr.IP.To4() == nil { + v6Found = true + continue + } + v4Found = true + } + + return v4Found && v6Found, nil +} diff --git a/pkg/internal/apis/config/validate_test.go b/pkg/internal/apis/config/validate_test.go index 5e46aa7397..9f69331f59 100644 --- a/pkg/internal/apis/config/validate_test.go +++ b/pkg/internal/apis/config/validate_test.go @@ -95,6 +95,87 @@ func TestClusterValidate(t *testing.T) { }(), ExpectErrors: 1, }, + { + Name: "bogus serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.ServiceSubnet = "aa" + return c + }(), + ExpectErrors: 1, + }, + { + Name: "invalid number of podSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,2.2.2.0/24" + return c + }(), + ExpectErrors: 1, + }, + { + Name: "valid dual stack podSubnet and serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.ServiceSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 0, + }, + { + Name: "invalid dual stack podSubnet and multiple serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.ServiceSubnet = "192.168.0.2/24,fd00:1::/25,10.0.0.0/16" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 1, + }, + { + Name: "valid dual stack podSubnet and single stack serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.ServiceSubnet = "192.168.0.2/24" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 0, + }, + { + Name: "valid dual stack serviceSubnet and single stack podSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24" + c.Networking.ServiceSubnet = "192.168.0.2/24,fd00:1::/25" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 0, + }, + + { + Name: "bad dual stack podSubnet and serviceSubnet", + Cluster: func() Cluster { + c := Cluster{} + SetDefaultsCluster(&c) + c.Networking.PodSubnet = "192.168.0.2/24,2.2.2.0/25" + c.Networking.ServiceSubnet = "192.168.0.2/24,2.2.2.0/25" + c.Networking.IPFamily = DualStackFamily + return c + }(), + ExpectErrors: 2, + }, { Name: "missing control-plane", Cluster: func() Cluster {