Add/nodeipamcontroller

cmd/aws-cloud-controller-manager/main.go

@@ -31,6 +31,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	cloudprovider "k8s.io/cloud-provider"
+	"k8s.io/cloud-provider-aws/pkg/controllers/nodeipam"
 	"k8s.io/cloud-provider-aws/pkg/controllers/tagging"
 	awsv1 "k8s.io/cloud-provider-aws/pkg/providers/v1"
 	"k8s.io/cloud-provider/app"
@@ -78,6 +79,19 @@ func main() {
 	controllerAliases := names.CCMControllerAliases()
 	controllerAliases[tagging.TaggingControllerKey] = tagging.TaggingControllerKey
 
+	nodeIpamControllerWrapper := nodeipam.ControllerWrapper{}
+	nodeIpamControllerWrapper.Options.AddFlags(fss.FlagSet("nodeipam controller"))
+
+	nodeIpamControllerConstructor := app.ControllerInitFuncConstructor{
+		InitContext: app.ControllerInitContext{
+			ClientName: nodeipam.NodeIpamControllerClientName,
+		},
+		Constructor: nodeIpamControllerWrapper.StartNodeIpamControllerWrapper,
+	}
+
+	controllerInitializers[nodeipam.NodeIpamControllerKey] = nodeIpamControllerConstructor
+	app.ControllersDisabledByDefault.Insert(nodeipam.NodeIpamControllerKey)
+
 	command := app.NewCloudControllerManagerCommand(opts, cloudInitializer, controllerInitializers, controllerAliases, fss, wait.NeverStop)
 
 	if err := command.Execute(); err != nil {

examples/existing-cluster/overlays/superset-role/cluster-role-patch.yaml

@@ -73,6 +73,7 @@ rules:
 - apiGroups:
   - ""
   resourceNames:
+  - nodeipam-controller
   - node-controller
   - service-controller
   - route-controller

go.mod

@@ -4,6 +4,7 @@ go 1.22.1
 
 require (
 	github.com/aws/aws-sdk-go v1.51.1
+	github.com/evanphx/json-patch v5.6.0+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
@@ -16,6 +17,7 @@ require (
 	k8s.io/cloud-provider v0.30.0-rc.0
 	k8s.io/code-generator v0.30.0-rc.0
 	k8s.io/component-base v0.30.0-rc.0
+	k8s.io/component-helpers v0.30.0-rc.0
 	k8s.io/controller-manager v0.30.0-rc.0
 	k8s.io/csi-translation-lib v0.30.0-rc.0
 	k8s.io/klog/v2 v2.120.1
@@ -36,7 +38,6 @@ require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
@@ -109,7 +110,6 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.30.0-rc.0 // indirect
-	k8s.io/component-helpers v0.30.0-rc.0 // indirect
 	k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
 	k8s.io/kms v0.30.0-rc.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect

hack/e2e/overlays/cluster-role-create-nodeipam-controller.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: provider-aws:nodeipam-controller-clusterrole
+  annotations:
+    resources.gardener.cloud/keep-object: "true"
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["patch","list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nodeipam-clusterrolebinding
+  annotations:
+    resources.gardener.cloud/keep-object: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: provider-aws:nodeipam-controller-clusterrole
+subjects:
+- kind: ServiceAccount
+  name: nodeipam-controller
+  namespace: kube-system

hack/e2e/overlays/cluster-role-patch-cloud-controller.yaml

@@ -0,0 +1,101 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    k8s-addon: aws-cloud-controller.addons.k8s.io
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - nodeipam-controller
+  - node-controller
+  - service-controller
+  - route-controller
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create

hack/e2e/run.sh

@@ -55,11 +55,12 @@ ZONES="${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}"
 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
 IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/provider-aws/cloud-controller-manager}
 IMAGE_TAG=${IMAGE_TAG:-${BUILD_VERSION}-${test_run_id}}
-
+DNS_ZONE="${DNS_ZONE:-example.com}"
 # Test args
 GINKGO_FOCUS=${GINKGO_FOCUS:-"\[cloud-provider-aws-e2e\]"}
 GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
 GINKGO_NODES=${GINKGO_NODES:-4}
+GINKGO_LABEL_FILTER="loadbalancer"
 
 EXPANDED_TEST_EXTRA_FLAGS="${EXPANDED_TEST_EXTRA_FLAGS:-}"
 
@@ -94,12 +95,13 @@ fi
 
 export PATH="${INSTALL_PATH}:${PATH}"
 
-echo "Starting test run ---"
+echo "Starting test run for loadbalancer ---"
 echo " + Region:              ${REGION} (${ZONES})"
 echo " + Cluster name:        ${CLUSTER_NAME}"
 echo " + Kubernetes version:  ${KUBERNETES_VERSION}"
 echo " + Focus:               ${GINKGO_FOCUS}"
 echo " + Skip:                ${GINKGO_SKIP}"
+echo " + Label Filter:        ${GINKGO_LABEL_FILTER}"
 echo " + kOps state store:    ${KOPS_STATE_STORE}"
 echo " + SSH public key path: ${SSH_PUBLIC_KEY_PATH}"
 echo " + Test run ID:         ${test_run_id}"
@@ -143,10 +145,62 @@ fi
 
 set -x
 pushd ./tests/e2e
-ginkgo . -v -p --nodes="${GINKGO_NODES}" --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" --report-dir="${ARTIFACTS}"
+ginkgo --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" --label-filter="${GINKGO_LABEL_FILTER}" . -v -p --nodes="${GINKGO_NODES}"  --report-dir="${ARTIFACTS}"
 popd
 
 if [[ "${DOWN}" = "yes" ]]; then
     # This should be changed to ${test_run}/kops once https://github.com/kubernetes/kops/pull/13217 is merged.
     ${test_run}/${test_run_id}/kops delete cluster --name "${CLUSTER_NAME}" --yes
 fi
+
+GINKGO_LABEL_FILTER="ipv6 prefix"
+
+echo "Starting test run for nodeipam controller ---"
+echo " + Region:              ${REGION} (${ZONES})"
+echo " + Cluster name:        ${CLUSTER_NAME}.${DNS_ZONE}"
+echo " + Kubernetes version:  ${KUBERNETES_VERSION}"
+echo " + Focus:               ${GINKGO_FOCUS}"
+echo " + Skip:                ${GINKGO_SKIP}"
+echo " + Label Filter:        ${GINKGO_LABEL_FILTER}"
+echo " + kOps state store:    ${KOPS_STATE_STORE}"
+echo " + SSH public key path: ${SSH_PUBLIC_KEY_PATH}"
+echo " + Test run ID:         ${test_run_id}"
+echo " + Kubetest run dir:    ${test_run}"
+echo " + Image:               ${IMAGE_NAME}:${IMAGE_TAG}"
+echo " + Create cluster:      ${UP}"
+echo " + Delete cluster:      ${DOWN}"
+
+if [[ "${UP}" = "yes" ]]; then
+    kubetest2 kops \
+      -v 2 \
+      --up \
+      --run-id="${test_run_id}" \
+      --cloud-provider=aws \
+      --cluster-name="${CLUSTER_NAME}.${DNS_ZONE}" \
+      --create-args="--dns-zone=${DNS_ZONE} --ipv6 --zones=${ZONES} --node-size=m5.large --master-size=m5.large --set cluster.spec.cloudControllerManager.cloudProvider=aws --set cluster.spec.cloudControllerManager.clusterCIDR=10.0.0.0/16 --set cluster.spec.cloudControllerManager.configureCloudRoutes=false --set cluster.spec.cloudControllerManager.controllers=cloud-node --set cluster.spec.cloudControllerManager.controllers=cloud-node-lifecycle --set cluster.spec.cloudControllerManager.controllers=nodeipam --set cluster.spec.cloudControllerManager.controllers=service --set cluster.spec.cloudControllerManager.controllers=route --set cluster.spec.cloudControllerManager.image=${IMAGE_NAME}:${IMAGE_TAG} --set cluster.spec.kubeControllerManager.configureCloudRoutes=false" \
+      --admin-access="0.0.0.0/0" \
+      --kubernetes-version="${KUBERNETES_VERSION}" \
+      --kops-version-marker=https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt \
+
+      # Use the kops tester once we have a way of consuming an arbitrary e2e.test binary.
+      #--test=kops \
+      #-- \
+      #--use-built-binaries=true \
+      #--focus-regex="${GINKGO_FOCUS}" \
+      #--parallel 25
+fi
+
+set -x
+pushd ./hack/e2e/overlays
+kubectl patch clusterrole system:cloud-controller-manager --type strategic  --patch-file cluster-role-patch-cloud-controller.yaml
+kubectl create -f cluster-role-create-nodeipam-controller.yaml
+popd
+
+pushd ./tests/e2e
+ginkgo --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" --label-filter="${GINKGO_LABEL_FILTER}" . -v -p --nodes="${GINKGO_NODES}" --report-dir="${ARTIFACTS}"
+popd
+
+if [[ "${DOWN}" = "yes" ]]; then
+    # This should be changed to ${test_run}/kops once https://github.com/kubernetes/kops/pull/13217 is merged.
+    ${test_run}/${test_run_id}/kops delete cluster --name "${CLUSTER_NAME}.${DNS_ZONE}" --yes
+fi

hack/install-e2e-tools.sh

@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-GINKGO_VERSION="${GINKGO_VERSION:-v1.14.0}"
+GINKGO_VERSION="${GINKGO_VERSION:-v2.13.2}"
 KOPS_ROOT="${KOPS_ROOT:-}"
 export GO111MODULE=on
 
@@ -31,8 +31,8 @@ cd "$(mktemp -d)" > /dev/null
 echo " + Installing kubetest2"
 go install "sigs.k8s.io/kubetest2@latest"
 
-echo " + Installing ginkgo"
-go install "github.com/onsi/ginkgo/ginkgo@${GINKGO_VERSION}"
+echo " + Installing ginkgo v2"
+go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
 
 if [[ -z "${KOPS_ROOT}" ]]; then
     git clone https://github.com/kubernetes/kops.git

pkg/controllers/nodeipam/config/types.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "net"
+
+// NodeIPAMControllerConfiguration contains elements describing NodeIPAMController.
+type NodeIPAMControllerConfiguration struct {
+	RateLimit    float64
+	BurstLimit   int
+	DualStack    bool
+	ClusterCIDRs []*net.IPNet
+	// NodeCIDRMaskSize is the mask size for node cidr in single-stack cluster.
+	// This can be used only with single stack clusters and is incompatible with dual stack clusters.
+	NodeCIDRMaskSize int32
+}