diff --git a/Makefile b/Makefile index d6914d5a3..c0348642b 100644 --- a/Makefile +++ b/Makefile @@ -406,8 +406,10 @@ bundle-push: ## Push the bundle image. .PHONY: protoc PROTOC = $(shell pwd)/bin/proto/bin/protoc +# map Go arch names to the ones used by protobuf release artifacts +PROTOC_ARCH = $(shell go env GOARCH | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch_64/') protoc: protoc-gen-go protoc-gen-go-grpc ## Download protoc (protocol buffers tool needed for gRPC) - test -f ${PROTOC} || (cd $(shell pwd)/bin/proto && curl -sSLo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v3.16.0/protoc-3.16.0-linux-x86_64.zip && unzip protoc.zip && rm protoc.zip) + test -f ${PROTOC} || (cd $(shell pwd)/bin/proto && curl -sSLo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v3.16.0/protoc-3.16.0-linux-$(PROTOC_ARCH).zip && unzip -o protoc.zip && rm protoc.zip) .PHONY: protoc-gen-go PROTOC_GEN_GO = $(shell pwd)/bin/proto/bin/protoc-gen-go diff --git a/api/v1alpha1/selfnoderemediationconfig_types.go b/api/v1alpha1/selfnoderemediationconfig_types.go index 69cad26ba..4d2d8853f 100644 --- a/api/v1alpha1/selfnoderemediationconfig_types.go +++ b/api/v1alpha1/selfnoderemediationconfig_types.go @@ -119,6 +119,15 @@ type SelfNodeRemediationConfigSpec struct { // +optional EndpointHealthCheckUrl string `json:"endpointHealthCheckUrl,omitempty"` + // PreferredAddressTypes is a list of node address types, that self node remediation agents which run on control-plane nodes will use to try to access the kubelet if they can't contact their peers. + // Takes the values accepted on `node.status.addresses.type`, or the special value `"NodeName"`, in which case the node's `metadata.name` will be used. + // Addresses are currently only read on startup, so a pod restart is required if node addresses change. + // This is a part of self diagnostics which will decide whether the node should be remediated or not. + // If empty, it will be equivalent to ["NodeName"] (the previous behaviour). + // +optional + // +kubebuilder:validation:items:Enum=NodeName;Hostname;InternalDNS;ExternalDNS;InternalIP;ExternalIP + PreferredAddressTypes []string `json:"preferredAddressTypes,omitempty"` + // HostPort is used for internal communication between SNR agents. // +kubebuilder:default:=30001 // +kubebuilder:validation:Minimum=1 diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1c2bea6d0..51b13ac9d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -150,6 +150,11 @@ func (in *SelfNodeRemediationConfigSpec) DeepCopyInto(out *SelfNodeRemediationCo *out = new(v1.Duration) **out = **in } + if in.PreferredAddressTypes != nil { + in, out := &in.PreferredAddressTypes, &out.PreferredAddressTypes + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.CustomDsTolerations != nil { in, out := &in.CustomDsTolerations, &out.CustomDsTolerations *out = make([]corev1.Toleration, len(*in)) diff --git a/bundle/manifests/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml b/bundle/manifests/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml index 42762f99c..abbc85043 100644 --- a/bundle/manifests/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml +++ b/bundle/manifests/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml @@ -199,6 +199,22 @@ spec: Valid time units are "ms", "s", "m", "h". pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ type: string + preferredAddressTypes: + description: |- + PreferredAddressTypes is a list of node address types, that self node remediation agents which run on control-plane nodes will use to try to access the kubelet if they can't contact their peers. + Takes the values accepted on `node.status.addresses.type`, or the special value `"NodeName"`, in which case the node's `metadata.name` will be used. + This is a part of self diagnostics which will decide whether the node should be remediated or not. + If empty, it will be equivalent to ["NodeName"] (the previous behaviour). + items: + enum: + - NodeName + - Hostname + - InternalDNS + - ExternalDNS + - InternalIP + - ExternalIP + type: string + type: array safeTimeToAssumeNodeRebootedSeconds: description: |- SafeTimeToAssumeNodeRebootedSeconds is the time after which the healthy self node remediation diff --git a/config/crd/bases/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml b/config/crd/bases/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml index 19ed57626..2a3dc6d79 100644 --- a/config/crd/bases/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml +++ b/config/crd/bases/self-node-remediation.medik8s.io_selfnoderemediationconfigs.yaml @@ -197,6 +197,22 @@ spec: Valid time units are "ms", "s", "m", "h". pattern: ^([0-9]+(\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$ type: string + preferredAddressTypes: + description: |- + PreferredAddressTypes is a list of node address types, that self node remediation agents which run on control-plane nodes will use to try to access the kubelet if they can't contact their peers. + Takes the values accepted on `node.status.addresses.type`, or the special value `"NodeName"`, in which case the node's `metadata.name` will be used. + This is a part of self diagnostics which will decide whether the node should be remediated or not. + If empty, it will be equivalent to ["NodeName"] (the previous behaviour). + items: + enum: + - NodeName + - Hostname + - InternalDNS + - ExternalDNS + - InternalIP + - ExternalIP + type: string + type: array safeTimeToAssumeNodeRebootedSeconds: description: |- SafeTimeToAssumeNodeRebootedSeconds is the time after which the healthy self node remediation diff --git a/install/self-node-remediation-deamonset.yaml b/install/self-node-remediation-deamonset.yaml index fcf0099fc..a396555de 100644 --- a/install/self-node-remediation-deamonset.yaml +++ b/install/self-node-remediation-deamonset.yaml @@ -70,6 +70,8 @@ spec: value: {{.IsSoftwareRebootEnabled}} - name: END_POINT_HEALTH_CHECK_URL value: {{.EndpointHealthCheckUrl}} + - name: PREFERRED_ADDRESS_TYPES + value: {{.PreferredAddressTypes | join ","}} - name: HOST_PORT value: "{{.HostPort}}" - name: MIN_PEERS_FOR_REMEDIATION @@ -108,4 +110,4 @@ spec: effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" operator: "Exists" - effect: "NoSchedule" \ No newline at end of file + effect: "NoSchedule" diff --git a/internal/controller/selfnoderemediationconfig_controller.go b/internal/controller/selfnoderemediationconfig_controller.go index 7e49c1c84..0d5fc36c5 100644 --- a/internal/controller/selfnoderemediationconfig_controller.go +++ b/internal/controller/selfnoderemediationconfig_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "os" + "strings" "time" "github.com/go-logr/logr" @@ -133,6 +134,10 @@ func (r *SelfNodeRemediationConfigReconciler) SetupWithManager(mgr ctrl.Manager) Complete(r) } +func join(sep string, s []string) string { + return strings.Join(s, sep) +} + func (r *SelfNodeRemediationConfigReconciler) syncConfigDaemonSet(ctx context.Context, snrConfig *selfnoderemediationv1alpha1.SelfNodeRemediationConfig) error { logger := r.Log.WithName("syncConfigDaemonset") logger.Info("Start to sync config daemonset") @@ -155,10 +160,13 @@ func (r *SelfNodeRemediationConfigReconciler) syncConfigDaemonSet(ctx context.Co data.Data["PeerRequestTimeout"] = snrConfig.Spec.PeerRequestTimeout.Nanoseconds() data.Data["MaxApiErrorThreshold"] = snrConfig.Spec.MaxApiErrorThreshold data.Data["EndpointHealthCheckUrl"] = snrConfig.Spec.EndpointHealthCheckUrl + data.Data["PreferredAddressTypes"] = snrConfig.Spec.PreferredAddressTypes data.Data["MinPeersForRemediation"] = snrConfig.Spec.MinPeersForRemediation data.Data["HostPort"] = snrConfig.Spec.HostPort data.Data["IsSoftwareRebootEnabled"] = fmt.Sprintf("\"%t\"", snrConfig.Spec.IsSoftwareRebootEnabled) + data.Funcs["join"] = join + objs, err := render.Dir(r.InstallFileFolder, &data) if err != nil { logger.Error(err, "Fail to render config daemon manifests") diff --git a/internal/controller/tests/config/selfnoderemediationconfig_controller_test.go b/internal/controller/tests/config/selfnoderemediationconfig_controller_test.go index e2e84367f..2932aa460 100644 --- a/internal/controller/tests/config/selfnoderemediationconfig_controller_test.go +++ b/internal/controller/tests/config/selfnoderemediationconfig_controller_test.go @@ -110,6 +110,7 @@ var _ = Describe("SNR Config Test", func() { Expect(container.Image).To(Equal(shared.DsDummyImageName)) envVars := getEnvVarMap(container.Env) Expect(envVars["WATCHDOG_PATH"].Value).To(Equal(config.Spec.WatchdogFilePath)) + Expect(envVars["PREFERRED_ADDRESS_TYPES"].Value).To(Equal("")) Expect(len(ds.OwnerReferences)).To(Equal(1)) Expect(ds.OwnerReferences[0].Name).To(Equal(config.Name)) @@ -123,6 +124,23 @@ var _ = Describe("SNR Config Test", func() { Expect(container.SecurityContext.Privileged).To(Equal(pointer.Bool(true))) Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(Equal(pointer.Bool(true))) }) + When("Configuration has customized address types", func() { + BeforeEach(func() { + config.Spec.PreferredAddressTypes = []string{"InternalDNS", "InternalIP"} + }) + It("Daemonset should have comma-separated address types in env var", func() { + Eventually(func(g Gomega) { + ds = &appsv1.DaemonSet{} + g.Expect(k8sClient.Get(context.Background(), dsKey, ds)).Should(BeNil()) + + dsContainers := ds.Spec.Template.Spec.Containers + g.Expect(len(dsContainers)).To(BeNumerically("==", 1)) + container := dsContainers[0] + envVars := getEnvVarMap(container.Env) + g.Expect(envVars["PREFERRED_ADDRESS_TYPES"].Value).To(Equal("InternalDNS,InternalIP")) + }, 10*time.Second, 250*time.Millisecond).Should(Succeed()) + }) + }) When("Configuration has customized tolerations", func() { var expectedToleration corev1.Toleration BeforeEach(func() { diff --git a/internal/controlplane/manager.go b/internal/controlplane/manager.go index 9603ec5e1..3a2efecfa 100644 --- a/internal/controlplane/manager.go +++ b/internal/controlplane/manager.go @@ -5,8 +5,10 @@ import ( "crypto/tls" "errors" "fmt" + "net" "net/http" "os" + "strings" "time" "github.com/go-logr/logr" @@ -22,13 +24,16 @@ import ( ) const ( - kubeletPort = "10250" + defaultKubeletPort = "10250" ) // Manager contains logic and info needed to fence and remediate controlplane nodes type Manager struct { nodeName string nodeRole peers.Role + preferredAddressTypes []string + nodeAddresses []corev1.NodeAddress + kubeletPort string endpointHealthCheckUrl string wasEndpointAccessibleAtStart bool client client.Client @@ -37,9 +42,22 @@ type Manager struct { // NewManager inits a new Manager return nil if init fails func NewManager(nodeName string, myClient client.Client) *Manager { + var preferredAddressTypes []string + rawPreferredAddressTypes := os.Getenv("PREFERRED_ADDRESS_TYPES") + if rawPreferredAddressTypes != "" { + preferredAddressTypes = strings.Split(rawPreferredAddressTypes, ",") + } else { + preferredAddressTypes = []string{"NodeName"} + } + for i, addressType := range preferredAddressTypes { + preferredAddressTypes[i] = strings.TrimSpace(addressType) + } + return &Manager{ nodeName: nodeName, endpointHealthCheckUrl: os.Getenv("END_POINT_HEALTH_CHECK_URL"), + preferredAddressTypes: preferredAddressTypes, + kubeletPort: defaultKubeletPort, client: myClient, wasEndpointAccessibleAtStart: false, log: ctrl.Log.WithName("controlPlane").WithName("Manager"), @@ -125,6 +143,7 @@ func (manager *Manager) initializeManager() error { return wrapWithInitError(err) } manager.setNodeRole(node) + manager.nodeAddresses = node.Status.Addresses manager.wasEndpointAccessibleAtStart = manager.isEndpointAccessible() return nil @@ -166,24 +185,45 @@ func (manager *Manager) isEndpointAccessible() bool { } func (manager *Manager) isKubeletServiceRunning() bool { - url := fmt.Sprintf("https://%s:%s/pods", manager.nodeName, kubeletPort) + for _, addressType := range manager.preferredAddressTypes { + if addressType == "NodeName" { + if manager.isKubeletServiceRunningOnAddress(manager.nodeName) { + return true + } + } else { + nodeAddressType := corev1.NodeAddressType(addressType) + for _, address := range manager.nodeAddresses { + if address.Type == nodeAddressType { + if manager.isKubeletServiceRunningOnAddress(address.Address) { + return true + } + } + } + } + } + + return false +} + +func (manager *Manager) isKubeletServiceRunningOnAddress(address string) bool { + url := fmt.Sprintf("https://%s/pods", net.JoinHostPort(address, manager.kubeletPort)) tr := &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, MinVersion: certificates.TLSMinVersion, }, } - httpClient := &http.Client{Transport: tr} + httpClient := &http.Client{Transport: tr, Timeout: 10 * time.Second} req, err := http.NewRequest("GET", url, nil) if err != nil { - manager.log.Error(err, "failed to create a kubelet service request", "node name", manager.nodeName) + manager.log.Error(err, "failed to create a kubelet service request", "address", address) return false } resp, err := httpClient.Do(req) if err != nil { - manager.log.Error(err, "kubelet service is down", "node name", manager.nodeName) + manager.log.Error(err, "kubelet service is down", "address", address) return false } defer resp.Body.Close() diff --git a/internal/controlplane/manager_test.go b/internal/controlplane/manager_test.go new file mode 100644 index 000000000..8b8ede4be --- /dev/null +++ b/internal/controlplane/manager_test.go @@ -0,0 +1,124 @@ +package controlplane + +import ( + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" +) + +var _ = Describe("Control-plane Manager", func() { + Describe("preferred address types configuration", func() { + AfterEach(func() { + Expect(os.Unsetenv("PREFERRED_ADDRESS_TYPES")).To(Succeed()) + }) + + It("should default to NodeName when the environment variable is not set", func() { + manager := NewManager("node-1", nil) + Expect(manager.preferredAddressTypes).To(Equal([]string{"NodeName"})) + }) + + It("should parse address types from the environment", func() { + Expect(os.Setenv("PREFERRED_ADDRESS_TYPES", "InternalIP,NodeName")).To(Succeed()) + manager := NewManager("node-1", nil) + Expect(manager.preferredAddressTypes).To(Equal([]string{"InternalIP", "NodeName"})) + }) + }) + + Describe("kubelet service check", func() { + // RFC 6761 reserves .invalid, so resolution fails fast without network access + const unreachableHost = "unreachable.invalid" + + var kubeletHost, kubeletPort string + + BeforeEach(func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + DeferCleanup(server.Close) + + serverUrl, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + kubeletHost = serverUrl.Hostname() + kubeletPort = serverUrl.Port() + }) + + newTestManager := func(nodeName string, preferredAddressTypes []string, nodeAddresses []corev1.NodeAddress) *Manager { + return &Manager{ + nodeName: nodeName, + preferredAddressTypes: preferredAddressTypes, + nodeAddresses: nodeAddresses, + kubeletPort: kubeletPort, + log: ctrl.Log.WithName("controlPlane").WithName("Manager"), + } + } + + It("should contact the kubelet via the node name", func() { + manager := newTestManager(kubeletHost, []string{"NodeName"}, nil) + Expect(manager.isKubeletServiceRunning()).To(BeTrue()) + }) + + It("should fail when the node name is not resolvable", func() { + manager := newTestManager(unreachableHost, []string{"NodeName"}, nil) + Expect(manager.isKubeletServiceRunning()).To(BeFalse()) + }) + + It("should contact the kubelet via a node address", func() { + addresses := []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: kubeletHost}} + manager := newTestManager(unreachableHost, []string{"InternalIP"}, addresses) + Expect(manager.isKubeletServiceRunning()).To(BeTrue()) + }) + + It("should ignore addresses of other types", func() { + addresses := []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: kubeletHost}} + manager := newTestManager(unreachableHost, []string{"ExternalIP"}, addresses) + Expect(manager.isKubeletServiceRunning()).To(BeFalse()) + }) + + It("should fall back to later address types", func() { + addresses := []corev1.NodeAddress{{Type: corev1.NodeInternalDNS, Address: unreachableHost}} + manager := newTestManager(kubeletHost, []string{"InternalDNS", "NodeName"}, addresses) + Expect(manager.isKubeletServiceRunning()).To(BeTrue()) + }) + + It("should contact the kubelet via an IPv6 address", func() { + listener, err := net.Listen("tcp", "[::1]:0") + if err != nil { + Skip("IPv6 loopback not available: " + err.Error()) + } + Expect(err).NotTo(HaveOccurred()) + + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + Expect(server.Listener.Close()).To(Succeed()) + server.Listener = listener + server.StartTLS() + DeferCleanup(server.Close) + + serverUrl, err := url.Parse(server.URL) + Expect(err).NotTo(HaveOccurred()) + + addresses := []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: serverUrl.Hostname()}} + manager := newTestManager(unreachableHost, []string{"InternalIP"}, addresses) + manager.kubeletPort = serverUrl.Port() + Expect(manager.isKubeletServiceRunning()).To(BeTrue()) + }) + + It("should try all addresses of a type", func() { + addresses := []corev1.NodeAddress{ + {Type: corev1.NodeInternalIP, Address: unreachableHost}, + {Type: corev1.NodeInternalIP, Address: kubeletHost}, + } + manager := newTestManager(unreachableHost, []string{"InternalIP"}, addresses) + Expect(manager.isKubeletServiceRunning()).To(BeTrue()) + }) + }) +}) diff --git a/internal/controlplane/suite_test.go b/internal/controlplane/suite_test.go new file mode 100644 index 000000000..1316e5150 --- /dev/null +++ b/internal/controlplane/suite_test.go @@ -0,0 +1,20 @@ +package controlplane + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestControlPlane(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Control Plane Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) +})