diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 87ee868e2de..9ca672297ec 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -327,6 +327,12 @@ func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBoo !IsKubernetesVersionGe(config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, "1.25.0") { kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "DisableAcceleratorUsageMetrics", false) } + + // https://kubernetes.io/docs/reference/instrumentation/cri-pod-container-metrics/ + // use cri metrics instead of cAdvisor + if IsKubeletCriMetricsEnabled(config) { + kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "PodAndContainerStatsFromCRI", true) + } } func validateAndSetWindowsNodeBootstrappingConfiguration(config *datamodel.NodeBootstrappingConfiguration) { diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index a063f2a152c..c0e2bec58c2 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -444,6 +444,13 @@ func IsKubeletServingCertificateRotationEnabled(config *datamodel.NodeBootstrapp return config.KubeletConfig["--rotate-server-certificates"] == "true" } +func IsKubeletCriMetricsEnabled(config *datamodel.NodeBootstrappingConfiguration) bool { + if config == nil || config.KubeletConfig == nil { + return false + } + return config.KubeletConfig["--pod-container-stats-cri"] == "true" +} + func getAKSKubeletConfiguration(kc map[string]string) *datamodel.AKSKubeletConfiguration { kubeletConfig := &datamodel.AKSKubeletConfiguration{ APIVersion: "kubelet.config.k8s.io/v1beta1", diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index d55123aca93..371b19196cd 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -69,6 +69,41 @@ func TestGetKubeletConfigFileFromFlags(t *testing.T) { } } +func getTelescopeKubeletConfig() map[string]string { + kc := map[string]string{ + "--address": "0.0.0.0", + "--pod-manifest-path": "/etc/kubernetes/manifests", + "--cluster-domain": "cluster.local", + "--cluster-dns": "10.0.0.10", + "--cgroups-per-qos": "true", + "--tls-cert-file": "/etc/kubernetes/certs/kubeletserver.crt", + "--tls-private-key-file": "/etc/kubernetes/certs/kubeletserver.key", + "--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", //nolint:lll + "--max-pods": "110", + "--node-status-update-frequency": "10s", + "--node-status-report-frequency": "5m0s", + "--image-gc-high-threshold": "85", + "--image-gc-low-threshold": "80", + "--event-qps": "0", + "--pod-max-pids": "-1", + "--enforce-node-allocatable": "pods", + "--streaming-connection-idle-timeout": "4h0m0s", + "--rotate-certificates": "true", + "--read-only-port": "10255", + "--protect-kernel-defaults": "true", + "--resolv-conf": "/etc/resolv.conf", + "--anonymous-auth": "false", + "--client-ca-file": "/etc/kubernetes/certs/ca.crt", + "--authentication-token-webhook": "true", + "--authorization-mode": "Webhook", + "--eviction-hard": "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%", + "--feature-gates": "PodAndContainerStatsFromCRI=true", + "--system-reserved": "cpu=2,memory=1Gi", + "--kube-reserved": "cpu=100m,memory=1638Mi", + } + return kc +} + func getExampleKcWithNodeStatusReportFrequency() map[string]string { kc := map[string]string{ "--address": "0.0.0.0", @@ -222,6 +257,74 @@ var expectedKubeletJSON = `{ "seccompDefault": true }` +var expectedKubeletTelescopeJSON = `{ + "kind": "KubeletConfiguration", + "apiVersion": "kubelet.config.k8s.io/v1beta1", + "staticPodPath": "/etc/kubernetes/manifests", + "address": "0.0.0.0", + "readOnlyPort": 10255, + "tlsCertFile": "/etc/kubernetes/certs/kubeletserver.crt", + "tlsPrivateKeyFile": "/etc/kubernetes/certs/kubeletserver.key", + "tlsCipherSuites": [ + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_256_GCM_SHA384", + "TLS_RSA_WITH_AES_128_GCM_SHA256" + ], + "rotateCertificates": true, + "authentication": { + "x509": { + "clientCAFile": "/etc/kubernetes/certs/ca.crt" + }, + "webhook": { + "enabled": true + }, + "anonymous": {} + }, + "authorization": { + "mode": "Webhook", + "webhook": {} + }, + "eventRecordQPS": 0, + "clusterDomain": "cluster.local", + "clusterDNS": [ + "10.0.0.10" + ], + "streamingConnectionIdleTimeout": "4h0m0s", + "nodeStatusUpdateFrequency": "10s", + "nodeStatusReportFrequency": "5m0s", + "imageGCHighThresholdPercent": 85, + "imageGCLowThresholdPercent": 80, + "cgroupsPerQOS": true, + "maxPods": 110, + "podPidsLimit": -1, + "resolvConf": "/etc/resolv.conf", + "evictionHard": { + "memory.available": "750Mi", + "nodefs.available": "10%", + "nodefs.inodesFree": "5%" + }, + "protectKernelDefaults": true, + "featureGates": { + "PodAndContainerStatsFromCRI": true + }, + "systemReserved": { + "cpu": "2", + "memory": "1Gi" + }, + "kubeReserved": { + "cpu": "100m", + "memory": "1638Mi" + }, + "enforceNodeAllocatable": [ + "pods" + ] +}` + var expectedKubeletJSONWithNodeStatusReportFrequency = `{ "kind": "KubeletConfiguration", "apiVersion": "kubelet.config.k8s.io/v1beta1", @@ -462,6 +565,16 @@ func TestGetKubeletConfigFileFlagsWithNodeStatusReportFrequency(t *testing.T) { } } +func TestGetKubeletConfigFileFlagsUsingCriMetrics(t *testing.T) { + kc := getTelescopeKubeletConfig() + customKc := &datamodel.CustomKubeletConfig{} + configFileStr := GetKubeletConfigFileContent(kc, customKc) + diff := cmp.Diff(expectedKubeletTelescopeJSON, configFileStr) + if diff != "" { + t.Errorf("Generated config file is different than expected: %s", diff) + } +} + func TestGetKubeletConfigFileFromFlagsWithContainerLogMaxSize(t *testing.T) { kc := getExampleKcWithContainerLogMaxSize() customKc := &datamodel.CustomKubeletConfig{