Skip to content

Commit

Permalink
add feature-gate to allow cri metrics instead of cAdvisor
Browse files Browse the repository at this point in the history
  • Loading branch information
@bravebeaver
bravebeaver committed Feb 17, 2025
1 parent c44de42 commit 27ca7dc
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 0 deletions.
6 changes: 6 additions & 0 deletions pkg/agent/baker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,12 @@ func ValidateAndSetLinuxNodeBootstrappingConfiguration(config *datamodel.NodeBoo
!IsKubernetesVersionGe(config.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, "1.25.0") {
kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "DisableAcceleratorUsageMetrics", false)
}

// https://kubernetes.io/docs/reference/instrumentation/cri-pod-container-metrics/
// use cri metrics instead of cAdvisor
if IsKubeletCriMetricsEnabled(config) {
kubeletFlags["--feature-gates"] = addFeatureGateString(kubeletFlags["--feature-gates"], "PodAndContainerStatsFromCRI", true)
}
}

func validateAndSetWindowsNodeBootstrappingConfiguration(config *datamodel.NodeBootstrappingConfiguration) {
Expand Down
7 changes: 7 additions & 0 deletions pkg/agent/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,13 @@ func IsKubeletServingCertificateRotationEnabled(config *datamodel.NodeBootstrapp
return config.KubeletConfig["--rotate-server-certificates"] == "true"
}

func IsKubeletCriMetricsEnabled(config *datamodel.NodeBootstrappingConfiguration) bool {
if config == nil || config.KubeletConfig == nil {
return false
}
return config.KubeletConfig["--pod-container-stats-cri"] == "true"
}

func getAKSKubeletConfiguration(kc map[string]string) *datamodel.AKSKubeletConfiguration {
kubeletConfig := &datamodel.AKSKubeletConfiguration{
APIVersion: "kubelet.config.k8s.io/v1beta1",
Expand Down
113 changes: 113 additions & 0 deletions pkg/agent/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,41 @@ func TestGetKubeletConfigFileFromFlags(t *testing.T) {
}
}

func getTelescopeKubeletConfig() map[string]string {
kc := map[string]string{
"--address": "0.0.0.0",
"--pod-manifest-path": "/etc/kubernetes/manifests",
"--cluster-domain": "cluster.local",
"--cluster-dns": "10.0.0.10",
"--cgroups-per-qos": "true",
"--tls-cert-file": "/etc/kubernetes/certs/kubeletserver.crt",
"--tls-private-key-file": "/etc/kubernetes/certs/kubeletserver.key",
"--tls-cipher-suites": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256", //nolint:lll
"--max-pods": "110",
"--node-status-update-frequency": "10s",
"--node-status-report-frequency": "5m0s",
"--image-gc-high-threshold": "85",
"--image-gc-low-threshold": "80",
"--event-qps": "0",
"--pod-max-pids": "-1",
"--enforce-node-allocatable": "pods",
"--streaming-connection-idle-timeout": "4h0m0s",
"--rotate-certificates": "true",
"--read-only-port": "10255",
"--protect-kernel-defaults": "true",
"--resolv-conf": "/etc/resolv.conf",
"--anonymous-auth": "false",
"--client-ca-file": "/etc/kubernetes/certs/ca.crt",
"--authentication-token-webhook": "true",
"--authorization-mode": "Webhook",
"--eviction-hard": "memory.available<750Mi,nodefs.available<10%,nodefs.inodesFree<5%",
"--feature-gates": "PodAndContainerStatsFromCRI=true",
"--system-reserved": "cpu=2,memory=1Gi",
"--kube-reserved": "cpu=100m,memory=1638Mi",
}
return kc
}

func getExampleKcWithNodeStatusReportFrequency() map[string]string {
kc := map[string]string{
"--address": "0.0.0.0",
Expand Down Expand Up @@ -222,6 +257,74 @@ var expectedKubeletJSON = `{
"seccompDefault": true
}`

var expectedKubeletTelescopeJSON = `{
"kind": "KubeletConfiguration",
"apiVersion": "kubelet.config.k8s.io/v1beta1",
"staticPodPath": "/etc/kubernetes/manifests",
"address": "0.0.0.0",
"readOnlyPort": 10255,
"tlsCertFile": "/etc/kubernetes/certs/kubeletserver.crt",
"tlsPrivateKeyFile": "/etc/kubernetes/certs/kubeletserver.key",
"tlsCipherSuites": [
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
"TLS_RSA_WITH_AES_256_GCM_SHA384",
"TLS_RSA_WITH_AES_128_GCM_SHA256"
],
"rotateCertificates": true,
"authentication": {
"x509": {
"clientCAFile": "/etc/kubernetes/certs/ca.crt"
},
"webhook": {
"enabled": true
},
"anonymous": {}
},
"authorization": {
"mode": "Webhook",
"webhook": {}
},
"eventRecordQPS": 0,
"clusterDomain": "cluster.local",
"clusterDNS": [
"10.0.0.10"
],
"streamingConnectionIdleTimeout": "4h0m0s",
"nodeStatusUpdateFrequency": "10s",
"nodeStatusReportFrequency": "5m0s",
"imageGCHighThresholdPercent": 85,
"imageGCLowThresholdPercent": 80,
"cgroupsPerQOS": true,
"maxPods": 110,
"podPidsLimit": -1,
"resolvConf": "/etc/resolv.conf",
"evictionHard": {
"memory.available": "750Mi",
"nodefs.available": "10%",
"nodefs.inodesFree": "5%"
},
"protectKernelDefaults": true,
"featureGates": {
"PodAndContainerStatsFromCRI": true
},
"systemReserved": {
"cpu": "2",
"memory": "1Gi"
},
"kubeReserved": {
"cpu": "100m",
"memory": "1638Mi"
},
"enforceNodeAllocatable": [
"pods"
]
}`

var expectedKubeletJSONWithNodeStatusReportFrequency = `{
"kind": "KubeletConfiguration",
"apiVersion": "kubelet.config.k8s.io/v1beta1",
Expand Down Expand Up @@ -462,6 +565,16 @@ func TestGetKubeletConfigFileFlagsWithNodeStatusReportFrequency(t *testing.T) {
}
}

func TestGetKubeletConfigFileFlagsUsingCriMetrics(t *testing.T) {
kc := getTelescopeKubeletConfig()
customKc := &datamodel.CustomKubeletConfig{}
configFileStr := GetKubeletConfigFileContent(kc, customKc)
diff := cmp.Diff(expectedKubeletTelescopeJSON, configFileStr)
if diff != "" {
t.Errorf("Generated config file is different than expected: %s", diff)
}
}

func TestGetKubeletConfigFileFromFlagsWithContainerLogMaxSize(t *testing.T) {
kc := getExampleKcWithContainerLogMaxSize()
customKc := &datamodel.CustomKubeletConfig{
Expand Down

0 comments on commit 27ca7dc

Please sign in to comment.