perf: validate credentials and check apiserver connectivity before starting kubelet

e2e/validation.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"strings"
 
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/stretchr/testify/require"
@@ -38,6 +39,10 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 	stdout := execResult.stdout.String()
 	require.NotContains(s.T, stdout, "--dynamic-config-dir", "kubelet flag '--dynamic-config-dir' should not be present in /etc/default/kubelet\nContents:\n%s")
 
+	kubeletLogs := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl").stdout.String()
+	validatedKubeletCredentials := strings.Contains(kubeletLogs, "kubelet client credential is valid") || strings.Contains(kubeletLogs, "kubelet bootstrap token credential is valid")
+	require.True(s.T, validatedKubeletCredentials, "expected kubelet to have validated its credential or bootstrap token before startup, but seemingly did not")
+
 	// the instructions belows expects the SSH key to be uploaded to the user pool VM.
 	// which happens as a side-effect of execCommandOnVMForScenario, it's ugly but works.
 	// maybe we should use a single ssh key per cluster, but need to be careful with parallel test runs.
@@ -62,7 +67,7 @@ func ValidateCommonLinux(ctx context.Context, s *Scenario) {
 		//"cloud-config.txt", // file with UserData
 	})
 
-	execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed")
+	_ = execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo curl http://168.63.129.16:32526/vmSettings", 0, "curl to wireserver failed")
 
 	execResult = execOnVMForScenarioOnUnprivilegedPod(ctx, s, "curl https://168.63.129.16/machine/?comp=goalstate -H 'x-ms-version: 2015-04-05' -s --connect-timeout 4")
 	require.Equal(s.T, "28", execResult.exitCode, "curl to wireserver should fail")

e2e/validators.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/tidwall/gjson"
 	"net"
 	"os"
 	"regexp"
 	"strings"
 	"time"
 
+	"github.com/tidwall/gjson"
+
 	"github.com/Azure/agentbaker/e2e/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -301,7 +302,7 @@ func ValidateContainerdWASMShims(ctx context.Context, s *Scenario) {
 
 func ValidateKubeletHasNotStopped(ctx context.Context, s *Scenario) {
 	command := "sudo journalctl -u kubelet"
-	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs")
+	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs with journalctl")
 	assert.NotContains(s.T, execResult.stdout.String(), "Stopped Kubelet")
 	assert.Contains(s.T, execResult.stdout.String(), "Started Kubelet")
 }
@@ -314,7 +315,7 @@ func ValidateServicesDoNotRestartKubelet(ctx context.Context, s *Scenario) {
 
 // ValidateKubeletHasFlags checks kubelet is started with the right flags and configs.
 func ValidateKubeletHasFlags(ctx context.Context, s *Scenario, filePath string) {
-	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not get kubelet logs")
+	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl")
 	configFileFlags := fmt.Sprintf("FLAG: --config=\"%s\"", filePath)
 	require.Containsf(s.T, execResult.stdout.String(), configFileFlags, "expected to find flag %s, but not found", "config")
 }

parts/linux/cloud-init/artifacts/kubelet.service

@@ -20,6 +20,8 @@ ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
 ExecStartPre=-/sbin/ebtables -t nat --list
 ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
+ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
+
 ExecStart=/usr/local/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \

parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -euo pipefail
+
+# this gives us logs_to_events and retry wrappers
+source /opt/azure/containers/provision_source.sh 
+
+KUBECONFIG_PATH="${KUBECONFIG_PATH:-/var/lib/kubelet/kubeconfig}"
+BOOTSTRAP_KUBECONFIG_PATH="${BOOTSTRAP_KUBECONFIG_PATH:-/var/lib/kubelet/bootstrap-kubeconfig}"
+
+VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES=${VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES:-10}
+VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS=${VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS:-3}
+VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS=${VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS:-5}
+
+function validateKubeconfig {
+    local kubeconfig_path=$1
+
+    if ! retrycmd_if_failure $VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES \
+        $VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS \
+        $VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS \
+        kubectl auth whoami --kubeconfig "$kubeconfig_path"; then
+
+        echo "kubelet credential validation failed"
+        exit 1
+    fi
+}
+
+function validateKubeletCredentials {
+    if [ ! -f "$KUBECONFIG_PATH" ] && [ ! -f "$BOOTSTRAP_KUBECONFIG_PATH" ]; then
+        echo "both kubeconfig: $KUBECONFIG_PATH and bootstrap-kubeconfig: $BOOTSTRAP_KUBECONFIG_PATH do not exist, unable to start kubelet"
+        exit 1
+    fi
+
+    if ! which kubectl; then
+        echo "kubectl not found, will skip kubelet credential validation"
+        exit 0
+    fi
+
+    if [ -f "$KUBECONFIG_PATH" ]; then
+        echo "will validate kubeconfig: $KUBECONFIG_PATH"
+        validateKubeconfig "$KUBECONFIG_PATH"
+        echo "kubelet client credential is valid"
+        exit 0
+    fi
+
+    echo "will validate bootstrap-kubeconfig: $BOOTSTRAP_KUBECONFIG_PATH"
+    validateKubeconfig "$BOOTSTRAP_KUBECONFIG_PATH"
+    echo "kubelet bootstrap token credential is valid"
+}
+
+logs_to_events "AKS.validateKubeletCredentials" validateKubeletCredentials

parts/linux/cloud-init/nodecustomdata.yml

@@ -409,6 +409,13 @@ write_files:
   content: !!binary |
     {{GetVariableProperty "cloudInitData" "ensureIMDSRestrictionScript"}}
 
+- path: /opt/azure/containers/validate-kubelet-credentials.sh
+  permissions: "0755"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    {{GetVariableProperty "cloudInitData" "validateKubeletCredentialsScript"}}
+
 - path: /etc/kubernetes/certs/ca.crt
   permissions: "0600"
   encoding: base64

pkg/agent/baker_test.go

@@ -787,7 +787,7 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 				config.KubeletConfig = map[string]string{}
 			}, nil),
 
-		Entry("AKSUbuntu1804 with kubelet client certificatet", "AKSUbuntu1804+WithKubeletClientCert", "1.18.3",
+		Entry("AKSUbuntu1804 with kubelet client certificate", "AKSUbuntu1804+WithKubeletClientCert", "1.18.3",
 			func(config *datamodel.NodeBootstrappingConfiguration) {
 				config.ContainerService.Properties.CertificateProfile = &datamodel.CertificateProfile{
 					ClientCertificate: "fooBarBaz",
@@ -798,12 +798,14 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 				etcDefaultKubelet := o.files["/etc/default/kubelet"].value
 				etcDefaultKubeletService := o.files["/etc/systemd/system/kubelet.service"].value
 				kubeletSh := o.files["/opt/azure/containers/kubelet.sh"].value
+				validateCredentials := o.files["/opt/azure/containers/validate-kubelet-credentials.sh"].value
 				caCRT := o.files["/etc/kubernetes/certs/ca.crt"].value
 				kubeconfig := o.files["/var/lib/kubelet/kubeconfig"].value
 
 				Expect(etcDefaultKubelet).NotTo(BeEmpty())
 				Expect(etcDefaultKubeletService).NotTo(BeEmpty())
 				Expect(kubeletSh).NotTo(BeEmpty())
+				Expect(validateCredentials).ToNot(BeEmpty())
 				Expect(caCRT).NotTo(BeEmpty())
 				Expect(kubeconfig).ToNot(BeEmpty())
 
@@ -822,13 +824,15 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 				etcDefaultKubelet := o.files["/etc/default/kubelet"].value
 				etcDefaultKubeletService := o.files["/etc/systemd/system/kubelet.service"].value
 				kubeletSh := o.files["/opt/azure/containers/kubelet.sh"].value
+				validateCredentials := o.files["/opt/azure/containers/validate-kubelet-credentials.sh"].value
 				bootstrapKubeconfig := o.files["/var/lib/kubelet/bootstrap-kubeconfig"].value
 				caCRT := o.files["/etc/kubernetes/certs/ca.crt"].value
 
 				Expect(etcDefaultKubelet).NotTo(BeEmpty())
 				Expect(bootstrapKubeconfig).NotTo(BeEmpty())
 				Expect(kubeletSh).NotTo(BeEmpty())
 				Expect(etcDefaultKubeletService).NotTo(BeEmpty())
+				Expect(validateCredentials).ToNot(BeEmpty())
 				Expect(caCRT).NotTo(BeEmpty())
 
 				Expect(bootstrapKubeconfig).To(ContainSubstring("token"))

pkg/agent/const.go

@@ -73,6 +73,7 @@ const (
 	migPartitionScript                  = "linux/cloud-init/artifacts/mig-partition.sh"
 	migPartitionSystemdService          = "linux/cloud-init/artifacts/mig-partition.service"
 	ensureIMDSRestrictionScript         = "linux/cloud-init/artifacts/ensure_imds_restriction.sh"
+	validateKubeletCredentialsScript    = "linux/cloud-init/artifacts/validate-kubelet-credentials.sh"
 
 	// scripts and service for enabling ipv6 dual stack.
 	dhcpv6SystemdService            = "linux/cloud-init/artifacts/dhcpv6.service"

pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData

@@ -102,7 +102,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAC/5SUT0/bTBDG7/4Uq4jD+x4Wt7Q35EMCDopIExQn6iFE1no9IaOsx9HObICWfvfK+UMbCIj65N39Pc88uxrNdEIos+gS2HpcCdaUXIcCHEh0UVOJzc6NkUX6gCycxIF97GprXFwgxcsd+t2QcEIg97Vf6pocEpyK8XcgytYkBgl8ecrg12ghas8F/MfxaJpt/2bRCFiMl8S4e/PI+2UGNjmLUlqjr6kCki46SGIQG5cwN8HJc9IsWAvM6QNKJkYCJ5+/fonSB7BZ43TjIdncrDC8UHG9ktj8CB7i51i8tzrlxb/pgDh4yLEqOffA4tE2r3vcp1qW6JVeqXhtfOyweL7BR1hL+FY2bVUL52qqTv6r6kCintSdh5W6bb2sdNtST+reKu3+V9qB+qRm6lzJAkhty27kWhdI5auYrzfO1Rxbx9LvbCqzBM0L4+G1W3So0zE3SijEFA5YaVFkGg+HLEdRXL1EKVTg0e5Ff1Rv9Li6jdTu0xqocdNNg4I/OKG6BO1MAY6T1snP60kn7afjfDC8TPN+u5P2s1+tA8E6OTtc1y5UoFcu3CHpEv22j5sUnkCA4y2xBfgv7cm+2Lif5Z3hcJyNR+2bvNtvX2XHsIvhoNu7yru9fvouNG73Buno8gNMPpoMxr1vW7uj5NVoOHkn0uYgiqY9YjHOzTaDBcrOY1IFJ6gDg98Niuh3AAAA///KhjADugQAAA==
+    H4sIAAAAAAAC/5SUT0/jPBDG7/kUVsXhfQ8m78vuDeXQQooqui1qWu2hVJHjTOmozqTyjAvsst99lf5ht1AQ5BTbv+eZx3E80wmhzKJLYOtxJVhTch0KcCDRRU0lNjM3RhbpA7JwEgf2sautcXGBFC936HdDwgmB3Nd+qWtySHAqxt+BKFuTGCTw5SmDX6OFqD0X8B/Ho2m2fZtFI2AxXhLj7s0j74cZ2OQsSmmNvqYKSLroIIlBbFzC3AQnz0mzYC0wpw8omRgJnPz/9UuUPoDNGqcbD8lmZ4XhhYrrlcTmR/AQP8fivdUpLz6nA+LgIceq5NwDi0fbfN3jPtWyRK/0SsVr42OHxfMOPsJawreyaataOFdTdfJPVQcS9aTuPKzUbetlpduWelL3Vmn3r9IO1H9qps6VLIDUtuxGrnWBVL6K+XriXM2xdSz9zqYyS9C8MB5eu0WHOh1zo4RCTOGAlRZFpvFwyHIUxdVLlEIFHu1e9LmjXBuHpRHQu3zaeiiBBI3j5jz/uL1xY9RtpHaP1kBNNt387uAPVqguQTtTgOOkdfLzetJJ++k4Hwwv07zf7qT97FfrQLBOzg7HtQsV6JULd0i6RL+9FU0KTyDA8ZbYAvyX9mRfbNzP8s5wOM7Go/ZN3u23r7Jj2MVw0O1d5d1eP30XGrd7g3R0+QEmH00G4963rd1R8mo0nLwTabMQRdMesRjnZps2BWXnMamCE9SBwe/aTvQ7AAD//6+ZIOoIBQAA
 
 - path: /etc/systemd/system/mig-partition.service
   permissions: "0644"
@@ -307,6 +307,13 @@ write_files:
   content: !!binary |
     H4sIAAAAAAAC/+xXe2/TPhT935/izL9K+yHV3YOHoGhCXVtQxPpQWoTQNFVpc9OapU5xXLGx9bsjJ+s73SgMCSryT6L4+vjec8+9tv/bO+hKddD14gEbaTn09HVd9pzmCc/dNF2nVnI/depOueM0i4LzCWdSxaSNOw6pHdU81Q+p7XVDsvZOvVV12x2nVml13Gqr7TrlttOod9wPZ9VOu9Gplerv7Ffp9KxaFIEXxjThjJSdbye5FBste0ZGysJV69ZwDW4+k1FvEIE3U7dRd8pwmkXkFuPgd0ZO4jYsGPR8HehxSDARhkkoMNaXInIbopyiVROf19CKyGVGwxmTAc4hvoEve4eL1zADUgwAUuzFccgYYxWTyYOupJGqXygUeGp8JQ2OWCAZIxWP9eqa1vf/n+AmMbarn4PnbjbElc9POE5OwI0eE8fFoldzz04piDQhxZCq/2Ns5tEbUO/S2n8dkBmQttipqRdq8vxrG0xs4llsicujZHYMYaZwooVb9DWNIAT2hY+jF68Kx8+fFe7eB0+PIUYwvRHEMH0JfxRpg5eH9k8vGg5JGQgx/eKl9y2L7vXJL8LzffLRvUapT8qcepekkcktgkivBx+QZ8aaOMRnVNxGc38eS0J/7g0EfcHhOr1zirMZXaYJUq0QrMmMtVpUx/RJR3A4+xnIlaQ622XzwRw5eNtwP5bcCo6wBxGvKf6Pzxtub6fVldAUxvTThRDI0FjBP1Ih3MH9K4RZISwT/JsLYXGxB3O0c4UQSDZhzJdxxg4322lSLsurQs+mNLufzZjdlS1grnpFWaq/T/F+RDFUZFKu1lu/iqCIfKtOn0IyxDc1rYod3izvQEfD7Tp9ZSbwRN5L6v4bxf0r+s3sDLvSuR/S75YCXm7ZjyvgrTr0rgl4kl4zkoN+5j3kvmP+xksEm6Uiu/WzQH4PAAD//7E5jbFFDgAA
 
+- path: /opt/azure/containers/validate-kubelet-credentials.sh
+  permissions: "0755"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    H4sIAAAAAAAC/5RTTU/bQBC9768YthYnjEFVL644GOK2EYGgxFRFFK3W9hqvWHYt7zi0RfnvlU0+iGWHJJco2Zn35r038+nAi6X2Ym5zYgWCKyoDhSxExqUixJqqTAR4pkCP/6tK4SVGI5dalNYrSjOTVhrN3sqObQ6EXN6ehxfj62/D7+wmiH6cUee19ZfvejNeekrG3lMVCyWw+U6MzuTjnJLz8TiaRpPghnVg9T52oMbGoMWSF+4GPvkZjIaDIAobiFEYsYtJOAivo2EwmrKr4BebhNFkGE7PnNddS3339GS+HbiuvGODcBTcsWk9+eBDgo4W3/28E080vArHt9F+TK0m3/0yJySrdILSaJhxJVOO4nJlJrwSAABlEq5g7TErOOZnzilpXmUGB1AKLP8mzymTGatXqyoFOLuaC78boPrj7OvXnr0tB951N/JQAa8wh5fc8GcJ7rvFAuq0HKBfAXOhyQpCJLkButhOSEqRCo2Sq6Wztcm1OSKl654/EuG0+ZlJ0heHEnixgrOLWGQG93AAbgbUaR0LhQc4PFw/914VhYeFik0RscH8XeI+tBmA6xS6DtCHfjZIDWiDtWiLR1BpHisBaMAiLxEWzvWZs962l1wm+TKxzvmXadZsmal0egQvUimwT7KArQm12E/a7Pd9hnfO0ZAuk9xq6Jq34w5pf3Vr6ZQUekOZtG+AW3V1DbtvuG8EncN/0LMpYcULaJ6E7tYyJ0SZR8vQMDETGi3Q4HJ63H8zdMs9/Q8AAP//dacediUHAAA=
+
 - path: /etc/kubernetes/certs/ca.crt
   permissions: "0600"
   encoding: base64

pkg/agent/testdata/AKSUbuntu1604+Containerd/line105.sh

@@ -19,6 +19,8 @@ ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
 ExecStartPre=-/sbin/ebtables -t nat --list
 ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
+ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
+
 ExecStart=/usr/local/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \

pkg/agent/testdata/AKSUbuntu1604+Containerd/line315.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+set -euo pipefail
+
+source /opt/azure/containers/provision_source.sh 
+
+KUBECONFIG_PATH="${KUBECONFIG_PATH:-/var/lib/kubelet/kubeconfig}"
+BOOTSTRAP_KUBECONFIG_PATH="${BOOTSTRAP_KUBECONFIG_PATH:-/var/lib/kubelet/bootstrap-kubeconfig}"
+
+VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES=${VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES:-10}
+VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS=${VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS:-3}
+VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS=${VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS:-5}
+
+function validateKubeconfig {
+    local kubeconfig_path=$1
+
+    if ! retrycmd_if_failure $VALIDATE_KUBELET_CREDENTIALS_MAX_RETRIES \
+        $VALIDATE_KUBELET_CREDENTIALS_RETRY_DELAY_SECONDS \
+        $VALIDATE_KUBELET_CREDENTIALS_RETRY_TIMEOUT_SECONDS \
+        kubectl auth whoami --kubeconfig "$kubeconfig_path"; then
+
+        echo "kubelet credential validation failed"
+        exit 1
+    fi
+}
+
+function validateKubeletCredentials {
+    if [ ! -f "$KUBECONFIG_PATH" ] && [ ! -f "$BOOTSTRAP_KUBECONFIG_PATH" ]; then
+        echo "both kubeconfig: $KUBECONFIG_PATH and bootstrap-kubeconfig: $BOOTSTRAP_KUBECONFIG_PATH do not exist, unable to start kubelet"
+        exit 1
+    fi
+
+    if ! which kubectl; then
+        echo "kubectl not found, will skip kubelet credential validation"
+        exit 0
+    fi
+
+    if [ -f "$KUBECONFIG_PATH" ]; then
+        echo "will validate kubeconfig: $KUBECONFIG_PATH"
+        validateKubeconfig "$KUBECONFIG_PATH"
+        echo "kubelet client credential is valid"
+        exit 0
+    fi
+
+    echo "will validate bootstrap-kubeconfig: $BOOTSTRAP_KUBECONFIG_PATH"
+    validateKubeconfig "$BOOTSTRAP_KUBECONFIG_PATH"
+    echo "kubelet bootstrap token credential is valid"
+}
+
+logs_to_events "AKS.validateKubeletCredentials" validateKubeletCredentials

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData

@@ -102,7 +102,7 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    H4sIAAAAAAAC/5SUT0/bTBDG7/4Uq4jD+x4Wt7Q35EMCDopIExQn6iFE1no9IaOsx9HObICWfvfK+UMbCIj65N39Pc88uxrNdEIos+gS2HpcCdaUXIcCHEh0UVOJzc6NkUX6gCycxIF97GprXFwgxcsd+t2QcEIg97Vf6pocEpyK8XcgytYkBgl8ecrg12ghas8F/MfxaJpt/2bRCFiMl8S4e/PI+2UGNjmLUlqjr6kCki46SGIQG5cwN8HJc9IsWAvM6QNKJkYCJ5+/fonSB7BZ43TjIdncrDC8UHG9ktj8CB7i51i8tzrlxb/pgDh4yLEqOffA4tE2r3vcp1qW6JVeqXhtfOyweL7BR1hL+FY2bVUL52qqTv6r6kCintSdh5W6bb2sdNtST+reKu3+V9qB+qRm6lzJAkhty27kWhdI5auYrzfO1Rxbx9LvbCqzBM0L4+G1W3So0zE3SijEFA5YaVFkGg+HLEdRXL1EKVTg0e5Ff1Rv9Li6jdTu0xqocdNNg4I/OKG6BO1MAY6T1snP60kn7afjfDC8TPN+u5P2s1+tA8E6OTtc1y5UoFcu3CHpEv22j5sUnkCA4y2xBfgv7cm+2Lif5Z3hcJyNR+2bvNtvX2XHsIvhoNu7yru9fvouNG73Buno8gNMPpoMxr1vW7uj5NVoOHkn0uYgiqY9YjHOzTaDBcrOY1IFJ6gDg98Niuh3AAAA///KhjADugQAAA==
+    H4sIAAAAAAAC/5SUT0/jPBDG7/kUVsXhfQ8m78vuDeXQQooqui1qWu2hVJHjTOmozqTyjAvsst99lf5ht1AQ5BTbv+eZx3E80wmhzKJLYOtxJVhTch0KcCDRRU0lNjM3RhbpA7JwEgf2sautcXGBFC936HdDwgmB3Nd+qWtySHAqxt+BKFuTGCTw5SmDX6OFqD0X8B/Ho2m2fZtFI2AxXhLj7s0j74cZ2OQsSmmNvqYKSLroIIlBbFzC3AQnz0mzYC0wpw8omRgJnPz/9UuUPoDNGqcbD8lmZ4XhhYrrlcTmR/AQP8fivdUpLz6nA+LgIceq5NwDi0fbfN3jPtWyRK/0SsVr42OHxfMOPsJawreyaataOFdTdfJPVQcS9aTuPKzUbetlpduWelL3Vmn3r9IO1H9qps6VLIDUtuxGrnWBVL6K+XriXM2xdSz9zqYyS9C8MB5eu0WHOh1zo4RCTOGAlRZFpvFwyHIUxdVLlEIFHu1e9LmjXBuHpRHQu3zaeiiBBI3j5jz/uL1xY9RtpHaP1kBNNt387uAPVqguQTtTgOOkdfLzetJJ++k4Hwwv07zf7qT97FfrQLBOzg7HtQsV6JULd0i6RL+9FU0KTyDA8ZbYAvyX9mRfbNzP8s5wOM7Go/ZN3u23r7Jj2MVw0O1d5d1eP30XGrd7g3R0+QEmH00G4963rd1R8mo0nLwTabMQRdMesRjnZps2BWXnMamCE9SBwe/aTvQ7AAD//6+ZIOoIBQAA
 
 - path: /etc/systemd/system/mig-partition.service
   permissions: "0644"
@@ -307,6 +307,13 @@ write_files:
   content: !!binary |
     H4sIAAAAAAAC/+xXe2/TPhT935/izL9K+yHV3YOHoGhCXVtQxPpQWoTQNFVpc9OapU5xXLGx9bsjJ+s73SgMCSryT6L4+vjec8+9tv/bO+hKddD14gEbaTn09HVd9pzmCc/dNF2nVnI/depOueM0i4LzCWdSxaSNOw6pHdU81Q+p7XVDsvZOvVV12x2nVml13Gqr7TrlttOod9wPZ9VOu9Gplerv7Ffp9KxaFIEXxjThjJSdbye5FBste0ZGysJV69ZwDW4+k1FvEIE3U7dRd8pwmkXkFuPgd0ZO4jYsGPR8HehxSDARhkkoMNaXInIbopyiVROf19CKyGVGwxmTAc4hvoEve4eL1zADUgwAUuzFccgYYxWTyYOupJGqXygUeGp8JQ2OWCAZIxWP9eqa1vf/n+AmMbarn4PnbjbElc9POE5OwI0eE8fFoldzz04piDQhxZCq/2Ns5tEbUO/S2n8dkBmQttipqRdq8vxrG0xs4llsicujZHYMYaZwooVb9DWNIAT2hY+jF68Kx8+fFe7eB0+PIUYwvRHEMH0JfxRpg5eH9k8vGg5JGQgx/eKl9y2L7vXJL8LzffLRvUapT8qcepekkcktgkivBx+QZ8aaOMRnVNxGc38eS0J/7g0EfcHhOr1zirMZXaYJUq0QrMmMtVpUx/RJR3A4+xnIlaQ622XzwRw5eNtwP5bcCo6wBxGvKf6Pzxtub6fVldAUxvTThRDI0FjBP1Ih3MH9K4RZISwT/JsLYXGxB3O0c4UQSDZhzJdxxg4322lSLsurQs+mNLufzZjdlS1grnpFWaq/T/F+RDFUZFKu1lu/iqCIfKtOn0IyxDc1rYod3izvQEfD7Tp9ZSbwRN5L6v4bxf0r+s3sDLvSuR/S75YCXm7ZjyvgrTr0rgl4kl4zkoN+5j3kvmP+xksEm6Uiu/WzQH4PAAD//7E5jbFFDgAA
 
+- path: /opt/azure/containers/validate-kubelet-credentials.sh
+  permissions: "0755"
+  encoding: gzip
+  owner: root
+  content: !!binary |
+    H4sIAAAAAAAC/5RTTU/bQBC9768YthYnjEFVL644GOK2EYGgxFRFFK3W9hqvWHYt7zi0RfnvlU0+iGWHJJco2Zn35r038+nAi6X2Ym5zYgWCKyoDhSxExqUixJqqTAR4pkCP/6tK4SVGI5dalNYrSjOTVhrN3sqObQ6EXN6ehxfj62/D7+wmiH6cUee19ZfvejNeekrG3lMVCyWw+U6MzuTjnJLz8TiaRpPghnVg9T52oMbGoMWSF+4GPvkZjIaDIAobiFEYsYtJOAivo2EwmrKr4BebhNFkGE7PnNddS3339GS+HbiuvGODcBTcsWk9+eBDgo4W3/28E080vArHt9F+TK0m3/0yJySrdILSaJhxJVOO4nJlJrwSAABlEq5g7TErOOZnzilpXmUGB1AKLP8mzymTGatXqyoFOLuaC78boPrj7OvXnr0tB951N/JQAa8wh5fc8GcJ7rvFAuq0HKBfAXOhyQpCJLkButhOSEqRCo2Sq6Wztcm1OSKl654/EuG0+ZlJ0heHEnixgrOLWGQG93AAbgbUaR0LhQc4PFw/914VhYeFik0RscH8XeI+tBmA6xS6DtCHfjZIDWiDtWiLR1BpHisBaMAiLxEWzvWZs962l1wm+TKxzvmXadZsmal0egQvUimwT7KArQm12E/a7Pd9hnfO0ZAuk9xq6Jq34w5pf3Vr6ZQUekOZtG+AW3V1DbtvuG8EncN/0LMpYcULaJ6E7tYyJ0SZR8vQMDETGi3Q4HJ63H8zdMs9/Q8AAP//dacediUHAAA=
+
 - path: /etc/kubernetes/certs/ca.crt
   permissions: "0600"
   encoding: base64

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line105.sh

@@ -19,6 +19,8 @@ ExecStartPre=/bin/mount --make-shared /var/lib/kubelet
 ExecStartPre=-/sbin/ebtables -t nat --list
 ExecStartPre=-/sbin/iptables -t nat --numeric --list
 
+ExecStartPre=/bin/bash /opt/azure/containers/validate-kubelet-credentials.sh
+
 ExecStart=/usr/local/bin/kubelet \
         --enable-server \
         --node-labels="${KUBELET_NODE_LABELS}" \