refactor: containerd config registry host file to cse

Azure · Jan 21, 2025 · 6a8791f · 6a8791f
1 parent bcdc76b
commit 6a8791f
Show file tree

Hide file tree

Showing 178 changed files with 1,434 additions and 110 deletions.
diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh
@@ -325,6 +325,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -336,6 +340,18 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  # TODO(binxi): need to update for sovereign cloud.
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }

diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml
@@ -421,14 +421,4 @@ write_files:
   encoding: gzip
   owner: root
   content: !!binary |
-    {{GetVariableProperty "cloudInitData" "customSearchDomainsScript"}}
-
-{{- if (ne GetBootstrapProfileContainerRegistryServer "") }}
-# TODO(binxi): need to update for sovereign cloud.
-- path: /etc/containerd/certs.d/mcr.microsoft.com/hosts.toml
-  permissions: "0644"
-  owner: root
-  content: |
-    [host."https://{{GetBootstrapProfileContainerRegistryServer}}"]
-      capabilities = ["pull", "resolve"]
-{{- end}}
+    {{GetVariableProperty "cloudInitData" "customSearchDomainsScript"}}
diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go
@@ -1605,11 +1605,7 @@ oom_score = 0
 						ContainerRegistryServer: "testserver.azurecr.io",
 					},
 				}
-			}, func(o *nodeBootstrappingOutput) {
-				containerdConfigFileContent := o.files["/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"].value
-				Expect(strings.Contains(containerdConfigFileContent, "[host.\"https://testserver.azurecr.io\"]")).To(BeTrue())
-				Expect(strings.Contains(containerdConfigFileContent, "capabilities = [\"pull\", \"resolve\"]")).To(BeTrue())
-			}),
+			}, nil),
 		Entry("AKSUbuntu2204 IMDSRestriction with enable restriction and insert to mangle table", "AKSUbuntu2204+IMDSRestrictionOnWithMangleTable", "1.24.2",
 			func(config *datamodel.NodeBootstrappingConfiguration) {
 				config.EnableIMDSRestriction = true

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh
@@ -315,6 +315,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -326,6 +330,17 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line70.sh
@@ -315,6 +315,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -326,6 +330,17 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }

diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line70.sh
@@ -315,6 +315,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -326,6 +330,17 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=false/line70.sh
@@ -315,6 +315,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -326,6 +330,17 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Disable1804SystemdResolved=true/line70.sh
@@ -315,6 +315,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -326,6 +330,17 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }

diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData b/pkg/agent/testdata/AKSUbuntu1604+Docker/CustomData
diff --git a/pkg/agent/testdata/AKSUbuntu1604+Docker/line70.sh b/pkg/agent/testdata/AKSUbuntu1604+Docker/line70.sh
@@ -315,6 +315,10 @@ EOF
     echo "${CONTAINERD_CONFIG_CONTENT}" | base64 -d > /etc/containerd/config.toml || exit $ERR_FILE_WATCH_TIMEOUT
   fi
 
+  if [[ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]]; then
+    logs_to_events "AKS.CSE.ensureContainerd.configureContainerdRegistryHost" configureContainerdRegistryHost
+  fi
+
   tee "/etc/sysctl.d/99-force-bridge-forward.conf" > /dev/null <<EOF 
 net.ipv4.ip_forward = 1
 net.ipv4.conf.all.forwarding = 1
@@ -326,6 +330,17 @@ EOF
   systemctlEnableAndStart containerd || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
+configureContainerdRegistryHost() {
+  CONTAINERD_CONFIG_REGISTRY_HOST_MCR="/etc/containerd/certs.d/mcr.microsoft.com/hosts.toml"
+  mkdir -p "$(dirname "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}")"
+  touch "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  chmod 0644 "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}"
+  tee "${CONTAINERD_CONFIG_REGISTRY_HOST_MCR}" > /dev/null <<EOF
+[host."https://${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}"]
+  capabilities = ["pull", "resolve"]
+EOF
+}
+
 ensureNoDupOnPromiscuBridge() {
     systemctlEnableAndStart ensure-no-dup || exit $ERR_SYSTEMCTL_START_FAIL
 }