Skip to content

Commit

Permalink
fix: fix scenario that kube binary version is different from k8s vers…
Browse files Browse the repository at this point in the history 
…ion for network isolated cluster (#5335)
  • Loading branch information
@fseldow
fseldow authored Mar 7, 2025
1 parent 80fcfe4 commit b3c2497
Show file tree
Hide file tree
Showing 274 changed files with 2,324 additions and 365 deletions.
2 changes: 0 additions & 2 deletions e2e/scenario_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,6 @@ func Test_AzureLinuxV2_ARM64AirGap(t *testing.T) {
Cluster: ClusterKubenetAirgap,
VHD: config.VHDAzureLinuxV2Gen2Arm64,
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.CustomKubeBinaryURL = "https://acs-mirror.azureedge.net/kubernetes/v1.24.9/binaries/kubernetes-node-linux-arm64.tar.gz"
nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5"
nbc.IsARM64 = true

Expand Down Expand Up @@ -379,7 +378,6 @@ func Test_MarinerV2_ARM64AirGap(t *testing.T) {
Cluster: ClusterKubenetAirgap,
VHD: config.VHDCBLMarinerV2Gen2Arm64,
BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.CustomKubeBinaryURL = "https://acs-mirror.azureedge.net/kubernetes/v1.24.9/binaries/kubernetes-node-linux-arm64.tar.gz"
nbc.AgentPoolProfile.VMSize = "Standard_D2pds_V5"
nbc.IsARM64 = true

Expand Down
26 changes: 26 additions & 0 deletions parts/linux/cloud-init/artifacts/cse_helpers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,32 @@ removeKubeletNodeLabel() {
fi
}

# generate kubenode binary registry url from acs-mirror url
updateKubeBinaryRegistryURL() {
# if rp already passes registry url, then directly use the registry url that rp passes
# this path should have not catch for now, but keep it for future
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
# however, the kubelet and kubectl binary version may different with kubernetes version due to hotfix or beta
# so that we still need to extract the binary version from kube_binary_url
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
if [[ -n ${KUBE_BINARY_URL} ]]; then
binary_version="v${KUBERNETES_VERSION}" # by default use Kubernetes versions
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
# Extract the version part
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

# removes the specified FLAG_STRING (which should be in the form of 'key=value') from KUBELET_FLAGS
removeKubeletFlag() {
local FLAG_STRING=$1
Expand Down
5 changes: 3 additions & 2 deletions parts/linux/cloud-init/artifacts/cse_install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,9 +616,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
# network isolated cluster
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads" # /opt folder will return permission error
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
# no egress traffic, default install will fail
# will exit if the download fails

Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
View file

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/line40.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,9 +562,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
if [[ -n ${KUBE_BINARY_URL} ]]; then
binary_version="v${KUBERNETES_VERSION}"
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
View file

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line40.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,9 +562,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line9.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
if [[ -n ${KUBE_BINARY_URL} ]]; then
binary_version="v${KUBERNETES_VERSION}"
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
View file

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line40.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,9 +562,10 @@ installKubeletKubectlAndKubeProxy() {
if [[ "$install_default_if_missing" == true ]]; then
if [[ -n ${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER} ]]; then
echo "Detect Bootstrap profile artifact is Cache, will use oras to pull artifact binary"
registry_url="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:v${KUBERNETES_VERSION}-linux-${CPU_ARCH}"
updateKubeBinaryRegistryURL

K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY="/tmp/kubernetes/downloads"
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} $registry_url false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}
logs_to_events "AKS.CSE.installKubeletKubectlAndKubeProxy.extractKubeBinaries" extractKubeBinaries ${KUBERNETES_VERSION} "${KUBE_BINARY_REGISTRY_URL:-}" false ${K8S_DOWNLOADS_TEMP_DIR_FROM_REGISTRY}

#TODO: remove the condition check on KUBE_BINARY_URL once RP change is released
elif (($(echo ${KUBERNETES_VERSION} | cut -d"." -f2) >= 17)) && [ -n "${KUBE_BINARY_URL}" ]; then
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line9.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,26 @@ removeKubeletNodeLabel() {
fi
}

updateKubeBinaryRegistryURL() {
if [[ -n "${KUBE_BINARY_URL}" ]] && isRegistryUrl "${KUBE_BINARY_URL}"; then
echo "KUBE_BINARY_URL is a registry url, will use it to pull the kube binary"
KUBE_BINARY_REGISTRY_URL="${KUBE_BINARY_URL}"
else
url_regex='https://[^/]+/kubernetes/v[0-9]+\.[0-9]+\..+/binaries/.+'
if [[ -n ${KUBE_BINARY_URL} ]]; then
binary_version="v${KUBERNETES_VERSION}"
if [[ ${KUBE_BINARY_URL} =~ $url_regex ]]; then
version_with_prefix="${KUBE_BINARY_URL#*kubernetes/}"
binary_version="${version_with_prefix%%/*}"
echo "Extracted version: $binary_version from KUBE_BINARY_URL: ${KUBE_BINARY_URL}"
else
echo "KUBE_BINARY_URL is formatted unexpectedly, will use the kubernetes version as binary version: $binary_version"
fi
fi
KUBE_BINARY_REGISTRY_URL="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/kubernetes-node:${binary_version}-linux-${CPU_ARCH}"
fi
}

removeKubeletFlag() {
local FLAG_STRING=$1
if grep -e ",${FLAG_STRING}" <<< "$KUBELET_FLAGS" > /dev/null 2>&1; then
Expand Down
Loading

0 comments on commit b3c2497

Please sign in to comment.