Merge branch 'master' into shafan/cloudprovider

.pipelines/.vsts-vhd-builder-pr-windows.yaml

@@ -27,6 +27,7 @@ pr:
       - vhdbuilder/packer/init-variables.sh
       - vhdbuilder/packer/windows/
       - parts/common/components.json
+      - parts/windows
     exclude:
       - vhdbuilder/release-notes
       - /**/*.md

.pipelines/.vsts-vhd-builder-release.yaml

@@ -189,6 +189,7 @@ parameters:
 variables:
   - group: aks-vuln-to-kusto
   - group: "AKS Node SIG UA Token (KV)"
+  - group: build_performance
 
 stages:
   - stage: build

.pipelines/.vsts-vhd-builder.yaml

@@ -37,6 +37,7 @@ pool:
 variables:
   - group: aks-vuln-to-kusto
   - group: "AKS Node SIG UA Token (KV)"
+  - group: build_performance
 
 stages:
   - stage: build

.pipelines/templates/.builder-release-template.yaml

@@ -205,6 +205,17 @@ steps:
     displayName: Convert Shared Image Gallery To VHD Blob In Classic Storage Account
     env:
       RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME)
+
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Build Performance Program
+    condition: and(succeeded(), eq(variables.ENVIRONMENT, 'test'))
+    inputs:
+      source: "specific"
+      downloadPath: "vhdbuilder/packer/buildperformance"
+      runVersion: "latest"
+      project: $(System.TeamProject)
+      pipeline: $(SOURCE_PIPELINE) 
+      artifactName: $(BUILD_PERFORMANCE_ARTIFACT_NAME)
 
   - bash: make -f packer.mk evaluate-build-performance
     condition: always()

packer.mk

@@ -113,7 +113,7 @@ test-scan-and-cleanup: az-login
 	@./vhdbuilder/packer/test-scan-and-cleanup.sh
 
 evaluate-build-performance: az-login
-	@./vhdbuilder/packer/build-performance/evaluate-build-performance.sh
+	@./vhdbuilder/packer/buildperformance/evaluate-build-performance.sh
 
 generate-prefetch-scripts:
 #ifeq (${MODE},linuxVhdMode)

parts/linux/cloud-init/artifacts/cse_benchmark_functions.sh

@@ -18,7 +18,6 @@ check_array_size() {
 }
 
 capture_benchmark() {
-  set +x
   local title="$1"
   title="${title//[[:space:]]/_}"
   title="${title//-/_}"
@@ -42,8 +41,6 @@ capture_benchmark() {
 }
 
 process_benchmarks() {
-  set +x
-
   if [ -z "${PERFORMANCE_DATA_FILE}" ] ; then
     return
   fi
@@ -53,17 +50,13 @@ process_benchmarks() {
   fi
 
   check_array_size benchmarks || { echo "Benchmarks array is empty"; return; }
-  # create script object, then append each section object to it in the for loop
-  script_object=$(jq -n --arg script_name "${SCRIPT_NAME}" '{($script_name): {}}')
 
   for ((i=0; i<${#benchmarks_order[@]}; i+=1)); do
     section_name=${benchmarks_order[i]}
     section_object=$(jq -n --arg section_name "${section_name}" --arg total_time_elapsed "${benchmarks[${section_name}]}" \
     '{($section_name): $total_time_elapsed'})
-    script_object=$(jq -n --argjson script_object "$script_object" --argjson section_object "$section_object" --arg script_name "${SCRIPT_NAME}" \
-    '$script_object | .[$script_name] += $section_object')
+    jq ". += $section_object" "${PERFORMANCE_DATA_FILE}" > temp-perf-file.json && mv temp-perf-file.json "${PERFORMANCE_DATA_FILE}"
   done
 
-  jq ". += $script_object" "${PERFORMANCE_DATA_FILE}" > temp-perf-file.json && mv temp-perf-file.json "${PERFORMANCE_DATA_FILE}"
   chmod 755 "${PERFORMANCE_DATA_FILE}"
 }

vhdbuilder/packer/build-performance/evaluate-build-performance.sh → vhdbuilder/packer/buildperformance/evaluate-build-performance.sh

@@ -22,7 +22,7 @@ if [[ $? -ne 0 ]]; then
 fi
 
 if [[ ${SCRIPT_COUNT} -eq 0 ]]; then
-  log_and_exit ${PERFORMANCE_DATA_FILE} "contains no scripts"
+  log_and_exit ${PERFORMANCE_DATA_FILE} "contains no data"
 fi
 
 echo -e "\nGenerating build performance data for ${SIG_IMAGE_NAME}...\n"
@@ -35,26 +35,36 @@ jq --arg sig_name "${SIG_IMAGE_NAME}" \
   --arg status "${JOB_STATUS}" \
   --arg branch "${GIT_BRANCH}" \
   --arg commit "${GIT_VERSION}" \
-  '{sig_image_name: $sig_name, architecture: $arch, captured_sig_version: $captured_sig_version, build_id: $build_id, build_datetime: $date,
-  build_status: $status, branch: $branch, commit: $commit, scripts: .}' \
-  ${PERFORMANCE_DATA_FILE} > ${SIG_IMAGE_NAME}-build-performance.json
+  'to_entries | ([
+  {key: "sig_image_name", value: $sig_name},
+  {key: "architecture", value: $arch},
+  {key: "captured_sig_version", value: $captured_sig_version},
+  {key: "build_id", value: $build_id},
+  {key: "build_datetime", value: $date},
+  {key: "outcome", value: $status},
+  {key: "branch", value: $branch},
+  {key: "commit", value: $commit}
+] + .) | from_entries' ${PERFORMANCE_DATA_FILE} > ${SIG_IMAGE_NAME}-build-performance.json
 
 rm ${PERFORMANCE_DATA_FILE}
 
-echo "##[group]Build Information"
-jq -C '. | {sig_image_name, architecture, captured_sig_version, build_id, build_datetime, build_status, branch, commit}' ${SIG_IMAGE_NAME}-build-performance.json
+echo "##[group]Build Performance"
+jq . -C ${SIG_IMAGE_NAME}-build-performance.json
 echo "##[endgroup]"
 
-scripts=()
-for entry in $(jq -rc '.scripts | to_entries[]' ${SIG_IMAGE_NAME}-build-performance.json); do
-  scripts+=("$(echo "$entry" | jq -r '.key')")
-done
+echo -e "\nENVIRONMENT is: ${ENVIRONMENT}"
+if [ "${ENVIRONMENT,,}" == "test" ]; then
+  mv ${SIG_IMAGE_NAME}-build-performance.json vhdbuilder/packer/buildperformance
+  pushd vhdbuilder/packer/buildperformance || exit 0
+    echo -e "\nRunning build performance evaluation program...\n"
+    chmod +x ${BUILD_PERFORMANCE_BINARY}
+    ./${BUILD_PERFORMANCE_BINARY}
+    rm ${BUILD_PERFORMANCE_BINARY}
+  popd || exit 0
+else
+  echo -e "Skipping build performance evaluation for prod"
+fi
 
-for script in "${scripts[@]}"; do
-  echo "##[group]${script}"
-  jq -C ".scripts.\"$script\"" ${SIG_IMAGE_NAME}-build-performance.json
-  echo "##[endgroup]"
-done
+rm vhdbuilder/packer/buildperformance/${SIG_IMAGE_NAME}-build-performance.json
 
-rm ${SIG_IMAGE_NAME}-build-performance.json
 echo -e "\nBuild performance evaluation script completed."

vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh

@@ -113,7 +113,6 @@ if [[ "$sas" == "None" ]]; then
  echo "sas token empty after trying both queries. Can't continue"
  exit 1
 fi
-
 capture_benchmark "${SCRIPT_NAME}_grant_access_to_disk"
 
 echo "Uploading $disk_resource_id to ${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd"

vhdbuilder/packer/install-dependencies.sh

@@ -31,7 +31,7 @@ PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json
 
 echo ""
 echo "Components downloaded in this VHD build (some of the below components might get deleted during cluster provisioning if they are not needed):" >> ${VHD_LOGS_FILEPATH}
-capture_benchmark "${SCRIPT_NAME}_declare_variables_and_source_packer_files"
+capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables"
 
 echo "Logging the kernel after purge and reinstall + reboot: $(uname -r)"
 # fix grub issue with cvm by reinstalling before other deps
@@ -41,6 +41,7 @@ if grep -q "cvm" <<< "$FEATURE_FLAGS"; then
   wait_for_apt_locks
   apt_get_install 30 1 600 grub-efi || exit 1
 fi
+capture_benchmark "${SCRIPT_NAME}_reinstall_grub_for_cvm"
 
 if [[ "$OS" == "$UBUNTU_OS_NAME" ]]; then
   # disable and mask all UU timers/services
@@ -56,7 +57,6 @@ APT::Periodic::AutocleanInterval "0";
 APT::Periodic::Unattended-Upgrade "0";
 EOF
 fi
-capture_benchmark "${SCRIPT_NAME}_purge_and_reinstall_ubuntu"
 
 # If the IMG_SKU does not contain "minimal", installDeps normally
 if [[ "$IMG_SKU" != *"minimal"* ]]; then
@@ -99,7 +99,7 @@ SystemMaxUse=1G
 RuntimeMaxUse=1G
 ForwardToSyslog=yes
 EOF
-capture_benchmark "${SCRIPT_NAME}_install_dependencies"
+capture_benchmark "${SCRIPT_NAME}_install_deps_and_set_configs"
 
 if [[ ${CONTAINER_RUNTIME:-""} != "containerd" ]]; then
   echo "Unsupported container runtime. Only containerd is supported for new VHD builds."
@@ -124,7 +124,7 @@ if ! isMarinerOrAzureLinux "$OS"; then
   overrideNetworkConfig || exit 1
   disableNtpAndTimesyncdInstallChrony || exit 1
 fi
-capture_benchmark "${SCRIPT_NAME}_check_container_runtime_and_network_configurations"
+capture_benchmark "${SCRIPT_NAME}_validate_container_runtime_and_override_ubuntu_net_config"
 
 CONTAINERD_SERVICE_DIR="/etc/systemd/system/containerd.service.d"
 mkdir -p "${CONTAINERD_SERVICE_DIR}"
@@ -139,6 +139,7 @@ net.ipv4.conf.all.forwarding = 1
 net.ipv6.conf.all.forwarding = 1
 net.bridge.bridge-nf-call-iptables = 1
 EOF
+capture_benchmark "${SCRIPT_NAME}_set_ip_forwarding"
 
 echo "set read ahead size to 15380 KB"
 AWK_PATH=$(command -v awk)
@@ -200,6 +201,7 @@ ENV{DEVTYPE}=="partition", ENV{AZURE_DISK_TYPE}=="?*", ENV{AZURE_DISK_SERIAL}=="
 LABEL="azure_disk_end"
 EOF
 udevadm control --reload
+capture_benchmark "${SCRIPT_NAME}_set_udev_rules"
 
 if isMarinerOrAzureLinux "$OS"; then
     disableSystemdResolvedCache
@@ -217,6 +219,7 @@ if isMarinerOrAzureLinux "$OS"; then
     enableCheckRestart
     activateNfConntrack
 fi
+capture_benchmark "${SCRIPT_NAME}_handle_azurelinux_configs"
 
 # doing this at vhd allows CSE to be faster with just mv 
 unpackTgzToCNIDownloadsDIR() {
@@ -241,7 +244,7 @@ downloadCNI() {
 
 echo "VHD will be built with containerd as the container runtime"
 updateAptWithMicrosoftPkg
-capture_benchmark "${SCRIPT_NAME}_create_containerd_service_directory_and_configure_runtime_and_network"
+capture_benchmark "${SCRIPT_NAME}_update_apt_with_msft_pkg"
 
 # check if COMPONENTS_FILEPATH exists
 if [ ! -f $COMPONENTS_FILEPATH ]; then
@@ -399,7 +402,7 @@ cliTool="ctr"
 
 INSTALLED_RUNC_VERSION=$(runc --version | head -n1 | sed 's/runc version //')
 echo "  - runc version ${INSTALLED_RUNC_VERSION}" >> ${VHD_LOGS_FILEPATH}
-capture_benchmark "${SCRIPT_NAME}_artifact_streaming_download"
+capture_benchmark "${SCRIPT_NAME}_configure_artifact_streaming_and_install_crictl"
 
 GPUContainerImages=$(jq  -c '.GPUContainerImages[]' $COMPONENTS_FILEPATH)
 
@@ -454,7 +457,7 @@ PRESENT_DIR=$(pwd)
 BCC_PID=$!
 
 echo "${CONTAINER_RUNTIME} images pre-pulled:" >> ${VHD_LOGS_FILEPATH}
-capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_image_and_run_installBcc_in_subshell"
+capture_benchmark "${SCRIPT_NAME}_pull_nvidia_driver_and_start_ebpf_downloads"
 
 string_replace() {
   echo ${1//\*/$2}
@@ -514,21 +517,20 @@ watcherStaticImg=${watcherBaseImg//\*/static}
 
 # can't use cliTool because crictl doesn't support retagging.
 retagContainerImage "ctr" ${watcherFullImg} ${watcherStaticImg}
-capture_benchmark "${SCRIPT_NAME}_pull_and_retag_container_images"
 
 # IPv6 nftables rules are only available on Ubuntu or Mariner/AzureLinux
 if [[ $OS == $UBUNTU_OS_NAME ]] || isMarinerOrAzureLinux "$OS"; then
   systemctlEnableAndStart ipv6_nftables || exit 1
 fi
-capture_benchmark "${SCRIPT_NAME}_configure_networking_and_interface"
+capture_benchmark "${SCRIPT_NAME}_pull_and_retag_container_images"
 
 if [[ $OS == $UBUNTU_OS_NAME && $(isARM64) != 1 ]]; then  # no ARM64 SKU with GPU now
 NVIDIA_DEVICE_PLUGIN_VERSION="v0.14.5"
 
 DEVICE_PLUGIN_CONTAINER_IMAGE="mcr.microsoft.com/oss/nvidia/k8s-device-plugin:${NVIDIA_DEVICE_PLUGIN_VERSION}"
 pullContainerImage ${cliTool} ${DEVICE_PLUGIN_CONTAINER_IMAGE}
 fi
-capture_benchmark "download_gpu_device_plugin"
+capture_benchmark "${SCRIPT_NAME}_download_gpu_device_plugin"
 
 mkdir -p /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events
 
@@ -548,8 +550,7 @@ fi
 
 cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/*
 rm -r /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1
-
-capture_benchmark "${SCRIPT_NAME}_configure_telemetry_create_logging_directory"
+capture_benchmark "${SCRIPT_NAME}_configure_telemetry"
 
 # download kubernetes package from the given URL using MSI for auth for azcopy
 # if it is a kube-proxy package, extract image from the downloaded package
@@ -588,6 +589,7 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
   # multi-user.target usually start at the end of the boot sequence
   sed -i 's/After=network-online.target/After=multi-user.target/g' /lib/systemd/system/motd-news.service
 fi
+capture_benchmark "${SCRIPT_NAME}_purge_and_update_ubuntu"
 
 wait $BCC_PID
 BCC_EXIT_CODE=$?

vhdbuilder/packer/post-install-dependencies.sh

@@ -16,7 +16,7 @@ PERFORMANCE_DATA_FILE=/opt/azure/vhd-build-performance-data.json
 
 # Hardcode the desired size of the OS disk so we don't accidently rely on extra disk space
 MAX_BLOCK_COUNT=30298176 # 30 GB
-capture_benchmark "${SCRIPT_NAME}_set_variables_and_source_packer_files"
+capture_benchmark "${SCRIPT_NAME}_source_packer_files_and_declare_variables"
 
 if [[ $OS == $UBUNTU_OS_NAME ]]; then
   # shellcheck disable=SC2021
@@ -34,15 +34,16 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
   retrycmd_if_failure 10 2 60 apt-get -y autoclean || exit 1
   retrycmd_if_failure 10 2 60 apt-get -y autoremove --purge || exit 1
   retrycmd_if_failure 10 2 60 apt-get -y clean || exit 1
+  capture_benchmark "${SCRIPT_NAME}_purge_ubuntu_kernels_and_packages"
 
   # Final step, if 18.04 or FIPS, log ua status, detach UA and clean up
   if [[ "${UBUNTU_RELEASE}" == "18.04" ]] || [[ "${UBUNTU_RELEASE}" == "20.04" ]] || [[ "${ENABLE_FIPS,,}" == "true" ]]; then
     # 'ua status' for logging
     ua status
     detachAndCleanUpUA
   fi
+  capture_benchmark "${SCRIPT_NAME}_log_and_detach_ua"
 fi
-capture_benchmark "${SCRIPT_NAME}_log_and_detach_ua"
 
 # shellcheck disable=SC2129
 echo "kubelet/kubectl downloaded:" >> ${VHD_LOGS_FILEPATH}
@@ -52,7 +53,6 @@ ls -ltr /usr/local/bin/* >> ${VHD_LOGS_FILEPATH}
 ls -ltr /dev/* | grep sgx >>  ${VHD_LOGS_FILEPATH} 
 
 echo -e "=== Installed Packages Begin\n$(listInstalledPackages)\n=== Installed Packages End" >> ${VHD_LOGS_FILEPATH}
-capture_benchmark "${SCRIPT_NAME}_list_installed_packages"
 
 echo "Disk usage:" >> ${VHD_LOGS_FILEPATH}
 df -h >> ${VHD_LOGS_FILEPATH}
@@ -64,7 +64,6 @@ usage=$(awk -v used=${used_blocks} -v capacity=${MAX_BLOCK_COUNT} 'BEGIN{print (
 usage=${usage%.*}
 [ ${usage} -ge 99 ] && echo "ERROR: root partition on OS device (${os_device}) already passed 99% of the 30GB cap!" && exit 1
 [ ${usage} -ge 75 ] && echo "WARNING: root partition on OS device (${os_device}) already passed 75% of the 30GB cap!"
-capture_benchmark "${SCRIPT_NAME}_determine_disk_usage"
 
 echo -e "=== os-release Begin" >> ${VHD_LOGS_FILEPATH}
 cat /etc/os-release >> ${VHD_LOGS_FILEPATH}
@@ -83,7 +82,7 @@ tee -a ${VHD_LOGS_FILEPATH} < /proc/version
   echo "Container runtime: ${CONTAINER_RUNTIME}"
   echo "FIPS enabled: ${ENABLE_FIPS}"
 } >> ${VHD_LOGS_FILEPATH}
-capture_benchmark "${SCRIPT_NAME}_write_logs"
+capture_benchmark "${SCRIPT_NAME}_finish_vhd_build_logs"
 
 if [[ $(isARM64) != 1 ]]; then
   # no asc-baseline-1.1.0-268.arm64.deb