validators.go

package e2e

import (
	"bytes"
	"context"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"net"
	"os"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/Masterminds/semver/v3"
	"github.com/samber/lo"
	"github.com/tidwall/gjson"

	"github.com/Azure/agentbaker/e2e/components"
	"github.com/Azure/agentbaker/e2e/config"
	"github.com/Azure/agentbaker/pkg/agent"
	"github.com/Azure/agentbaker/pkg/agent/datamodel"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	certv1 "k8s.io/api/certificates/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
)

func ValidateTLSBootstrapping(ctx context.Context, s *Scenario) {
	switch s.VHD.OS {
	case config.OSWindows:
		validateTLSBootstrappingWindows(ctx, s)
	default:
		validateTLSBootstrappingLinux(ctx, s)
	}
}

func validateTLSBootstrappingLinux(ctx context.Context, s *Scenario) {
	kubeletLogs := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl").stdout
	switch {
	case s.SecureTLSBootstrappingEnabled() && s.Tags.BootstrapTokenFallback:
		s.T.Logf("will validate bootstrapping mode: secure TLS bootstrapping failure with bootstrap token fallback")
		require.True(
			s.T,
			!strings.Contains(kubeletLogs, "unable to validate bootstrap credentials") && strings.Contains(kubeletLogs, "kubelet bootstrap token credential is valid"),
			"expected to have successfully validated bootstrap token credential before kubelet startup, but did not",
		)
	case s.SecureTLSBootstrappingEnabled():
		s.T.Logf("will validate bootstrapping mode: secure TLS bootstrapping")
		ValidateSystemdUnitIsRunning(ctx, s, "secure-tls-bootstrap")
		validateKubeletClientCSRCreatedBySecureTLSBootstrapping(ctx, s)
		require.True(
			s.T,
			!strings.Contains(kubeletLogs, "unable to validate bootstrap credentials") && strings.Contains(kubeletLogs, "client credential already exists within kubeconfig"),
			"expected to already have a valid kubeconfig before kubelet start-up obtained through secure TLS bootstrapping, but did not",
		)
	default:
		s.T.Logf("will validate bootstrapping mode: bootstrap token")
		ValidateSystemdUnitIsNotRunning(ctx, s, "secure-tls-bootstrap")
		ValidateSystemdUnitIsNotFailed(ctx, s, "secure-tls-bootstrap")
		require.True(
			s.T,
			!strings.Contains(kubeletLogs, "unable to validate bootstrap credentials") && strings.Contains(kubeletLogs, "kubelet bootstrap token credential is valid"),
			"expected to have successfully validated bootstrap token credential before kubelet startup, but did not",
		)
	}
	if s.KubeletConfigFileEnabled() {
		ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", "\"rotateCertificates\": true")
	} else {
		ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--rotate-certificates=true")
	}
	ValidateDirectoryContent(ctx, s, "/var/lib/kubelet", []string{"kubeconfig"})
	ValidateDirectoryContent(ctx, s, "/var/lib/kubelet/pki", []string{"kubelet-client-current.pem"})
}

func validateTLSBootstrappingWindows(ctx context.Context, s *Scenario) {
	ValidateWindowsProcessContainsArgumentStrings(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true"})
	ValidateDirectoryContent(ctx, s, "c:\\k", []string{" config "})
	ValidateDirectoryContent(ctx, s, "c:\\k\\pki", []string{"kubelet-client-current.pem"})
	switch {
	case s.SecureTLSBootstrappingEnabled() && s.Tags.BootstrapTokenFallback:
		s.T.Logf("will validate bootstrapping mode: secure TLS bootstrapping failure with bootstrap token fallback")
		// nothing to validate other than node readiness
	case s.SecureTLSBootstrappingEnabled():
		s.T.Logf("will validate bootstrapping mode: secure TLS bootstrapping")
		validateKubeletClientCSRCreatedBySecureTLSBootstrapping(ctx, s)
	default:
		s.T.Logf("will validate bootstrapping mode: bootstrap token")
		// nothing to validate other than node readiness
	}
}

func ValidateKubeletServingCertificateRotation(ctx context.Context, s *Scenario) {
	switch s.VHD.OS {
	case config.OSWindows:
		validateKubeletServingCertificateRotationWindows(ctx, s)
	default:
		validateKubeletServingCertificateRotationLinux(ctx, s)
	}
}

func validateKubeletServingCertificateRotationLinux(ctx context.Context, s *Scenario) {
	if _, ok := s.Runtime.VM.VMSS.Tags["aks-disable-kubelet-serving-certificate-rotation"]; ok {
		s.T.Logf("linux VMSS has KSCR disablement tag, will validate that KSCR has been disabled")
		ValidateDirectoryContent(ctx, s, "/etc/kubernetes/certs", []string{"kubeletserver.crt", "kubeletserver.key"})
		ValidateFileExcludesContent(ctx, s, "/etc/default/kubelet", "kubernetes.azure.com/kubelet-serving-ca=cluster")
		if s.KubeletConfigFileEnabled() {
			ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", "\"tlsCertFile\": \"/etc/kubernetes/certs/kubeletserver.crt\"")
			ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", "\"tlsPrivateKeyFile\": \"/etc/kubernetes/certs/kubeletserver.key\"")
			ValidateFileExcludesContent(ctx, s, "/etc/default/kubeletconfig.json", "\"serverTLSBootstrap\": true")
		} else {
			ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--tls-cert-file")
			ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--tls-private-key-file")
			ValidateFileExcludesContent(ctx, s, "/etc/default/kubelet", "--rotate-server-certificates=true")
		}
		return
	}
	s.T.Logf("will validate linux KSCR enablement")
	ValidateDirectoryContent(ctx, s, "/var/lib/kubelet/pki", []string{"kubelet-server-current.pem"})
	ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "kubernetes.azure.com/kubelet-serving-ca=cluster")
	if s.KubeletConfigFileEnabled() {
		ValidateFileExcludesContent(ctx, s, "/etc/default/kubeletconfig.json", "\"tlsCertFile\": \"/etc/kubernetes/certs/kubeletserver.crt\"")
		ValidateFileExcludesContent(ctx, s, "/etc/default/kubeletconfig.json", "\"tlsPrivateKeyFile\": \"/etc/kubernetes/certs/kubeletserver.key\"")
		ValidateFileHasContent(ctx, s, "/etc/default/kubeletconfig.json", "\"serverTLSBootstrap\": true")
	} else {
		ValidateFileExcludesContent(ctx, s, "/etc/default/kubelet", "--tls-cert-file")
		ValidateFileExcludesContent(ctx, s, "/etc/default/kubelet", "--tls-private-key-file")
		ValidateFileHasContent(ctx, s, "/etc/default/kubelet", "--rotate-server-certificates=true")
	}
}

func validateKubeletServingCertificateRotationWindows(ctx context.Context, s *Scenario) {
	if _, ok := s.Runtime.VM.VMSS.Tags["aks-disable-kubelet-serving-certificate-rotation"]; ok {
		s.T.Logf("windows VMSS has KSCR disablement tag, will validate that KSCR has been disabled")
		ValidateDirectoryContent(ctx, s, "c:\\k\\pki", []string{"kubelet.crt", "kubelet.key"})
		ValidateWindowsProcessDoesNotContainArgumentStrings(ctx, s, "kubelet.exe", []string{"--rotate-server-certificates=true", "kubernetes.azure.com/kubelet-serving-ca=cluster"})
		return
	}
	s.T.Logf("will validate windows KSCR enablement")
	ValidateDirectoryContent(ctx, s, "c:\\k\\pki", []string{"kubelet-server-current.pem"})
	ValidateWindowsProcessContainsArgumentStrings(ctx, s, "kubelet.exe", []string{"--rotate-server-certificates=true", "kubernetes.azure.com/kubelet-serving-ca=cluster"})
	ValidateWindowsProcessDoesNotContainArgumentStrings(ctx, s, "kubelet.exe", []string{"--tls-cert-file", "--tls-private-key-file"})
}

func validateKubeletClientCSRCreatedBySecureTLSBootstrapping(ctx context.Context, s *Scenario) {
	fieldSelector := fmt.Sprintf("spec.signerName=%s", certv1.KubeAPIServerClientKubeletSignerName)
	kubeletClientCSRs, err := s.Runtime.Kube.Typed.CertificatesV1().CertificateSigningRequests().List(ctx, metav1.ListOptions{
		FieldSelector: fieldSelector,
	})
	require.NoError(s.T, err, "failed to list CSRs with field selector: %s", fieldSelector)
	var hasValidCSR bool
	for _, csr := range kubeletClientCSRs.Items {
		if len(csr.Status.Certificate) == 0 {
			continue
		}
		if strings.HasPrefix(strings.ToLower(csr.Spec.Username), "system:bootstrap:") {
			continue
		}
		if getNodeNameFromCSR(s, csr) == s.Runtime.VM.KubeName {
			hasValidCSR = true
			break
		}
	}
	require.True(s.T, hasValidCSR, "expected node %s to have created a kubelet client CSR which was approved and issued, using secure TLS bootstrapping", s.Runtime.VM.KubeName)
}

func getNodeNameFromCSR(s *Scenario, csr certv1.CertificateSigningRequest) string {
	block, _ := pem.Decode(csr.Spec.Request)
	require.NotNil(s.T, block)
	req, err := x509.ParseCertificateRequest(block.Bytes)
	require.NoError(s.T, err)
	return strings.TrimPrefix(req.Subject.CommonName, "system:node:")
}

func ValidateSystemdWatchdogForKubernetes132Plus(ctx context.Context, s *Scenario) {
	if k8sVersion := s.GetK8sVersion(); k8sVersion != "" && agent.IsKubernetesVersionGe(k8sVersion, "1.32.0") {
		// Validate systemd watchdog is enabled and configured for kubelet
		ValidateSystemdUnitIsRunning(ctx, s, "kubelet.service")
		ValidateFileHasContent(ctx, s, "/etc/systemd/system/kubelet.service.d/10-watchdog.conf", "WatchdogSec=60s")
		ValidateJournalctlOutput(ctx, s, "kubelet.service", "Starting systemd watchdog with interval")
	}
}

func ValidateAKSLogCollector(ctx context.Context, s *Scenario) {
	ValidateSystemdUnitIsNotFailed(ctx, s, "aks-log-collector")
}

func ValidateDiskQueueService(ctx context.Context, s *Scenario) {
	ValidateSystemdUnitIsRunning(ctx, s, "disk_queue.service")
}

func ValidateLeakedSecrets(ctx context.Context, s *Scenario) {
	secrets := map[string]string{
		"client private key":       base64.StdEncoding.EncodeToString([]byte(s.GetClientPrivateKey())),
		"service principal secret": base64.StdEncoding.EncodeToString([]byte(s.GetServicePrincipalSecret())),
		"bootstrap token":          s.GetTLSBootstrapToken(),
	}
	for _, logFile := range []string{"/var/log/azure/cluster-provision.log", "/var/log/azure/aks-node-controller.log"} {
		for _, secretValue := range secrets {
			if secretValue != "" {
				ValidateFileExcludesExactContent(ctx, s, logFile, secretValue)
			}
		}
	}
}

func ValidateSSHServiceEnabled(ctx context.Context, s *Scenario) {
	// Verify SSH service is active and running
	ValidateSystemdUnitIsRunning(ctx, s, "ssh")

	// Verify socket-based activation is disabled
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl is-active ssh.socket", 3, "could not check ssh.socket status")
	require.Contains(s.T, execResult.stdout, "inactive", "ssh.socket should be inactive")

	// Check that systemd recognizes SSH service should be active at boot
	execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl is-enabled ssh.service", 0, "could not check ssh.service status")
	require.Contains(s.T, execResult.stdout, "enabled", "ssh.service should be enabled at boot")
}

func ValidateDirectoryContent(ctx context.Context, s *Scenario, path string, files []string) {
	s.T.Helper()
	var steps []string
	if s.IsWindows() {
		steps = []string{
			"$ErrorActionPreference = \"Stop\"",
			fmt.Sprintf("Get-ChildItem -Path %s", path),
		}
	} else {
		steps = []string{
			"set -ex",
			fmt.Sprintf("sudo ls -la %s", path),
		}
	}
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not get directory contents")
	for _, file := range files {
		require.Contains(s.T, execResult.stdout, file, "expected to find file %s within directory %s, but did not.\nDirectory contents:\n%s", file, path, execResult.stdout)
	}
}

func ValidateSysctlConfig(ctx context.Context, s *Scenario, customSysctls map[string]string) {
	s.T.Helper()
	keysToCheck := make([]string, len(customSysctls))
	for k := range customSysctls {
		keysToCheck = append(keysToCheck, k)
	}
	command := []string{
		"set -ex",
		fmt.Sprintf("sudo sysctl %s | sed -E 's/([0-9])\\s+([0-9])/\\1 \\2/g'", strings.Join(keysToCheck, " ")),
	}
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "systmctl command failed")
	for name, value := range customSysctls {
		require.Contains(s.T, execResult.stdout, fmt.Sprintf("%s = %v", name, value), "expected to find %s set to %v, but was not.\nStdout:\n%s", name, value, execResult.stdout)
	}
}

// ValidateNetworkInterfaceConfig validates network interface configuration settings using ethtool.
// It identifies network interfaces with slot names matching the enP* pattern (same logic as the udev rule),
// then verifies that each interface has the expected configuration settings (e.g., rx buffer size).
// The nicConfig map specifies the ethtool settings to validate (key: setting name, value: expected value).
func ValidateNetworkInterfaceConfig(ctx context.Context, s *Scenario, nicConfig map[string]string) {
	s.T.Helper()

	// Get list of NICs using udevadm (same logic as udev rule)
	getNicsCommand := []string{
		"#!/usr/bin/env bash",
		"set -euo pipefail",
		"echo '=== NICs to Configure ==='",
		"enp_ifaces=()",
		"for dev in /sys/class/net/*; do",
		"  iface=\"$(basename \"$dev\")\"",
		"  slot=\"$(udevadm info -q property -p \"$dev\" 2>/dev/null | awk -F= '$1==\"ID_NET_NAME_SLOT\"{print $2; exit}' || true)\"",
		"  [[ \"$slot\" == enP* ]] && enp_ifaces+=(\"$iface\")",
		"done",
		"IFS=,; echo \"${enp_ifaces[*]}\"",
	}
	nicsResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(getNicsCommand, "\n"), 0, "could not get nics to configure")
	s.T.Logf("NICs to configure:\n%s", nicsResult.stdout)

	// Parse NIC output - it may be multi-line with header
	lines := strings.Split(strings.TrimSpace(nicsResult.stdout), "\n")
	nicsOutput := ""
	for _, line := range lines {
		line = strings.TrimSpace(line)
		// Skip header lines
		if strings.Contains(line, "===") || line == "" {
			continue
		}
		nicsOutput = line
		break
	}

	nics := strings.Split(nicsOutput, ",")

	s.T.Logf("Parsed NICs list: %v (count: %d)", nics, len(nics))

	if len(nics) == 0 || (len(nics) == 1 && strings.TrimSpace(nics[0]) == "") {
		s.T.Logf("No PCI devices (NICs) with enP* slot pattern found - skipping network interface config validation")
		return
	}

	for _, nic := range nics {
		// Skip empty entries
		nic = strings.TrimSpace(nic)
		if nic == "" {
			continue
		}

		s.T.Logf("Validating network interface config for NIC: %s", nic)

		// Get full ethtool output for debugging
		debugCommand := []string{
			"set -ex",
			fmt.Sprintf("echo '=== Full ethtool output for %s ==='", nic),
			fmt.Sprintf("sudo ethtool -g %s", nic),
		}
		debugResult := execScriptOnVMForScenario(ctx, s, strings.Join(debugCommand, "\n"))
		s.T.Logf("Full ethtool output for %s:\n%s", nic, debugResult.stdout)
		oldEthtool := strings.Contains(debugResult.stdout, "Current hardware settings")

		for setting, expectedValue := range nicConfig {
			var cmd string
			if oldEthtool {
				cmd = fmt.Sprintf("sudo ethtool -g %s | grep -A 5 'Current hardware settings' | grep -i %s: | awk '{print $2}'", nic, setting)
			} else {
				cmd = fmt.Sprintf("sudo ethtool --json -g %s | jq -r .[0].\\\"%s\\\"", nic, setting)
			}
			command := []string{
				"set -ex",
				cmd,
			}
			execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "could not get ethtool config")
			actualValue := strings.TrimSpace(execResult.stdout)
			s.T.Logf("Ethtool setting %s for NIC %s: expected=%s, actual=%s", setting, nic, expectedValue, actualValue)
			require.Equal(s.T, expectedValue, actualValue, "expected %s to be %s on nic %s, but got %s.\nFull ethtool output:\n%s", setting, expectedValue, nic, actualValue, debugResult.stdout)
		}
	}
}

// ValidateAzureNetworkFiles checks that udev rules files exist.
func ValidateAzureNetworkFiles(ctx context.Context, s *Scenario) {
	s.T.Helper()

	ValidateFileExists(ctx, s, "/opt/azure-network/configure-azure-network.sh")
	ValidateFileExists(ctx, s, "/etc/udev/rules.d/99-azure-network.rules")
}

func ValidateNvidiaSMINotInstalled(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		"sudo nvidia-smi",
	}
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 1, "")
	require.Contains(s.T, execResult.stderr, "nvidia-smi: command not found", "expected stderr to contain 'nvidia-smi: command not found', but got %q", execResult.stderr)
}

func ValidateNvidiaSMIInstalled(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{"set -ex", "sudo nvidia-smi"}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "could not execute nvidia-smi command")
}

func ValidateNvidiaModProbeInstalled(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		"sudo nvidia-modprobe",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "could not execute nvidia-modprobe command")
}

func ValidateNvidiaGRIDLicenseValid(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Capture the license status output, or continue silently if not found
		"license_status=$(sudo nvidia-smi -q | grep 'License Status' | grep 'Licensed' || true)",
		// If the output is empty, print an error message and exit with a nonzero code
		"if [ -z \"$license_status\" ]; then echo 'License status not valid or not found'; exit 1; fi",
		// Check that nvidia-gridd is active by capturing its is-active output
		"active_status=$(sudo systemctl is-active nvidia-gridd)",
		"if [ \"$active_status\" != \"active\" ]; then echo \"nvidia-gridd is not active: $active_status\"; exit 1; fi",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to validate nvidia-smi license state or nvidia-gridd service status")
}

func ValidateNvidiaPersistencedRunning(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Check that nvidia-persistenced.service is active by capturing its is-active output
		"active_status=$(sudo systemctl is-active nvidia-persistenced.service)",
		"if [ \"$active_status\" != \"active\" ]; then echo \"nvidia-gridd is not active: $active_status\"; exit 1; fi",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to validate nvidia-persistenced.service status")
}

// ValidateNvidiaGridV20DriverInstalled asserts the node installed the grid-v20
// (595.x) driver from the aks-gpu-grid-v20 image rather than falling back to a
// cuda/grid driver. This is the grid-v20-specific check: if SKU->driver-type
// selection regressed, nvidia-smi would report a different driver major.
func ValidateNvidiaGridV20DriverInstalled(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		"driver_version=$(sudo nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -n1 | tr -d '[:space:]')",
		"echo \"nvidia driver_version=$driver_version\"",
		"case \"$driver_version\" in 595.*) ;; *) echo \"expected grid-v20 595.x driver, got '$driver_version'\"; exit 1 ;; esac",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "expected grid-v20 (595.x) NVIDIA driver version")
}

func ValidateNonEmptyDirectory(ctx context.Context, s *Scenario, dirName string) {
	s.T.Helper()
	command := []string{
		"set -ex",
		fmt.Sprintf("sudo ls -1q %s | grep -q '^.*$' && true || false", dirName),
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "either could not find expected file, or something went wrong")
}

func ValidateInspektorGadget(ctx context.Context, s *Scenario) {
	s.T.Helper()

	skipFile := "/etc/ig.d/skip_vhd_ig"
	serviceName := "ig-import-gadgets.service"
	servicePath := "/usr/lib/systemd/system/" + serviceName

	// Check if IG is installed on this VHD by looking for the skip sentinel file.
	// The skip file is only present on VHDs that have IG installed (Ubuntu and Azure Linux non-OSGuard).
	// Flatcar, OSGuard, and older VHDs do not have IG installed and will not have the skip file.
	if !fileExist(ctx, s, skipFile) {
		s.T.Logf("Skipping Inspektor Gadget validation: sentinel file %s not found (VHD does not have IG installed)", skipFile)
		return
	}

	s.T.Logf("skip_vhd_ig sentinel file found, validating Inspektor Gadget installation")

	ValidateSystemdUnitIsNotFailed(ctx, s, serviceName)
	execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-enabled %s", serviceName), 0, fmt.Sprintf("%s should be enabled", serviceName))

	ValidateFileExists(ctx, s, skipFile)
	ValidateFileExists(ctx, s, servicePath)

	// Validate that gadgets were actually imported
	trackingFile := "/var/lib/ig/imported-gadgets.txt"
	ValidateFileExists(ctx, s, trackingFile)
	s.T.Logf("Validating imported gadgets tracking file is not empty")
	execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("test -s %s", trackingFile), 0, "tracking file should not be empty")

	// Verify ig image list shows imported gadgets
	s.T.Logf("Validating ig image list shows imported gadgets")
	result := execScriptOnVMForScenario(ctx, s, "sudo ig image list")
	if result.exitCode != "0" {
		s.T.Fatalf("ig image list failed with exit code %s, stderr: %s", result.exitCode, result.stderr)
	}
	if len(result.stdout) == 0 {
		s.T.Fatal("ig image list returned empty output, expected at least one imported gadget")
	}
	s.T.Logf("ig image list output:\n%s", result.stdout)

	// Run a simple gadget as a functional test.
	// We dynamically get the trace_exec tag from ig image list since gadgets are imported
	// with version tags (e.g., v0.45.0) matching components.json, not :latest.
	// IG requires root privileges to run eBPF programs.
	// We use timeout(1) to kill the gadget after 3s in case it hangs.
	// The ig --timeout flag expects an integer (seconds), not a duration string.
	// Exit codes: 0 = success, 124 = timeout killed it (also OK), anything else = failure.
	s.T.Logf("Running functional test with trace_exec gadget")
	funcTestScript := `
set -e
TRACE_EXEC_TAG=$(sudo ig image list | grep trace_exec | awk '{print $2}')
if [ -z "$TRACE_EXEC_TAG" ]; then
    echo "trace_exec gadget not found in ig image list"
    exit 1
fi
echo "Using trace_exec:$TRACE_EXEC_TAG"
timeout 3s sudo ig run "trace_exec:$TRACE_EXEC_TAG" --timeout 2 || EXIT_CODE=$?
if [ "${EXIT_CODE:-0}" != "0" ] && [ "${EXIT_CODE:-0}" != "124" ]; then
    echo "trace_exec gadget failed with exit code ${EXIT_CODE}"
    exit 1
fi
echo "trace_exec gadget ran successfully"
`
	execScriptOnVMForScenarioValidateExitCode(ctx, s, funcTestScript, 0, "trace_exec gadget should run successfully")
	s.T.Logf("Inspektor Gadget functional validation passed")
}

func ValidateFileExists(ctx context.Context, s *Scenario, fileName string) {
	s.T.Helper()
	if !fileExist(ctx, s, fileName) {
		s.T.Fatalf("expected file %s, but it does not", fileName)
	}
}

// ValidateACLFIPSEnabled asserts ACL-specific FIPS markers are present on the node:
// the /etc/system-fips marker file written by vhdbuilder/scripts/linux/acl/tool_installs_acl.sh.
// Kernel FIPS mode (/proc/sys/crypto/fips_enabled == 1) is universal and is asserted by
// ValidateFIPSProvider; callers should compose the two validators when both are needed.
func ValidateACLFIPSEnabled(ctx context.Context, s *Scenario) {
	s.T.Helper()
	ValidateFileExists(ctx, s, "/etc/system-fips")
}

func ValidateFileDoesNotExist(ctx context.Context, s *Scenario, fileName string) {
	s.T.Helper()
	if fileExist(ctx, s, fileName) {
		s.T.Fatalf("expected file %s to no exist, but it does", fileName)
	}
}

func ValidateFileIsRegularFile(ctx context.Context, s *Scenario, fileName string) {
	s.T.Helper()

	steps := []string{
		"set -ex",
		fmt.Sprintf("stat --printf=%%F %s | grep 'regular file'", fileName),
	}

	if execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n")).exitCode != "0" {
		s.T.Fatalf("expected %s to be a regular file, but it is not", fileName)
	}
}

func fileExist(ctx context.Context, s *Scenario, fileName string) bool {
	s.T.Helper()
	if s.IsWindows() {
		steps := []string{
			"$ErrorActionPreference = \"Stop\"",
			fmt.Sprintf("if (Test-Path -Path '%s') { exit 0 } else { exit 1 }", fileName),
		}
		execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n"))
		s.T.Logf("stdout: %s\nstderr: %s", execResult.stdout, execResult.stderr)
		return execResult.exitCode == "0"
	} else {
		steps := []string{
			"set -ex",
			fmt.Sprintf("test -f %s", fileName),
		}
		execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n"))
		return execResult.exitCode == "0"
	}
}

func getFileContent(ctx context.Context, s *Scenario, fileName string) (string, error) {
	s.T.Helper()
	var steps []string

	if s.IsWindows() {
		steps = []string{
			"$ErrorActionPreference = \"Stop\"",
			fmt.Sprintf("Get-Content %s", fileName),
		}
	} else {
		steps = []string{
			"set -ex",
			fmt.Sprintf("sudo cat %s", fileName),
		}
	}

	execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n"))
	if execResult.exitCode != "0" {
		return "", fmt.Errorf("failed to get file content for %s: exit code %s\nStdout: %s\nStderr: %s", fileName, execResult.exitCode, execResult.stdout, execResult.stderr)
	}

	return execResult.stdout, nil
}

func fileHasContent(ctx context.Context, s *Scenario, fileName string, contents string) bool {
	s.T.Helper()
	require.NotEmpty(s.T, contents, "Test setup failure: Can't validate that a file has contents with an empty string. Filename: %s", fileName)
	var steps []string
	if s.IsWindows() {
		steps = []string{
			"$ErrorActionPreference = \"Stop\"",
			fmt.Sprintf("if ( -not ( Test-Path -Path %s ) ) { exit 2 }", fileName),
			fmt.Sprintf("if (Select-String -Path %s -Pattern \"%s\" -SimpleMatch -Quiet) { exit 0 } else { exit 1 }", fileName, contents),
		}
	} else {
		steps = []string{
			"set -ex",
			fmt.Sprintf("(sudo cat %s | grep -q -F -e %q)", fileName, contents),
		}
	}

	execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n"))
	return execResult.exitCode == "0"

}

func fileHasExactContent(ctx context.Context, s *Scenario, fileName string, contents string) bool {
	s.T.Helper()
	require.NotEmpty(s.T, contents, "Test setup failure: Can't validate that a file has contents with an empty string. Filename: %s", fileName)
	encodedPattern := base64.StdEncoding.EncodeToString([]byte(contents))
	if s.IsWindows() {
		steps := []string{
			"$ErrorActionPreference = \"Stop\"",
			fmt.Sprintf("if ( -not ( Test-Path -Path %s ) ) { exit 2 }", fileName),
			fmt.Sprintf("$pattern = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('%s'))", encodedPattern),
			fmt.Sprintf("$content = Get-Content -Path %s -Raw", fileName),
			"$escaped = [regex]::Escape($pattern)",
			"if ([regex]::Match($content, \"(?<!\\w)\" + $escaped + \"(?!\\w)\").Success) { exit 0 } else { exit 1 }",
		}
		execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n"))
		return execResult.exitCode == "0"
	} else {
		steps := []string{
			"set -ex",
			fmt.Sprintf("if [ ! -f %s ]; then exit 2; fi", fileName),
			fmt.Sprintf("pattern=$(printf '%%s' '%s' | base64 -d)", encodedPattern),
			"escaped=$(printf '%s\n' \"$pattern\" | sed -e 's/[.\\[\\()*?^$+{}|]/\\\\&/g')",
			"regex='(^|[^[:alnum:]_])'\"$escaped\"'([^[:alnum:]_]|$)'",
			fmt.Sprintf("if sudo grep -Eq \"$regex\" %s; then exit 0; else exit 1; fi", fileName),
		}
		execResult := execScriptOnVMForScenario(ctx, s, strings.Join(steps, "\n"))
		return execResult.exitCode == "0"
	}
}

// ValidateFileHasContent passes the test if the specified file contains the specified contents.
// The contents doesn't need to be surrounded by non-word characters.
// E.g.: searching "bcd" in "abcdef" is a match, thus the validation passes.
func ValidateFileHasContent(ctx context.Context, s *Scenario, fileName string, contents string) {
	s.T.Helper()
	if !fileHasContent(ctx, s, fileName, contents) {
		actualContents, err := getFileContent(ctx, s, fileName)
		if err != nil {
			s.T.Fatalf("Expected file %s to have contents %q. Could not determine actual contents due to %s", fileName, contents, err.Error())
		} else {
			s.T.Fatalf("expected file %s to have contents %q, but it does not. It had contents %s", fileName, contents, actualContents)
		}
	}
}

// ValidateFileExcludesContent fails the test if the specified file contains the specified contents.
// The contents doesn't need to be surrounded by non-word characters.
// E.g.: searching "bcd" in "abcdef" is a match, thus the validation fails.
func ValidateFileExcludesContent(ctx context.Context, s *Scenario, fileName string, contents string) {
	s.T.Helper()
	if fileHasContent(ctx, s, fileName, contents) {
		actualContents, err := getFileContent(ctx, s, fileName)
		if err != nil {
			s.T.Fatalf("Expected file %s to have contents %q. Could not determine actual contents due to %s", fileName, contents, err.Error())
		} else {
			s.T.Fatalf("expected file %s to have contents %q, but it does. It had contents %s", fileName, contents, actualContents)
		}
	}
}

// ValidateFileExcludesExactContent fails the test if the specified file contains the specified contents.
// The contents needs to be surrounded by non-word characters.
// E.g.: searching "bcd" in "abcdef" is not a match, thus the validation passes.
func ValidateFileExcludesExactContent(ctx context.Context, s *Scenario, fileName string, contents string) {
	s.T.Helper()
	if fileHasExactContent(ctx, s, fileName, contents) {
		s.T.Fatalf("expected file %s to not have exact contents %q, but it does", fileName, contents)
	}
}

// ValidateFIPSProvider verifies that FIPS is properly configured on the node:
//  1. Kernel FIPS mode is enabled (/proc/sys/crypto/fips_enabled == 1).
//  2. OpenSSL (3.x) has an active FIPS or SymCrypt provider loaded. The check is
//     skipped on hosts shipping OpenSSL 1.1.x (e.g. Ubuntu 20.04 FIPS), which use
//     the legacy FIPS module rather than the providers interface.
//  3. /opt/cni/bin/portmap runs without panicking (regression guard for ICM 51000001009688
//     where the OpenSSL FIPS provider was not loaded on AzureLinux V3 FIPS nodes).
func ValidateFIPSProvider(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// 1. Kernel FIPS mode.
	fipsEnabled := execScriptOnVMForScenarioValidateExitCode(ctx, s, "cat /proc/sys/crypto/fips_enabled", 0, "could not read /proc/sys/crypto/fips_enabled")
	require.Equal(s.T, "1", strings.TrimSpace(fipsEnabled.stdout), "expected /proc/sys/crypto/fips_enabled to be 1, got %q", fipsEnabled.stdout)

	// 2. OpenSSL provider must include an active fips or symcrypt provider on OpenSSL 3.x.
	// 1.1.x (Ubuntu 20.04 FIPS) uses the legacy FIPS module and is skipped. Merge stderr
	// (`2>&1`) so a version banner written to stderr still parses.
	opensslVersion := execScriptOnVMForScenarioValidateExitCode(ctx, s, "openssl version 2>&1", 0, "could not run openssl version")
	versionFields := strings.Fields(opensslVersion.stdout)
	require.GreaterOrEqual(s.T, len(versionFields), 2,
		"could not parse openssl version output: %q", opensslVersion.stdout)
	version := versionFields[1]
	switch {
	case strings.HasPrefix(version, "3."):
		providers := execScriptOnVMForScenarioValidateExitCode(ctx, s, "openssl list -providers", 0, "could not list openssl providers")
		// Prefix match so "symcrypt" covers AzureLinux V3 / ACL's "symcryptprovider". See ICM 51000001009688.
		require.True(s.T, opensslProviderActive(providers.stdout, "fips", "symcrypt"),
			"expected openssl to have an active fips or symcrypt provider, got:\n%s", providers.stdout)
	case strings.HasPrefix(version, "1.1."):
		s.T.Logf("openssl providers check skipped: detected version %q (legacy FIPS module)", strings.TrimSpace(opensslVersion.stdout))
	default:
		s.T.Fatalf("unexpected openssl version %q: FIPS VHDs are expected to ship OpenSSL 3.x or 1.1.x", strings.TrimSpace(opensslVersion.stdout))
	}

	// 3. portmap panic check (best-effort). The original FIPS regression manifested as
	// /opt/cni/bin/portmap panicking when the OpenSSL FIPS provider was missing. Skip if
	// the binary isn't at the standard CNI path (e.g. ACL stages it under /opt/cni/downloads/);
	// checks 1 and 2 are already authoritative. Match specific Go runtime panic markers
	// rather than the bare substring `runtime error:` which appears in CNI usage text.
	portmapBin := "/opt/cni/bin/portmap"
	portmapPresent := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("test -x %s", portmapBin))
	if portmapPresent.exitCode != "0" {
		s.T.Logf("portmap panic check skipped: %s not present or not executable on this VHD", portmapBin)
		s.T.Logf("FIPS provider validation passed")
		return
	}
	portmap := execScriptOnVMForScenario(ctx, s, fmt.Sprintf("%s < /dev/null", portmapBin))
	panicMarkers := []*regexp.Regexp{
		regexp.MustCompile(`(?m)^panic:`),
		regexp.MustCompile(`(?m)^fatal error:`),
		regexp.MustCompile(`runtime\.gopanic`),
		regexp.MustCompile(`goroutine \d+ \[running\]`),
	}
	for _, re := range panicMarkers {
		require.False(s.T, re.MatchString(portmap.stderr),
			"portmap runtime failure matched %q, indicating FIPS provider misconfiguration:\nstdout:\n%s\nstderr:\n%s", re, portmap.stdout, portmap.stderr)
		require.False(s.T, re.MatchString(portmap.stdout),
			"portmap runtime failure matched %q, indicating FIPS provider misconfiguration:\nstdout:\n%s\nstderr:\n%s", re, portmap.stdout, portmap.stderr)
	}

	s.T.Logf("FIPS provider validation passed")
}

// Package-level regex compiled once at init.
var (
	// Provider header: indented (spaces or tabs) name with no key/value separator.
	// Group 1 = indent (used to scope status lines to their block), group 2 = name.
	opensslProviderHeaderRE = regexp.MustCompile(`^([ \t]+)(\S+)\s*$`)
	opensslStatusLineRE     = regexp.MustCompile(`^[ \t]+status:\s*(\S+)`)
)

// opensslProviderActive parses `openssl list -providers` output and returns true if any
// provider whose name has one of the given prefixes is reported as `status: active`.
// The status line is scoped to its enclosing provider block (strictly more indented than
// the header) so an active default provider cannot mask an inactive fips/symcrypt one.
func opensslProviderActive(output string, providerPrefixes ...string) bool {
	matches := func(name string) bool {
		for _, p := range providerPrefixes {
			if strings.HasPrefix(name, p) {
				return true
			}
		}
		return false
	}
	var current string
	var headerIndent int
	for _, line := range strings.Split(output, "\n") {
		line = strings.TrimRight(line, "\r")
		if m := opensslProviderHeaderRE.FindStringSubmatch(line); m != nil {
			headerIndent = len(m[1])
			current = m[2]
			continue
		}
		if current == "" || !matches(current) {
			continue
		}
		if m := opensslStatusLineRE.FindStringSubmatch(line); m != nil && m[1] == "active" {
			leading := len(line) - len(strings.TrimLeft(line, " \t"))
			if leading > headerIndent {
				return true
			}
		}
	}
	return false
}

func ServiceCanRestartValidator(ctx context.Context, s *Scenario, serviceName string, restartTimeoutInSeconds int) {
	s.T.Helper()
	steps := []string{
		"set -ex",
		// Verify the service is active - print the state then verify so we have logs
		fmt.Sprintf("(systemctl -n 5 status %s || true)", serviceName),
		fmt.Sprintf("systemctl is-active %s", serviceName),

		// get the PID of the service, so we can check it's changed
		fmt.Sprintf("INITIAL_PID=`sudo pgrep %s`", serviceName),
		"echo INITIAL_PID: $INITIAL_PID",

		// we use systemctl kill rather than kill -9 because container restrictions stop us sending a kill sig to a process
		fmt.Sprintf("sudo systemctl kill %s", serviceName),

		// sleep for restartTimeoutInSeconds seconds to give the service time tor restart
		fmt.Sprintf("sleep %d", restartTimeoutInSeconds),

		// print the status of the service and then verify it is active.
		fmt.Sprintf("(systemctl -n 5 status %s || true)", serviceName),
		fmt.Sprintf("systemctl is-active %s", serviceName),

		// get the PID of the service after restart, so we can check it's changed
		fmt.Sprintf("POST_PID=`sudo pgrep %s`", serviceName),
		"echo POST_PID: $POST_PID",

		// verify the PID has changed.
		"if [[ \"$INITIAL_PID\" == \"$POST_PID\" ]]; then echo PID did not change after restart, failing validator. ; exit 1; fi",
	}

	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "command to restart service failed")
}

func ValidateSystemdUnitIsRunning(ctx context.Context, s *Scenario, serviceName string) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Print the service status for logging purposes
		fmt.Sprintf("systemctl -n 5 status %s || true", serviceName),
		// Verify the service is active
		fmt.Sprintf("systemctl is-active %s", serviceName),
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		fmt.Sprintf("service %s is not running", serviceName))
}

func ValidateSystemdUnitIsNotRunning(ctx context.Context, s *Scenario, serviceName string) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Print the service status for logging purposes (allow failure)
		fmt.Sprintf("systemctl -n 5 status %s || true", serviceName),
		// Check if service is active - we expect this to fail
		fmt.Sprintf("! systemctl is-active %s", serviceName),
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		fmt.Sprintf("service %s is unexpectedly running", serviceName))
}

func ValidateWindowsServiceIsRunning(ctx context.Context, s *Scenario, serviceName string) {
	s.T.Helper()
	command := []string{
		"$ErrorActionPreference = \"Stop\"",
		// Print the service status for logging purposes
		fmt.Sprintf("Get-Service -Name %s", serviceName),
		// Verify the service is running
		fmt.Sprintf("$service = Get-Service -Name %s", serviceName),
		"if ($service.Status -ne 'Running') { throw \"Service is not running: $($service.Status)\" }",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		fmt.Sprintf("Windows service %s is not running", serviceName))
}

func ValidateWindowsServiceIsNotRunning(ctx context.Context, s *Scenario, serviceName string) {
	s.T.Helper()
	command := []string{
		"$ErrorActionPreference = \"Continue\"",
		// Print the service status for logging purposes
		fmt.Sprintf("Get-Service -Name %s -ErrorAction SilentlyContinue", serviceName),
		// Check if service exists and is not running
		fmt.Sprintf("$service = Get-Service -Name %s -ErrorAction SilentlyContinue", serviceName),
		"if ($service -and $service.Status -eq 'Running') { throw \"Service is unexpectedly running: $($service.Status)\" }",
		"if ($service -and $service.Status -ne 'Running') { Write-Host \"Service exists but is not running: $($service.Status)\" }",
		"if (-not $service) { Write-Host \"Service does not exist\" }",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		fmt.Sprintf("Windows service %s validation failed", serviceName))
}

func ValidateDotnetNotInstalledWindows(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"$ErrorActionPreference = \"Continue\"",
		"$dotnetCmd = Get-Command dotnet -ErrorAction SilentlyContinue",
		"if ($dotnetCmd) {",
		"  throw \".NET is installed at $($dotnetCmd.Source) but should not be present on the VHD\"",
		"}",
		"Write-Host \".NET is not installed\"",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		".NET should not be installed on the Windows node")
}

func ValidateWindowsSystemServiceRestartConfiguration(ctx context.Context, s *Scenario, serviceName string) {
	s.T.Helper()

	command := []string{
		fmt.Sprintf("sc.exe qfailure %s", serviceName),
	}

	execResult := execScriptOnVMForScenarioValidateExitCode(
		ctx,
		s,
		strings.Join(command, "\n"),
		0,
		fmt.Sprintf("failed to validate restart configuration for Windows service %s", serviceName),
	)

	var RESET_PERIOD = "RESET_PERIOD"
	var FAILURE_ACTIONS = "FAILURE_ACTIONS"

	fields := map[string]string{}
	sdtout := execResult.stdout
	lines := strings.Split(sdtout, "\n")
	for _, line := range lines {
		parts := strings.SplitN(line, ":", 2)
		if len(parts) != 2 {
			continue
		}
		key := strings.TrimSpace(parts[0])
		value := strings.TrimSpace(parts[1])
		if strings.Contains(key, RESET_PERIOD) {
			fields[RESET_PERIOD] = value
		}
		if strings.Contains(key, FAILURE_ACTIONS) {
			fields[FAILURE_ACTIONS] = value
		}
	}
	if fields[RESET_PERIOD] != "900" {
		s.T.Fatalf("Expected 'Reset fail counter after' to be set to 900 seconds for service %s, but got: %s", serviceName, sdtout)
	}
	if fields[FAILURE_ACTIONS] != "RESTART -- Delay = 60000 milliseconds." {
		s.T.Fatalf("Expected 'Failure actions' to be set to 'RESTART -- Delay = 60000 milliseconds.' for service %s, but got: %s", serviceName, sdtout)
	}
}

func ValidateWindowsSystemServicesRestartConfiguration(ctx context.Context, s *Scenario) {
	ValidateWindowsSystemServiceRestartConfiguration(ctx, s, "kubelet")
	ValidateWindowsSystemServiceRestartConfiguration(ctx, s, "containerd")
	ValidateWindowsSystemServiceRestartConfiguration(ctx, s, "kubeproxy")
}

func ValidateSystemdUnitIsNotFailed(ctx context.Context, s *Scenario, serviceName string) {
	s.T.Helper()
	command := []string{
		"set -ex",
		fmt.Sprintf("systemctl --no-pager -n 5 status %s || true", serviceName),
		fmt.Sprintf("systemctl is-failed %s", serviceName),
	}
	require.NotEqual(
		s.T,
		"0",
		execScriptOnVMForScenario(ctx, s, strings.Join(command, "\n")).exitCode,
		`expected "systemctl is-failed" to exit with a non-zero exit code for unit %q, unit is in a failed state`,
		serviceName,
	)
}

func ValidateNoFailedSystemdUnits(ctx context.Context, s *Scenario) {
	if s.VHD != nil && s.VHD.SkipOldVHDValidations {
		return
	}
	unitFailureAllowList := map[string]bool{
		// this service depends on non-network-isolated environment - E2Es are run in an environment
		// which simulates network-isolation by only allowing egress to recommended domains outlined
		// on public AKS documentation via a firewall. This service depends on some other domain which is
		// not currently allowed by the firewall. It also seems that this service is only installed on
		// Ubuntu - do we even need it? it seems that it's coming from the base image
		"fwupd-refresh.service": true,
	}
	// cloud-init creates temporary directories under /run/cloud-init/tmp/ during provisioning.
	// systemd auto-generates transient .mount units for these (e.g., run-cloud\x2dinit-tmp-tmpXXXXX.mount).
	// When cloud-init cleans up the temp directory, the mount unit may enter a "failed" state due to
	// a race in timing between cleanup and systemd state tracking. For this validator, that failure is
	// treated as benign, and the unit name contains a random suffix, so we use prefix matching instead
	// of exact string matching.
	mountFailureAllowPrefixes := []string{
		`run-cloud\x2dinit-tmp-`,
	}
	if s.Tags.BootstrapTokenFallback {
		// secure-tls-bootstrap.service is expected to fail within scenarios that test bootstrap token fall-back behavior
		unitFailureAllowList["secure-tls-bootstrap.service"] = true
	}
	if s.VHD.IgnoreFailedCgroupTelemetryServices {
		unitFailureAllowList["cgroup-memory-telemetry.service"] = true
		unitFailureAllowList["cgroup-pressure-telemetry.service"] = true
	}
	if s.VHD.OS == config.OSACL {
		// systemd-sysupdate.service: known upstream Flatcar issue (flatcar/Flatcar#1979). The timer
		// (OnBootSec=15min) fires the service which exits with "No transfer definitions found" because
		// ACL VHDs don't ship sysupdate transfer configs. Whether it fails depends on whether the timer
		// fires before the validator checks.
		unitFailureAllowList["systemd-sysupdate.service"] = true
	}

	type systemdUnit struct {
		Name string `json:"unit,omitempty"`
	}
	var failedUnits []systemdUnit
	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, "systemctl list-units --failed --output json", 0, fmt.Sprintf("unable to list failed systemd units"))
	assert.NoError(s.T, json.Unmarshal([]byte(result.stdout), &failedUnits), `unable to parse and unmarshal "systemctl list-units" command output`)
	failedUnits = lo.Filter(failedUnits, func(unit systemdUnit, _ int) bool {
		if unitFailureAllowList[unit.Name] {
			return false
		}
		if strings.HasSuffix(unit.Name, ".mount") {
			for _, prefix := range mountFailureAllowPrefixes {
				if strings.HasPrefix(unit.Name, prefix) {
					return false
				}
			}
		}
		return true
	})

	if len(failedUnits) < 1 {
		// no unexpectedly failed units
		return
	}

	// extract failed unit logs
	failedUnitLogs := make(map[string]string, len(failedUnits))
	for _, unit := range failedUnits {
		failedUnitLogs[unit.Name+".log"] = execScriptOnVMForScenario(ctx, s, fmt.Sprintf("journalctl -u %s", unit.Name)).String()
	}
	assert.NoError(s.T, dumpFileMapToDir(s.T, failedUnitLogs), "failed to dump failed systemd unit logs")

	s.T.Fatalf(
		"the following systemd units have unexpectedly entered a failed state: %s - failed unit logs will be included in scenario log bundle within <service-name>.service.log",
		lo.Map(failedUnits, func(unit systemdUnit, _ int) string {
			return unit.Name
		}),
	)
}

func ValidateUlimitSettings(ctx context.Context, s *Scenario, ulimits map[string]string) {
	s.T.Helper()
	ulimitKeys := make([]string, 0, len(ulimits))
	for k := range ulimits {
		ulimitKeys = append(ulimitKeys, k)
	}

	command := fmt.Sprintf("sudo systemctl cat containerd.service | grep -E -i '%s'", strings.Join(ulimitKeys, "|"))
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not read containerd.service file")

	for name, value := range ulimits {
		require.Contains(s.T, execResult.stdout, fmt.Sprintf("%s=%v", name, value), "expected to find %s set to %v, but was not", name, value)
	}
}

func ValidateInstalledPackageVersion(ctx context.Context, s *Scenario, component, version string) {
	s.T.Helper()
	installedCommand := func() string {
		switch s.VHD.OS {
		case config.OSUbuntu:
			return "sudo apt list --installed"
		case config.OSMariner, config.OSAzureLinux:
			return "sudo dnf list installed"
		default:
			s.T.Fatalf("command to get package list isn't implemented for OS %s", s.VHD.OS)
			return ""
		}
	}()
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, installedCommand, 0, "could not get package list")
	for _, line := range strings.Split(execResult.stdout, "\n") {
		if strings.Contains(line, component) && strings.Contains(line, version) {
			s.T.Logf("found %s %s in the installed packages", component, version)
			return
		}
	}
	s.T.Errorf("expected to find %s %s in the installed packages, but did not", component, version)
}

func ValidateKubeletNodeIP(ctx context.Context, s *Scenario) {
	s.T.Helper()
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo cat /etc/default/kubelet", 0, "could not read kubelet config")
	stdout := execResult.stdout

	// Search for "--node-ip" flag and its value.
	matches := regexp.MustCompile(`--node-ip=([a-zA-Z0-9.:,]*)`).FindStringSubmatch(stdout)
	require.NotNil(s.T, matches, "could not find kubelet flag --node-ip\nStdout: \n%s", stdout)
	require.GreaterOrEqual(s.T, len(matches), 2, "could not find kubelet flag --node-ip.\nStdout: \n%s", stdout)

	ipAddresses := strings.Split(matches[1], ",") // Could be multiple for dual-stack.
	require.GreaterOrEqual(s.T, len(ipAddresses), 1, "expected at least one --node-ip address, but got none\nStdout: \n%s", stdout)
	require.LessOrEqual(s.T, len(ipAddresses), 2, "expected at most two --node-ip addresses, but got %d\nStdout: \n%s", len(ipAddresses), stdout)

	// Check that each IP is a valid address.
	for _, ipAddress := range ipAddresses {
		require.NotNil(s.T, net.ParseIP(ipAddress), "--node-ip value %q is not a valid IP address\nStdout: \n%s", ipAddress, stdout)
	}
}

func ValidateIMDSRestrictionRule(ctx context.Context, s *Scenario, table string) {
	s.T.Helper()
	cmd := fmt.Sprintf("sudo iptables -t %s -S | grep -q 'AKS managed: added by AgentBaker ensureIMDSRestriction for IMDS restriction feature'", table)
	execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0, "expected to find IMDS restriction rule, but did not")
}

func ValidateMultipleKubeProxyVersionsExist(ctx context.Context, s *Scenario) {
	s.T.Helper()
	execResult := execScriptOnVMForScenario(ctx, s, "sudo ctr --namespace k8s.io images list | grep kube-proxy | awk '{print $1}' | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'")
	if execResult.exitCode != "0" {
		s.T.Errorf("Failed to list kube-proxy images: %s", execResult.stderr)
		return
	}

	versions := bytes.NewBufferString(strings.TrimSpace(execResult.stdout))
	versionMap := make(map[string]struct{})
	for _, version := range strings.Split(versions.String(), "\n") {
		if version != "" {
			versionMap[version] = struct{}{}
		}
	}

	switch len(versionMap) {
	case 0:
		s.T.Errorf("No kube-proxy versions found.")
	case 1:
		s.T.Errorf("Only one kube-proxy version exists: %v", versionMap)
	default:
		s.T.Logf("Multiple kube-proxy versions exist: %v", versionMap)
	}
}

func ValidateKubeletHasNotStopped(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := "sudo journalctl -u kubelet"
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "could not retrieve kubelet logs with journalctl")
	stdout := strings.ToLower(execResult.stdout)
	assert.NotContains(s.T, stdout, "stopped kubelet")
	assert.Contains(s.T, stdout, "started kubelet")
}

func ValidateServicesDoNotRestartKubelet(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// grep all filesin /etc/systemd/system/ for /restart\s+kubelet/ and count results
	command := "sudo grep -rl 'restart[[:space:]]\\+kubelet' /etc/systemd/system/"
	execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 1, "expected to find no services containing 'restart kubelet' in /etc/systemd/system/")
}

// ValidateKubeletHasFlags checks kubelet is started with the right flags and configs.
func ValidateKubeletHasFlags(ctx context.Context, s *Scenario, filePath string) {
	s.T.Helper()
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo journalctl -u kubelet", 0, "could not retrieve kubelet logs with journalctl")
	configFileFlags := fmt.Sprintf("FLAG: --config=\"%s\"", filePath)
	require.Containsf(s.T, execResult.stdout, configFileFlags, "expected to find flag %s, but not found", "config")
}

func ValidateContainerd2Properties(ctx context.Context, s *Scenario, versions []string) {
	s.T.Helper()
	require.Lenf(s.T, versions, 1, "Expected exactly one version for moby-containerd but got %d", len(versions))
	// assert versions[0] value starts with '2.'
	require.Truef(s.T, strings.HasPrefix(versions[0], "2."), "expected moby-containerd version to start with '2.', got %v", versions[0])

	ValidateInstalledPackageVersion(ctx, s, "moby-containerd", versions[0])

	execResult := execOnVMForScenarioOnUnprivilegedPod(ctx, s, "containerd config dump ")
	// validate containerd config dump has no warnings
	require.NotContains(s.T, execResult.stdout, "level=warning", "do not expect warning message when converting config file %", execResult.stdout)
}

func ValidateContainerRuntimePlugins(ctx context.Context, s *Scenario) {
	// nri plugin is enabled by default
	ValidateDirectoryContent(ctx, s, "/var/run/nri", []string{"nri.sock"})
}

func ValidateNPDGPUCountPlugin(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Check NPD GPU count plugin config exists
		"test -f /etc/node-problem-detector.d/custom-plugin-monitor/gpu_checks/custom-plugin-gpu-count.json",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "NPD GPU count plugin configuration does not exist")
}

func validateNPDCondition(ctx context.Context, s *Scenario, conditionType, conditionReason string, conditionStatus corev1.ConditionStatus, conditionMessage, conditionMessageErr string) {
	s.T.Helper()
	// Wait for NPD to report initial condition
	var condition *corev1.NodeCondition
	err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) {
		node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{})
		if err != nil {
			s.T.Logf("Failed to get node %q: %v", s.Runtime.VM.KubeName, err)
			return false, nil // Continue polling on transient errors
		}

		// Check for condition with correct reason
		for i := range node.Status.Conditions {
			if string(node.Status.Conditions[i].Type) == conditionType && string(node.Status.Conditions[i].Reason) == conditionReason {
				condition = &node.Status.Conditions[i] // Found the partial condition we are looking for
			}

			if strings.Contains(node.Status.Conditions[i].Message, conditionMessage) {
				condition = &node.Status.Conditions[i]
				return true, nil // Found the exact condition we are looking for
			}
		}

		return false, nil // Continue polling until the condition is found or timeout occurs
	})
	if err != nil && condition == nil {
		require.NoError(s.T, err, "timed out waiting for %s condition with reason %s to appear on node %q", conditionType, conditionReason, s.Runtime.VM.KubeName)
	}

	require.NotNil(s.T, condition, "expected to find %s condition with %s reason on node", conditionType, conditionReason)
	require.Equal(s.T, condition.Status, conditionStatus, "expected %s condition to be %s", conditionType, conditionStatus)
	require.Contains(s.T, condition.Message, conditionMessage, conditionMessageErr)
}

func ValidateNPDGPUCountCondition(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Validate that NPD is reporting healthy GPU count
	validateNPDCondition(ctx, s, "GPUMissing", "NoGPUMissing", corev1.ConditionFalse,
		"All GPUs are present", "expected GPUMissing message to indicate correct count")
}

func ValidateNPDGPUCountAfterFailure(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Stop all services that are holding on to the GPUs
		"sudo systemctl stop nvidia-persistenced.service || true",
		"sudo systemctl stop nvidia-fabricmanager || true",
		// Disable and reset the first GPU
		"sudo nvidia-smi -i 0 -pm 0", // Disable persistence mode
		"sudo nvidia-smi -i 0 -c 0",  // Set compute mode to default
		// sed converts the output into the format needed for NVreg_ExcludeDevices
		"PCI_ID=$(sudo nvidia-smi -i 0 --query-gpu=pci.bus_id --format=csv,noheader | sed 's/^0000//')",
		"echo ${PCI_ID} | tee /tmp/npd_test_disabled_pci_id",
		"echo ${PCI_ID} | sudo tee /sys/bus/pci/drivers/nvidia/unbind", // Reset the GPU
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to disable GPU")

	// Validate that NPD reports the GPU count mismatch
	validateNPDCondition(ctx, s, "GPUMissing", "GPUMissing", corev1.ConditionTrue,
		"Expected to see 8 GPUs but found 7. FaultCode: NHC2009", "expected GPUMissing message to indicate GPU count mismatch")

	command = []string{
		"set -ex",
		"cat /tmp/npd_test_disabled_pci_id | sudo tee /sys/bus/pci/drivers/nvidia/bind",
		"rm -f /tmp/npd_test_disabled_pci_id", // Clean up the temporary file
		"sudo systemctl start nvidia-persistenced.service || true",
	}
	// Put the VM back to the original state, re-enable the GPU.
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to re-enable GPU")
}

func ValidateNPDIBLinkFlappingCondition(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Validate that NPD is reporting no IB link flapping
	validateNPDCondition(ctx, s, "IBLinkFlapping", "NoIBLinkFlapping", corev1.ConditionFalse,
		"IB link is stable", "expected IBLinkFlapping message to indicate no flapping")
}

func ValidateNPDIBLinkFlappingAfterFailure(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Simulate IB link flapping
	command := []string{
		"set -ex",
		"echo \"$(date '+%b %d %H:%M:%S') $(hostname) fake error 0: [12346.123456] ib0: lost carrier\" | sudo tee -a /var/log/syslog",
		"sleep 60",
		"echo \"$(date '+%b %d %H:%M:%S') $(hostname) fake error 1: [12346.123456] ib0: lost carrier\" | sudo tee -a /var/log/syslog",
		"sleep 60",
		"echo \"$(date '+%b %d %H:%M:%S') $(hostname) fake error 2: [12346.123456] ib0: lost carrier\" | sudo tee -a /var/log/syslog",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to simulate IB link flapping")

	// Validate that NPD reports IB link flapping
	expectedMessage := "check_ib_link_flapping: IB link flapping detected, multiple IB link flapping events within 6 hours. FaultCode: NHC2005"
	validateNPDCondition(ctx, s, "IBLinkFlapping", "IBLinkFlapping", corev1.ConditionTrue,
		expectedMessage, "expected IBLinkFlapping message to indicate flapping")
}

func ValidateNPDUnhealthyNvidiaDevicePlugin(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Check NPD unhealthy Nvidia device plugin config exists
		"test -f /etc/node-problem-detector.d/custom-plugin-monitor/gpu_checks/custom-plugin-nvidia-device-plugin.json",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "NPD Nvidia device plugin configuration does not exist")
}

func ValidateNPDUnhealthyNvidiaDevicePluginCondition(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Validate that NPD is reporting healthy Nvidia device plugin
	validateNPDCondition(ctx, s, "UnhealthyNvidiaDevicePlugin", "HealthyNvidiaDevicePlugin", corev1.ConditionFalse,
		"NVIDIA device plugin is running properly", "expected UnhealthyNvidiaDevicePlugin message to indicate healthy status")
}

func ValidateNPDUnhealthyNvidiaDevicePluginAfterFailure(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Stop Nvidia device plugin systemd service to simulate failure
	command := []string{
		"set -ex",
		"sudo systemctl stop nvidia-device-plugin.service",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to stop Nvidia device plugin service")

	// Validate that NPD reports unhealthy Nvidia device plugin
	validateNPDCondition(ctx, s, "UnhealthyNvidiaDevicePlugin", "UnhealthyNvidiaDevicePlugin", corev1.ConditionTrue,
		"Systemd service nvidia-device-plugin is not active", "expected UnhealthyNvidiaDevicePlugin message to indicate unhealthy status")

	// Restart Nvidia device plugin systemd service
	command = []string{
		"set -ex",
		"sudo systemctl restart nvidia-device-plugin.service || true",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to restart Nvidia device plugin service")
}

func ValidateNPDUnhealthyNvidiaDCGMServices(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Check NPD unhealthy Nvidia DCGM services config exists
		"test -f /etc/node-problem-detector.d/custom-plugin-monitor/gpu_checks/custom-plugin-nvidia-dcgm-services.json",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "NPD Nvidia DCGM services configuration does not exist")
}

func ValidateNPDUnhealthyNvidiaDCGMServicesCondition(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Validate that NPD is reporting healthy Nvidia DCGM services
	validateNPDCondition(ctx, s, "UnhealthyNvidiaDCGMServices", "HealthyNvidiaDCGMServices", corev1.ConditionFalse,
		"NVIDIA DCGM services are running properly", "expected UnhealthyNvidiaDCGMServices message to indicate healthy status")
}

func ValidateNPDUnhealthyNvidiaDCGMServicesAfterFailure(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Stop nvidia-dcgm systemd service to simulate failure
	command := []string{
		"set -ex",
		"sudo systemctl stop nvidia-dcgm.service",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to stop Nvidia DCGM service")

	// Validate that NPD reports unhealthy Nvidia DCGM services
	validateNPDCondition(ctx, s, "UnhealthyNvidiaDCGMServices", "UnhealthyNvidiaDCGMServices", corev1.ConditionTrue,
		"Systemd service(s) nvidia-dcgm are not active", "expected UnhealthyNvidiaDCGMServices message to indicate unhealthy status")

	// Stop the nvidia-dcgm-exporter system service to simulate failure
	command = []string{
		"set -ex",
		"sudo systemctl stop nvidia-dcgm-exporter.service",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to stop Nvidia DCGM Exporter service")

	// Validate that NPD still reports unhealthy Nvidia DCGM services
	validateNPDCondition(ctx, s, "UnhealthyNvidiaDCGMServices", "UnhealthyNvidiaDCGMServices", corev1.ConditionTrue,
		"Systemd service(s) nvidia-dcgm nvidia-dcgm-exporter are not active", "expected UnhealthyNvidiaDCGMServices message to indicate unhealthy status for both services")

	// Restart Nvidia DCGM services
	command = []string{
		"set -ex",
		"sudo systemctl restart nvidia-dcgm.service || true",
		"sudo systemctl restart nvidia-dcgm-exporter.service || true",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to restart Nvidia DCGM services")
}

func ValidateNPDHealthyNvidiaGridLicenseStatus(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Check NPD unhealthy Nvidia GRID license check config exists
		"test -f /etc/node-problem-detector.d/custom-plugin-monitor/gpu_checks/custom-plugin-nvidia-grid-status.json",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "NPD Nvidia Grid License check configuration does not exist")
	// Validate that NPD is reporting healthy Nvidia GRID license status
	validateNPDCondition(ctx, s, "NVIDIAGRIDStatusInvalid", "NVIDIAGRIDStatusValid", corev1.ConditionFalse,
		"NVIDIA Grid Status Valid", "expected NVIDIAGRIDStatusValid message to indicate healthy status")
}

func ValidateNPDUnhealthyNvidiaGridLicenseStatusAfterFailure(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// Stop nvidia-gridd systemd service to simulate failure
	command := []string{
		"set -ex",
		"sudo systemctl stop nvidia-gridd.service",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to stop Nvidia GRID service")

	// Validate that NPD reports unhealthy Nvidia GRID services
	validateNPDCondition(ctx, s, "NVIDIA GRID Status Invalid", "NVIDIA GRID Status Valid", corev1.ConditionTrue,
		"nvidia-gridd is not active", "expected UnhealthyNVIDIA GRID Status message to indicate unhealthy status")

	// Restart Nvidia Grid services
	command = []string{
		"set -ex",
		"sudo systemctl restart nvidia-gridd.service || true",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to restart Nvidia GRID services")
}

func ValidateRuncVersion(ctx context.Context, s *Scenario, versions []string) {
	s.T.Helper()
	require.Lenf(s.T, versions, 1, "Expected exactly one version for moby-runc but got %d", len(versions))
	// check if versions[0] is great than or equal to 1.2.0
	// check semantic version
	parsedVersion, err := semver.NewVersion(versions[0])
	require.NoError(s.T, err, "failed to parse semver from moby-runc version")
	require.GreaterOrEqual(s.T, int(parsedVersion.Major()), 1, "expected moby-runc major version to be at least 1, got %d", parsedVersion.Major())
	require.GreaterOrEqual(s.T, int(parsedVersion.Minor()), 2, "expected moby-runc minor version to be at least 2, got %d", parsedVersion.Minor())
	ValidateInstalledPackageVersion(ctx, s, "moby-runc", versions[0])
}

func ValidateWindowsProcessHasCliArguments(ctx context.Context, s *Scenario, processName string, arguments []string) {
	steps := []string{
		fmt.Sprintf("(Get-CimInstance Win32_Process -Filter \"name='%[1]s'\")[0].CommandLine", processName),
	}

	podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong")

	actualArgs := strings.Split(podExecResult.stdout, " ")

	for i := range arguments {
		expectedArgument := arguments[i]
		require.Contains(s.T, actualArgs, expectedArgument)
	}
}

func ValidateWindowsProcessContainsArgumentStrings(ctx context.Context, s *Scenario, processName string, substrings []string) {
	validateWindowsProccessArgumentString(ctx, s, processName, substrings, require.Contains)
}

func ValidateWindowsProcessDoesNotContainArgumentStrings(ctx context.Context, s *Scenario, processName string, substrings []string) {
	validateWindowsProccessArgumentString(ctx, s, processName, substrings, require.NotContains)
}

func validateWindowsProccessArgumentString(ctx context.Context, s *Scenario, processName string, substrings []string, assert func(t require.TestingT, s any, contains any, msgAndArgs ...any)) {
	steps := []string{
		fmt.Sprintf("(Get-CimInstance Win32_Process -Filter \"name='%[1]s'\")[0].CommandLine", processName),
	}
	podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command argument string - might mean file does not have params, might mean something went wrong")
	argString := podExecResult.stdout
	for _, str := range substrings {
		assert(s.T, argString, str)
	}
}

func ValidateWindowsVersionFromWindowsSettings(ctx context.Context, s *Scenario, windowsVersion string) {
	s.T.Helper()
	steps := []string{
		"(Get-ItemProperty -Path \"HKLM:\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\" -Name BuildLabEx).BuildLabEx",
	}

	jsonBytes := getWindowsSettingsJson()
	osVersion := gjson.GetBytes(jsonBytes, fmt.Sprintf("WindowsBaseVersions.%s.base_image_version", windowsVersion))
	versionSliced := strings.Split(osVersion.String(), ".")
	osMajorVersion := versionSliced[0]

	podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong")
	podExecResultStdout := strings.TrimSpace(podExecResult.stdout)

	s.T.Logf("Found windows version in windows_settings: \"%s\": \"%s\" (\"%s\")", windowsVersion, osMajorVersion, osVersion)
	s.T.Logf("Windows version returned from VM \"%s\"", podExecResultStdout)

	require.Contains(s.T, podExecResultStdout, osMajorVersion)
}

func ValidateWindowsProductName(ctx context.Context, s *Scenario, productName string) {
	s.T.Helper()
	steps := []string{
		"(Get-ItemProperty \"HKLM:\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\").ProductName",
	}

	podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong")
	podExecResultStdout := strings.TrimSpace(podExecResult.stdout)

	require.Contains(s.T, podExecResultStdout, productName)
}

func ValidateWindowsDisplayVersion(ctx context.Context, s *Scenario, displayVersion string) {
	s.T.Helper()
	steps := []string{
		"(Get-ItemProperty \"HKLM:\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\").DisplayVersion",
	}

	podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong")
	podExecResultStdout := strings.TrimSpace(podExecResult.stdout)

	s.T.Logf("Windows display version returned from VM \"%s\". Expected display version \"%s\"", podExecResultStdout, displayVersion)

	require.Contains(s.T, podExecResultStdout, displayVersion)
}

func getWindowsSettingsJson() []byte {
	jsonBytes, _ := os.ReadFile("../vhdbuilder/packer/windows/windows_settings.json")
	return jsonBytes
}

func ValidateCiliumIsRunningWindows(ctx context.Context, s *Scenario) {
	s.T.Helper()
	ValidateJsonFileHasField(ctx, s, "/k/azurecni/netconf/10-azure.conflist", "plugins.ipam.type", "azure-cns")
}

func ValidateCiliumIsNotRunningWindows(ctx context.Context, s *Scenario) {
	s.T.Helper()
	ValidateJsonFileDoesNotHaveField(ctx, s, "/k/azurecni/netconf/10-azure.conflist", "plugins.ipam.type", "azure-cns")
}

func ValidateWindowsCiliumIsRunning(ctx context.Context, s *Scenario) {
	s.T.Helper()

	expectedServices := []string{"ebpfcore", "netebpfext", "neteventebpfext", "xdp", "wtc", "hns"}
	for _, serviceName := range expectedServices {
		ValidateWindowsServiceIsRunning(ctx, s, serviceName)
	}

	expectedDlls := []string{"cncapi.dll", "wcnagent.dll"}
	for _, dllName := range expectedDlls {
		ValidateDllLoadedWindows(ctx, s, dllName)
	}
}

func ValidateWindowsCiliumIsNotRunning(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// some of the services used by windows cilium are dependencies of other services, so they may be running even if cilium is not
	// for example, ebpfcore is used by Guest Proxy Agent (GPA), so it may be running even if cilium is not
	// so, we only check that cilium-specific dlls are not loaded, as that is a stronger indication that cilium is not running
	unexpectedDlls := []string{"cncapi.dll", "wcnagent.dll"}
	for _, dllName := range unexpectedDlls {
		ValidateDllIsNotLoadedWindows(ctx, s, dllName)
	}
}

func ValidateDllLoadedWindows(ctx context.Context, s *Scenario, dllName string) {
	s.T.Helper()
	if !dllLoadedWindows(ctx, s, dllName) {
		s.T.Fatalf("expected DLL %s to be loaded, but it is not", dllName)
	}
}

func ValidateDllIsNotLoadedWindows(ctx context.Context, s *Scenario, dllName string) {
	s.T.Helper()
	if dllLoadedWindows(ctx, s, dllName) {
		s.T.Fatalf("expected DLL %s to not be loaded, but it is", dllName)
	}
}

func ValidateJsonFileHasField(ctx context.Context, s *Scenario, fileName string, jsonPath string, expectedValue string) {
	s.T.Helper()
	require.Equal(s.T, GetFieldFromJsonObjectOnNode(ctx, s, fileName, jsonPath), expectedValue)
}

func ValidateJsonFileDoesNotHaveField(ctx context.Context, s *Scenario, fileName string, jsonPath string, valueNotToBe string) {
	s.T.Helper()
	require.NotEqual(s.T, GetFieldFromJsonObjectOnNode(ctx, s, fileName, jsonPath), valueNotToBe)
}

func GetFieldFromJsonObjectOnNode(ctx context.Context, s *Scenario, fileName string, jsonPath string) string {
	steps := []string{
		fmt.Sprintf("Get-Content %[1]s", fileName),
		fmt.Sprintf("$content.%s", jsonPath),
	}

	podExecResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate command has parameters - might mean file does not have params, might mean something went wrong")

	return podExecResult.stdout
}

// ValidateTaints checks if the node has the expected taints that are set in the kubelet config with --register-with-taints flag
func ValidateTaints(ctx context.Context, s *Scenario, expectedTaints string) {
	s.T.Helper()
	node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{})
	require.NoError(s.T, err, "failed to get node %q", s.Runtime.VM.KubeName)
	var taints []string
	for _, taint := range node.Spec.Taints {
		if strings.Contains(taint.Key, "node.kubernetes.io") {
			continue
		}
		taints = append(taints, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
	}
	actualTaints := strings.Join(taints, ",")
	require.Equal(s.T, expectedTaints, actualTaints, "expected node %q to have taint %q, but got %q", s.Runtime.VM.KubeName, expectedTaints, actualTaints)
}

// ValidateLocalDNSService checks if the localdns service is in the expected state (enabled or disabled).
func ValidateLocalDNSService(ctx context.Context, s *Scenario, state string) {
	s.T.Helper()
	serviceName := "localdns"

	var script string
	switch state {
	case "enabled":
		script = fmt.Sprintf(`set -euo pipefail
svc=%q
systemctl status -n 5 "$svc" || true
active=$(systemctl is-active "$svc" 2>/dev/null || true)
enabled=$(systemctl is-enabled "$svc" 2>/dev/null || true)
echo "localdns: active=$active enabled=$enabled"
test "$active" = "active"   || { echo "expected active, got $active"; exit 1; }
test "$enabled" = "enabled" || { echo "expected enabled, got $enabled"; exit 1; }
`, serviceName)

		execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0, "localdns should be running and enabled")

	case "disabled":
		script = fmt.Sprintf(`set -euo pipefail
svc=%q
systemctl status -n 5 "$svc" || true
active=$(systemctl is-active "$svc" 2>/dev/null || true)
enabled=$(systemctl is-enabled "$svc" 2>/dev/null || true)
echo "localdns: active=$active enabled=$enabled"
test "$active" = "inactive" || { echo "expected inactive, got $active"; exit 1; }
test "$enabled" = "disabled" || { echo "expected disabled, got $enabled"; exit 1; }
`, serviceName)

		execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0, "localdns should be stopped and disabled")

	default:
		s.T.Fatalf("unknown state %q; expected 'enable' or 'disable'", state)
	}
}

// ValidateLocalDNSResolution checks if the DNS resolution for an external domain is successful from localdns clusterlistenerIP.
// It uses the 'dig' command to check the DNS resolution and expects a successful response.
func ValidateLocalDNSResolution(ctx context.Context, s *Scenario, server string) {
	s.T.Helper()
	testdomain := "bing.com"
	command := fmt.Sprintf("dig %s +timeout=1 +tries=1", testdomain)
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, "dns resolution failed")
	assert.Contains(s.T, execResult.stdout, "status: NOERROR")
	assert.Contains(s.T, execResult.stdout, fmt.Sprintf("SERVER: %s", server))
}

// ValidateLocalDNSHostsFile checks that /etc/localdns/hosts contains at least one IPv4 entry for each critical FQDN.
// This validation approach avoids flakiness with CDN/frontdoor-backed FQDNs (like mcr.microsoft.com) whose A records
// can rotate between queries. We verify presence, not exact IP matching.
// The hosts file is populated asynchronously by the aks-localdns-hosts-setup timer/service, so we poll with a timeout.
func ValidateLocalDNSHostsFile(ctx context.Context, s *Scenario, fqdns []string) {
	s.T.Helper()

	// Build script that polls until all FQDNs have at least one IPv4 entry in hosts file
	script := fmt.Sprintf(`set -euo pipefail
hosts_file="/etc/localdns/hosts"
fqdns=(%s)
timeout_secs=60
poll_interval_secs=5
deadline=$((SECONDS + timeout_secs))

while true; do
    echo "Current hosts file contents:"
    if [ -f "$hosts_file" ]; then
        cat "$hosts_file"
    else
        echo "Hosts file not found yet: $hosts_file"
    fi
    echo ""

    errors=0
    for fqdn in "${fqdns[@]}"; do
        echo "Checking FQDN: $fqdn"
        # Escape dots in FQDN for use in regex (. is a regex wildcard)
        fqdn_escaped=$(echo "$fqdn" | sed 's/\./\\./g')

        # Validate that there is at least one IPv4 entry for this FQDN in the hosts file,
        # rather than requiring every currently resolved IP to be present. This avoids
        # flakiness for CDN/frontdoor-backed FQDNs whose A records can rotate.
        if [ -f "$hosts_file" ] && grep -Eq '^[0-9]{1,3}(\.[0-9]{1,3}){3}[[:space:]]+'"$fqdn_escaped"'([[:space:]]|$)' "$hosts_file"; then
            echo "  OK: Found at least one IPv4 entry for $fqdn in hosts file"
        else
            echo "  WAIT: No IPv4 entry found for $fqdn in hosts file yet"
            errors=$((errors + 1))
        fi
    done

    echo ""
    if [ "$errors" -eq 0 ]; then
        echo "SUCCESS: All critical FQDNs have at least one IPv4 entry in hosts file"
        exit 0
    fi

    if [ "$SECONDS" -ge "$deadline" ]; then
        echo "FAILED: Timed out after ${timeout_secs}s waiting for $errors FQDN(s) to appear in hosts file"
        echo ""
        echo "Final hosts file contents:"
        if [ -f "$hosts_file" ]; then
            cat "$hosts_file"
        else
            echo "Hosts file not found: $hosts_file"
        fi
        echo ""
        echo "Recent journal output for aks-localdns-hosts-setup.service and timer:"
        journalctl --no-pager -u aks-localdns-hosts-setup.service -u aks-localdns-hosts-setup.timer -n 50 || true
        exit 1
    fi

    echo "Retrying in ${poll_interval_secs}s..."
    sleep "$poll_interval_secs"
done
`, quoteFQDNsForBash(fqdns))

	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
		"hosts file should contain resolved IPs for critical FQDNs")
}

// quoteFQDNsForBash converts a slice of FQDNs to a bash array string
func quoteFQDNsForBash(fqdns []string) string {
	return strings.Join(lo.Map(fqdns, func(fqdn string, _ int) string {
		return fmt.Sprintf("%q", fqdn)
	}), " ")
}

// ValidateAKSLocalDNSHostsSetupService checks that aks-localdns-hosts-setup.service ran successfully
// and the aks-localdns-hosts-setup.timer is active to ensure periodic refresh of /etc/localdns/hosts.
func ValidateAKSLocalDNSHostsSetupService(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Check that aks-localdns-hosts-setup.service (oneshot) completed without failure
	ValidateSystemdUnitIsNotFailed(ctx, s, "aks-localdns-hosts-setup.service")

	// Check that aks-localdns-hosts-setup.timer is active for periodic refresh
	ValidateSystemdUnitIsRunning(ctx, s, "aks-localdns-hosts-setup.timer")
}

// ValidateLocalDNSHostsPluginBypass verifies that localdns serves FQDNs from /etc/localdns/hosts
// via the CoreDNS hosts plugin. It checks:
//  1. The node has the kubernetes.azure.com/localdns-hosts-plugin=enabled annotation
//  2. The Corefile has the hosts plugin configured in both VnetDNS and KubeDNS listeners
//  3. The IPs returned by dig match the entries in /etc/localdns/hosts for the same FQDN
//
// We intentionally do NOT assert on DNS flags (AA, RA) because CoreDNS can set these
// regardless of which plugin served the response.
func ValidateLocalDNSHostsPluginBypass(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Step 1: Verify the node has the hosts plugin annotation
	// The annotation is set asynchronously by localdns.sh (background job waiting for kubeconfig + node registration)
	// Poll for up to 5 minutes with exponential backoff to avoid flaky failures
	s.T.Log("Polling for node annotation kubernetes.azure.com/localdns-hosts-plugin=enabled...")
	annotationKey := "kubernetes.azure.com/localdns-hosts-plugin"

	var node *corev1.Node
	var err error
	var annotationValue string
	var exists bool
	maxAttempts := 33 // ~5 minutes: first 4 attempts use 1+2+4+8=15s, then ~29 attempts at 10s cap = ~305s

	for attempt := 1; attempt <= maxAttempts; attempt++ {
		node, err = s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{})
		require.NoError(s.T, err, "failed to get node %q", s.Runtime.VM.KubeName)

		annotationValue, exists = node.Annotations[annotationKey]
		if exists && annotationValue == "enabled" {
			s.T.Logf("✓ Node annotation %s=%s found after %d attempts", annotationKey, annotationValue, attempt)
			break
		}

		if attempt == maxAttempts {
			s.T.Logf("WARNING: node %q annotation %q not found or not 'enabled' after %d attempts (~5 minutes). Current value: exists=%v, value=%q. Annotation is best-effort in production, continuing with stronger validators.",
				s.Runtime.VM.KubeName, annotationKey, maxAttempts, exists, annotationValue)
		}

		// Exponential backoff: 1s, 2s, 4s, 8s, max 10s
		sleepDuration := time.Duration(1<<uint(attempt-1)) * time.Second
		if sleepDuration > 10*time.Second {
			sleepDuration = 10 * time.Second
		}
		s.T.Logf("Attempt %d/%d: annotation not ready (exists=%v, value=%q), retrying in %v...", attempt, maxAttempts, exists, annotationValue, sleepDuration)
		time.Sleep(sleepDuration)
	}

	// Step 2: Verify the Corefile has the hosts plugin configured
	s.T.Log("Verifying Corefile contains hosts plugin configuration...")
	corefileCheckScript := `set -euo pipefail
corefile="/opt/azure/containers/localdns/updated.localdns.corefile"

echo "=== Verifying Corefile configuration ==="
echo "Checking if $corefile exists..."
if [ ! -f "$corefile" ]; then
    echo "ERROR: Corefile $corefile does not exist"
    exit 1
fi
echo "✓ Corefile exists"
echo ""

echo "Checking if Corefile contains hosts plugin directive..."
if ! grep -q "hosts /etc/localdns/hosts" "$corefile"; then
    echo "ERROR: Corefile does not contain 'hosts /etc/localdns/hosts' directive"
    echo ""
    echo "Corefile contents:"
    cat "$corefile"
    exit 1
fi
echo "✓ Found 'hosts /etc/localdns/hosts' directive in Corefile"
echo ""

echo "Checking hosts plugin has fallthrough directive..."
if ! grep -A5 "hosts /etc/localdns/hosts" "$corefile" | grep -q "fallthrough"; then
    echo "WARNING: hosts plugin may be missing 'fallthrough' directive"
fi
echo "✓ hosts plugin configuration looks correct"
echo ""

echo "Corefile contents:"
cat "$corefile"
echo ""
echo "=== Corefile validation successful ==="
`

	execScriptOnVMForScenarioValidateExitCode(ctx, s, corefileCheckScript, 0,
		"Corefile should contain hosts plugin configuration")

	// Step 3: Test that localdns resolves real FQDNs from /etc/localdns/hosts
	// This validates the hosts plugin is working by checking that the IPs returned by dig
	// match the entries in /etc/localdns/hosts for the same FQDN. If the response came from
	// upstream DNS instead of the hosts plugin, the IPs would likely differ due to CDN rotation,
	// load balancer failover, or regional DNS updates.
	//
	// We intentionally do NOT assert on DNS flags (AA, RA) because CoreDNS can set these
	// regardless of which plugin served the response — they reflect server capabilities,
	// not response source. The IP match combined with corefile verification (step 2) and
	// node annotation (step 1) is sufficient proof.
	//
	// We use the first FQDN from GetDefaultFQDNsForValidation() (e.g. mcr.microsoft.com) because
	// it's a real FQDN that aks-localdns-hosts-setup.service populates from the NBC's CriticalFQDNs list.
	// This avoids race conditions with the aks-localdns-hosts-setup.timer overwriting fake test entries.
	testFQDN := s.GetDefaultFQDNsForValidation()[0]
	s.T.Logf("Testing hosts plugin resolves %s from /etc/localdns/hosts with matching IPs", testFQDN)

	script := fmt.Sprintf(`set -euo pipefail
test_fqdn=%q
hosts_file="/etc/localdns/hosts"

echo "=== Testing localdns hosts plugin functionality ==="
echo "Testing FQDN: $test_fqdn"
echo ""

# Step 1: Get the expected IPs from /etc/localdns/hosts
echo "Reading expected IPs from $hosts_file..."
if [ ! -f "$hosts_file" ]; then
    echo "ERROR: Hosts file $hosts_file does not exist"
    exit 1
fi

# Extract IPv4 addresses for the test FQDN from hosts file
# Escape dots in FQDN for regex (. matches any char in ERE)
test_fqdn_escaped=$(echo "$test_fqdn" | sed 's/\./\\./g')
expected_ips=$(grep -E "^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+[[:space:]]+${test_fqdn_escaped}([[:space:]]|$)" "$hosts_file" | awk '{print $1}' | sort)
if [ -z "$expected_ips" ]; then
    echo "ERROR: No IPv4 entries found for $test_fqdn in $hosts_file"
    echo "Hosts file contents:"
    cat "$hosts_file"
    exit 1
fi

echo "Expected IPs from hosts file:"
echo "$expected_ips"
echo ""

# Step 2: Query localdns with dig and capture full output
echo "Querying localdns for $test_fqdn at 169.254.10.10..."
dig_output=$(dig "$test_fqdn" @169.254.10.10 -t A +timeout=5 +tries=2 2>&1)
echo "Full dig output:"
echo "$dig_output"
echo ""

# Step 3: Extract resolved IPs from dig ANSWER section and compare with hosts file
# Reuse escaped FQDN for regex matching of dig output
resolved_ips=$(echo "$dig_output" | grep -E "^${test_fqdn_escaped}\..*IN[[:space:]]+A[[:space:]]" | awk '{print $NF}' | sort)
if [ -z "$resolved_ips" ]; then
    echo "ERROR: No A records returned from dig query"
    exit 1
fi

echo "Resolved IPs from dig:"
echo "$resolved_ips"
echo ""

echo "Comparing resolved IPs with hosts file entries..."
if [ "$expected_ips" != "$resolved_ips" ]; then
    echo "ERROR: Resolved IPs do not match hosts file entries"
    echo "Expected (from hosts file):"
    echo "$expected_ips"
    echo "Got (from dig):"
    echo "$resolved_ips"
    exit 1
fi
echo "✓ Resolved IPs match hosts file entries"
echo ""

echo "=== SUCCESS ==="
echo "The localdns hosts plugin is working correctly:"
echo "  Resolved IPs match /etc/localdns/hosts entries"
`, testFQDN)

	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
		"localdns should resolve FQDN from hosts file with matching IPs")
}

// ValidateLocalDNSHostsPluginIPv6 checks that IPv6 entries in /etc/localdns/hosts are
// properly served by CoreDNS's hosts plugin. If the hosts file has no IPv6 entries
// (some FQDNs don't have AAAA records), the test is skipped gracefully.
//
// Test flow:
//  1. Find the first FQDN with an IPv6 entry in the hosts file
//  2. Query localdns for AAAA records for that FQDN
//  3. Verify the returned IPv6 addresses match the hosts file entries
func ValidateLocalDNSHostsPluginIPv6(ctx context.Context, s *Scenario) {
	s.T.Helper()

	s.T.Log("Testing hosts plugin serves IPv6 entries from hosts file")

	script := `set -euo pipefail
hosts_file="/etc/localdns/hosts"

echo "=== Validating IPv6 handling in hosts plugin ==="
echo ""

if [ ! -f "$hosts_file" ]; then
    echo "ERROR: Hosts file $hosts_file does not exist"
    exit 1
fi

# Find the first FQDN that has an IPv6 entry in the hosts file
# IPv6 addresses contain colons, so we look for lines starting with hex:hex patterns
ipv6_line=$(grep -E '^[0-9a-fA-F]*:[0-9a-fA-F:]+[[:space:]]' "$hosts_file" | head -1 || true)

if [ -z "$ipv6_line" ]; then
    echo "No IPv6 entries found in hosts file — skipping IPv6 validation"
    echo "This is expected if none of the critical FQDNs have AAAA records"
    exit 0
fi

# Extract the FQDN and expected IPv6 address from the hosts file line
ipv6_fqdn=$(echo "$ipv6_line" | awk '{print $2}')
echo "Found IPv6 entry for FQDN: $ipv6_fqdn"
echo ""

# Get all IPv6 addresses for this FQDN from the hosts file
expected_ipv6=$(grep -E '^[0-9a-fA-F]*:[0-9a-fA-F:]+[[:space:]]' "$hosts_file" | \
    awk -v fqdn="$ipv6_fqdn" '$2 == fqdn {print $1}' | sort)
echo "Expected IPv6 addresses from hosts file:"
echo "$expected_ipv6"
echo ""

# Query localdns for AAAA records
echo "Querying localdns for AAAA records..."
dig_result=$(dig "$ipv6_fqdn" @169.254.10.10 -t AAAA +short +timeout=5 +tries=2 2>&1 || true)
echo "Dig AAAA result: '$dig_result'"
echo ""

# Filter to only valid IPv6 addresses (contain colons)
resolved_ipv6=$(echo "$dig_result" | grep ':' | sort)

if [ -z "$resolved_ipv6" ]; then
    echo "ERROR: No AAAA records returned by localdns for $ipv6_fqdn"
    echo "Expected at least one IPv6 address from hosts file"
    exit 1
fi

echo "Resolved IPv6 addresses:"
echo "$resolved_ipv6"
echo ""

# Verify at least one expected IPv6 address is in the dig result
match_found=false
for expected in $expected_ipv6; do
    if echo "$resolved_ipv6" | grep -qi "$expected"; then
        echo "✓ IPv6 address $expected matches"
        match_found=true
    fi
done

if [ "$match_found" != "true" ]; then
    echo "ERROR: None of the expected IPv6 addresses from hosts file were returned by localdns"
    echo "Expected (from hosts file): $expected_ipv6"
    echo "Got (from dig): $resolved_ipv6"
    exit 1
fi

echo ""
echo "=== SUCCESS ==="
echo "IPv6 entries in hosts file are correctly served by CoreDNS hosts plugin"
`

	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
		"CoreDNS hosts plugin should serve IPv6 entries from hosts file")
}

// ValidateLocalDNSHostsPluginColdStart verifies that localdns works correctly when started
// with an empty hosts file — the exact scenario that occurs when localdns starts before
// aks-localdns-hosts-setup finishes resolving FQDNs.
//
// Test flow:
//  1. Truncate hosts file, stop/start localdns — CoreDNS starts fresh with empty hosts file and empty cache
//  2. Verify critical and non-critical FQDNs resolve via fallthrough (upstream DNS)
//  3. Populate hosts file with a canary entry (simulates aks-localdns-hosts-setup completing)
//  4. Wait for CoreDNS reload (5s), verify canary resolves (hosts plugin picks up new file)
//  5. Restore original hosts file and stop/start localdns to leave node in clean state
func ValidateLocalDNSHostsPluginColdStart(ctx context.Context, s *Scenario) {
	s.T.Helper()

	s.T.Log("Testing localdns cold start with empty hosts file then population")

	script := `#!/bin/bash
set -euo pipefail
hosts_file="/etc/localdns/hosts"
canary_fqdn="canary.localdns.test"
canary_ip="192.0.2.99"
localdns_ip="169.254.10.10"
resolv_conf="/run/systemd/resolve/resolv.conf"

# Helper: validate that dig output contains at least one valid IP address (not error text).
has_valid_ip() {
    echo "$1" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'
}

# Helper: dump localdns state for debugging.
dump_localdns_diagnostics() {
    echo "--- localdns service status ---"
    sudo systemctl status localdns --no-pager 2>&1 || true
    echo "--- localdns journal (last 50 lines) ---"
    sudo journalctl -u localdns --no-pager -n 50 2>&1 || true
}

# Helper: wait for localdns teardown side effects to settle before starting again.
# localdns stop triggers cleanup that removes the DNS drop-in, reloads network config,
# tears down the dummy interface, and exits CoreDNS. Some of that state converges
# asynchronously outside the systemd unit, so we explicitly wait for it here instead of
# relying on a one-shot systemctl restart.
wait_for_localdns_shutdown_stability() {
    local reason="$1"
    local max_wait_seconds=30
    local attempt=1
    local current_dns systemd_state interface_present

    echo "Waiting for localdns teardown to settle (${reason})..."
    while [ "$attempt" -le "$max_wait_seconds" ]; do
        current_dns=$(awk '/^nameserver/ {print $2}' "$resolv_conf" 2>/dev/null | paste -sd' ')
        systemd_state=$(sudo systemctl is-active localdns 2>&1 || true)
        interface_present="false"
        if ip link show dev localdns >/dev/null 2>&1; then
            interface_present="true"
        fi

        if [ "$systemd_state" != "active" ] && [ "$systemd_state" != "activating" ] && [ "$systemd_state" != "deactivating" ] && \
            ! grep -qwF "$localdns_ip" <<< "$current_dns" && [ "$interface_present" = "false" ]; then
            echo "localdns teardown settled after ${attempt}s (${reason}); systemd=${systemd_state}, DNS=${current_dns:-<none>}"
            return 0
        fi

        sleep 1
        attempt=$((attempt + 1))
    done

    echo "ERROR: localdns teardown did not settle after ${max_wait_seconds}s (${reason})" >&2
    echo "systemctl is-active localdns: $(sudo systemctl is-active localdns 2>&1 || true)"
    echo "Current DNS: $(awk '/^nameserver/ {print $2}' "$resolv_conf" 2>/dev/null | paste -sd' ')"
    if ip link show dev localdns >/dev/null 2>&1; then
        echo "localdns interface is still present"
    else
        echo "localdns interface has been removed"
    fi
    return 1
}

# Helper: cold-start localdns by splitting restart into stop -> wait for stable teardown -> start.
# This targets the actual flaky edge in the E2E: external DNS/network state settling after stop.
restart_localdns_cleanly() {
    local reason="$1"
    local rc=0

    echo "Stopping localdns (${reason})..."
    if sudo systemctl stop localdns; then
        :
    else
        rc=$?
        echo "ERROR: localdns stop failed (${reason}, rc=$rc)" >&2
        dump_localdns_diagnostics
        return "$rc"
    fi

    if ! wait_for_localdns_shutdown_stability "$reason"; then
        dump_localdns_diagnostics
        return 1
    fi

    sudo systemctl reset-failed localdns || true

    echo "Starting localdns (${reason})..."
    if sudo systemctl start localdns; then
        :
    else
        rc=$?
        echo "ERROR: localdns start failed (${reason}, rc=$rc)" >&2
        dump_localdns_diagnostics
        return "$rc"
    fi

    echo "localdns start completed (${reason})"
    return 0
}

echo "=== Testing localdns cold start with empty hosts file ==="
echo ""

# Step 0: Save current hosts file
echo "Step 0: Saving current hosts file..."
if [ ! -f "$hosts_file" ]; then
    echo "ERROR: Hosts file $hosts_file does not exist"
    exit 1
fi
saved_content=$(cat "$hosts_file")
echo "Saved hosts file ($(echo "$saved_content" | wc -l) lines)"
echo ""

# Step 1: Truncate hosts file and cold-start localdns (clears both hosts map and DNS cache)
echo "Step 1: Truncating hosts file and cold-starting localdns..."
sudo truncate -s 0 "$hosts_file"
restart_localdns_cleanly "start with empty hosts file"
echo "localdns started with empty hosts file"
echo ""

# Wait for localdns to be fully ready
echo "Waiting for localdns to be ready..."
for i in $(seq 1 30); do
    if dig "health-check.localdns.local" @169.254.10.10 +short +timeout=1 +tries=1 >/dev/null 2>&1; then
        echo "localdns is ready after ${i}s"
        break
    fi
    if [ "$i" -eq 30 ]; then
        echo "ERROR: localdns did not become ready after 30s"
        dump_localdns_diagnostics
        echo "$saved_content" | sudo tee "$hosts_file" > /dev/null
        restart_localdns_cleanly "restore after readiness timeout" || true
        exit 1
    fi
    sleep 1
done
echo ""

# Step 2: Verify DNS resolves via fallthrough with empty hosts file
echo "Step 2: Verifying DNS resolves via fallthrough after cold start..."

critical_fqdn="mcr.microsoft.com"
critical_result=$(dig "$critical_fqdn" @169.254.10.10 -t A +short +timeout=5 +tries=2 2>&1 || true)
echo "Critical FQDN ($critical_fqdn): '$critical_result'"
if ! has_valid_ip "$critical_result"; then
    echo "ERROR: Critical FQDN did not return a valid IP after cold start with empty hosts file"
    dump_localdns_diagnostics
    echo "$saved_content" | sudo tee "$hosts_file" > /dev/null
    restart_localdns_cleanly "restore after critical FQDN failure" || true
    exit 1
fi
echo "✓ Critical FQDN resolves via fallthrough"

noncritical_fqdn="www.bing.com"
noncritical_result=$(dig "$noncritical_fqdn" @169.254.10.10 -t A +short +timeout=5 +tries=2 2>&1 || true)
echo "Non-critical FQDN ($noncritical_fqdn): '$noncritical_result'"
if ! has_valid_ip "$noncritical_result"; then
    echo "ERROR: Non-critical FQDN did not return a valid IP after cold start with empty hosts file"
    dump_localdns_diagnostics
    echo "$saved_content" | sudo tee "$hosts_file" > /dev/null
    restart_localdns_cleanly "restore after non-critical FQDN failure" || true
    exit 1
fi
echo "✓ Non-critical FQDN resolves via fallthrough"
echo ""

# Step 3: Populate hosts file with canary (simulates aks-localdns-hosts-setup completing)
echo "Step 3: Populating hosts file with original content + canary entry..."
{ echo "$saved_content"; echo "${canary_ip} ${canary_fqdn}"; } | sudo tee "$hosts_file" > /dev/null
echo ""

# Step 4: Wait for hosts plugin reload then verify canary resolves
# IMPORTANT: We must wait for the hosts plugin to reload (configured with "reload 5s")
# BEFORE making the first canary query. If we query too early, the hosts plugin hasn't
# picked up the new file yet and the query falls through to the forward plugin, which
# returns NXDOMAIN. The cache plugin then caches that NXDOMAIN for the SOA TTL (can be
# minutes/hours), making all subsequent queries return NXDOMAIN even after the hosts
# plugin reloads — the cached negative response shadows the hosts entry.
echo "Step 4: Waiting 15s for hosts plugin reload (5s cycle + safety margin)..."
sleep 15
echo "Polling for canary resolution (up to 60s)..."
canary_resolved=false
for i in $(seq 1 30); do
    canary_result=$(dig "$canary_fqdn" @169.254.10.10 -t A +short +timeout=2 +tries=1 2>&1 || true)
    if [ "$canary_result" = "$canary_ip" ]; then
        echo "✓ Canary resolves after 15s+${i}x2s — CoreDNS picked up the hosts file after cold start"
        canary_resolved=true
        break
    fi
    sleep 2
done

if [ "$canary_resolved" != "true" ]; then
    echo "ERROR: Canary did not resolve after 60s — hot-reload after cold start broken"
    echo "Expected: $canary_ip"
    echo "Last dig result: '$canary_result'"
    dump_localdns_diagnostics
    echo "$saved_content" | sudo tee "$hosts_file" > /dev/null
    restart_localdns_cleanly "restore after canary failure" || true
    exit 1
fi
echo ""

# Step 5: Cleanup — restore original hosts file and cold-start localdns
echo "Step 5: Cleaning up — restoring original hosts file and cold-starting localdns..."
echo "$saved_content" | sudo tee "$hosts_file" > /dev/null
restart_localdns_cleanly "final cleanup restore"
echo ""

# Wait for localdns to be ready after final cold start
for i in $(seq 1 30); do
    if dig "health-check.localdns.local" @169.254.10.10 +short +timeout=1 +tries=1 >/dev/null 2>&1; then
        echo "localdns is ready after cleanup"
        break
    fi
    if [ "$i" -eq 30 ]; then
        echo "WARNING: localdns did not become ready after cleanup (30s)"
        dump_localdns_diagnostics
    fi
    sleep 1
done

echo "=== SUCCESS ==="
echo "localdns cold start with empty hosts file verified:"
echo "  1. Start with empty hosts file: DNS resolves via fallthrough"
echo "  2. Hosts file populated later: CoreDNS picks it up via reload"
`

	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
		"localdns should work after cold start with empty hosts file and pick up populated file")
}

// ValidateJournalctlOutput checks if specific content exists in the systemd service logs
func ValidateJournalctlOutput(ctx context.Context, s *Scenario, serviceName string, expectedContent string) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Get the service logs and check for the expected content
		fmt.Sprintf("sudo journalctl -u %s | grep -q '%s'", serviceName, expectedContent),
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		fmt.Sprintf("expected content '%s' not found in %s service logs", expectedContent, serviceName))
}

func ValidateNodeProblemDetector(ctx context.Context, s *Scenario) {
	command := []string{
		"set -ex",
		// Verify node-problem-detector service is running
		"systemctl is-active node-problem-detector",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Node Problem Detector (NPD) service validation failed")
}

func RestartNodeProblemDetector(ctx context.Context, s *Scenario) {
	s.T.Helper()
	s.T.Log("restarting node-problem-detector to pick up managed GPU health checks")
	command := []string{
		"set -ex",
		"sudo systemctl restart node-problem-detector",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		"failed to restart Node Problem Detector (NPD) service")
}

func ValidateNodeExporter(ctx context.Context, s *Scenario) {
	s.T.Helper()

	skipFile := "/etc/node-exporter.d/skip_vhd_node_exporter"
	serviceName := "node-exporter.service"

	// Check if node-exporter is installed on this VHD by looking for the skip sentinel file.
	// The skip file is only present on VHDs that have node-exporter installed (Ubuntu, Mariner, Azure Linux).
	// Flatcar, ACL, OSGuard, and older VHDs do not have node-exporter installed and will not have the skip file.
	if !fileExist(ctx, s, skipFile) {
		s.T.Logf("Skipping node-exporter validation: sentinel file %s not found (VHD does not have node-exporter installed)", skipFile)
		return
	}

	s.T.Logf("skip_vhd_node_exporter sentinel file found, validating node-exporter installation")

	// Validate service is running
	ValidateSystemdUnitIsRunning(ctx, s, serviceName)
	ValidateSystemdUnitIsNotFailed(ctx, s, serviceName)

	// Validate service is enabled
	execScriptOnVMForScenarioValidateExitCode(ctx, s, fmt.Sprintf("systemctl is-enabled %s", serviceName), 0, fmt.Sprintf("%s should be enabled", serviceName))

	// Validate binary exists and is executable
	// The binary is installed at /usr/bin and symlinked to /opt/bin for consistency with other binaries (kubelet, etc.)
	ValidateFileExists(ctx, s, "/usr/bin/node-exporter")
	ValidateFileExists(ctx, s, "/opt/bin/node-exporter")
	ValidateFileExists(ctx, s, "/opt/bin/node-exporter-startup.sh")

	// Validate configuration files exist
	ValidateFileExists(ctx, s, skipFile)
	ValidateFileExists(ctx, s, "/etc/node-exporter.d/web-config.yml")

	// Validate that node-exporter is listening on port 19100 and serving metrics on the node ip.
	// TLS is disabled by default (opt-in via NODE_EXPORTER_TLS_ENABLED=true in /etc/default/node-exporter),
	// so we validate by making a plain HTTP request to the metrics endpoint.
	// We avoid curl -sf here so that diagnostic messages (e.g. "Client sent an HTTP request to an HTTPS server")
	// are visible in test logs rather than silently swallowed.
	// We curl the node's private IP directly rather than discovering the listen address from ss,
	// so we validate that the metrics endpoint is reachable on the actual node IP used by monitoring infrastructure.
	s.T.Logf("Validating node-exporter is listening on port 19100 and serving metrics")
	command := []string{
		"set -ex",
		fmt.Sprintf("echo \"node IP: %s\"", s.Runtime.VM.PrivateIP),
		fmt.Sprintf("curl -sS --max-time 10 http://%s:19100/metrics 2>&1 | head -20", s.Runtime.VM.PrivateIP),
		fmt.Sprintf("curl -sS --max-time 10 http://%s:19100/metrics 2>&1 | grep -q 'node_'", s.Runtime.VM.PrivateIP),
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "node-exporter should be listening on port 19100 and serving metrics over HTTP")

	s.T.Logf("node-exporter validation passed")
}

func ValidateNPDFilesystemCorruption(ctx context.Context, s *Scenario) {
	command := []string{
		"set -ex",
		// Check if the filesystem corruption monitor NPD plugin configuration file exists
		"test -f /etc/node-problem-detector.d/custom-plugin-monitor/custom-fs-corruption-monitor.json",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "NPD Custom Plugin configuration for FilesystemCorruptionProblem not found")

	// Log the NPD plugin config and check script for diagnostics
	diagCmd := []string{
		"set -x",
		"cat /etc/node-problem-detector.d/custom-plugin-monitor/custom-fs-corruption-monitor.json",
		"echo '--- check_fs_corruption.sh ---'",
		"cat /etc/node-problem-detector.d/plugin/check_fs_corruption.sh",
	}
	diagResult := execScriptOnVMForScenario(ctx, s, strings.Join(diagCmd, "\n"))
	s.T.Logf("NPD filesystem corruption plugin config and script:\nstdout:\n%s\nstderr:\n%s", diagResult.stdout, diagResult.stderr)

	// Simulate filesystem corruption by replacing the check script with one that
	// always reports corruption. This is the most reliable approach because:
	// - On cgroup v2 (Ubuntu 24.04), systemd protects service cgroups from external
	//   process migration, so we cannot inject journal entries under containerd.service
	// - Writing to /proc/PID/fd/2 on a journal stream socket is unreliable
	// - The test's goal is to verify NPD is installed, configured, and correctly sets
	//   node conditions — not to unit-test the journal grep mechanism
	command = []string{
		"set -ex",
		`sudo cp /etc/node-problem-detector.d/plugin/check_fs_corruption.sh /etc/node-problem-detector.d/plugin/check_fs_corruption.sh.bak`,
		`printf '#!/bin/bash\necho "Found '\''structure needs cleaning'\'' in containerd journal."\nexit 1\n' | sudo tee /etc/node-problem-detector.d/plugin/check_fs_corruption.sh > /dev/null`,
		`sudo chmod +x /etc/node-problem-detector.d/plugin/check_fs_corruption.sh`,
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Failed to replace check_fs_corruption.sh to simulate corruption")
	defer func() {
		restoreCmd := []string{
			"set -ex",
			`if [ -f /etc/node-problem-detector.d/plugin/check_fs_corruption.sh.bak ]; then`,
			`  sudo mv /etc/node-problem-detector.d/plugin/check_fs_corruption.sh.bak /etc/node-problem-detector.d/plugin/check_fs_corruption.sh`,
			`  sudo chmod +x /etc/node-problem-detector.d/plugin/check_fs_corruption.sh`,
			`fi`,
		}
		restoreResult := execScriptOnVMForScenario(ctx, s, strings.Join(restoreCmd, "\n"))
		s.T.Logf("Restored original check_fs_corruption.sh:\nstdout:\n%s\nstderr:\n%s", restoreResult.stdout, restoreResult.stderr)
	}()

	// Verify the replacement script works correctly
	verifyCmd := []string{
		"set -x",
		"cat /etc/node-problem-detector.d/plugin/check_fs_corruption.sh",
		"echo '--- manual check script run ---'",
		"sudo /etc/node-problem-detector.d/plugin/check_fs_corruption.sh; echo \"exit_code=$?\"",
	}
	verifyResult := execScriptOnVMForScenario(ctx, s, strings.Join(verifyCmd, "\n"))
	s.T.Logf("Simulation verification:\nstdout:\n%s\nstderr:\n%s", verifyResult.stdout, verifyResult.stderr)

	// Wait for NPD to detect the problem. NPD's custom plugin monitor polls
	// every 5 minutes. With continuous simulation, the first check cycle after
	// our start should detect it. Use 8 minutes as a safety margin.
	var filesystemCorruptionProblem *corev1.NodeCondition
	err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 8*time.Minute, true, func(ctx context.Context) (bool, error) {
		node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{})
		if err != nil {
			s.T.Logf("Failed to get node %q: %v", s.Runtime.VM.KubeName, err)
			return false, nil // Continue polling on transient errors
		}

		for i := range node.Status.Conditions {
			if node.Status.Conditions[i].Type == "FilesystemCorruptionProblem" && node.Status.Conditions[i].Reason == "FilesystemCorruptionDetected" {
				filesystemCorruptionProblem = &node.Status.Conditions[i]
				return true, nil
			}
		}
		return false, nil // Continue polling
	})
	require.NoError(s.T, err, "timed out waiting for FilesystemCorruptionProblem condition to appear on node %q", s.Runtime.VM.KubeName)

	require.NotNil(s.T, filesystemCorruptionProblem, "expected FilesystemCorruptionProblem condition to be present on node")
	require.Equal(s.T, corev1.ConditionTrue, filesystemCorruptionProblem.Status, "expected FilesystemCorruptionProblem condition to be True on node")
	require.Contains(s.T, filesystemCorruptionProblem.Message, "Found 'structure needs cleaning' in containerd journal.", "expected FilesystemCorruptionProblem condition message to contain: Found 'structure needs cleaning' in containerd journal.")
}

func ValidateEnableNvidiaResource(ctx context.Context, s *Scenario) {
	s.T.Logf("waiting for Nvidia GPU resource to be available")
	waitUntilResourceAvailable(ctx, s, "nvidia.com/gpu")
}

func ValidateNvidiaDevicePluginServiceRunning(ctx context.Context, s *Scenario) {
	s.T.Helper()
	s.T.Logf("validating that NVIDIA device plugin systemd service is running")

	command := []string{
		"set -ex",
		"systemctl is-active nvidia-device-plugin.service",
		"systemctl is-enabled nvidia-device-plugin.service",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "NVIDIA device plugin systemd service should be active and enabled")
}

func ValidateNodeAdvertisesGPUResources(ctx context.Context, s *Scenario, gpuCountExpected int64, resourceName string) {
	s.T.Helper()
	s.T.Logf("validating that node advertises GPU resources")

	// First, wait for the GPU resource to be available
	waitUntilResourceAvailable(ctx, s, resourceName)

	// Get the node using the Kubernetes client from the test framework
	nodeName := s.Runtime.VM.KubeName
	node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
	require.NoError(s.T, err, "failed to get node %q", nodeName)

	// Check if the node advertises GPU capacity
	gpuCapacity, exists := node.Status.Capacity[corev1.ResourceName(resourceName)]
	require.True(s.T, exists, "node should advertise resource %s", resourceName)

	gpuCount := gpuCapacity.Value()
	require.Equal(s.T, gpuCount, gpuCountExpected, "node should advertise %s=%d, but got %s=%d", resourceName, gpuCountExpected, resourceName, gpuCount)
	s.T.Logf("node %s advertises %s=%d resources", nodeName, resourceName, gpuCount)
}

func ValidateGPUWorkloadSchedulable(ctx context.Context, s *Scenario, gpuCount int, resourceName string) {
	s.T.Helper()
	s.T.Logf("validating that GPU workloads can be scheduled")

	// Wait for resources to be available and add delay for device health
	waitUntilResourceAvailable(ctx, s, resourceName)
	time.Sleep(20 * time.Second) // Same delay as existing GPU tests

	// Create a GPU test pod using the same pattern as podRunNvidiaWorkload
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      fmt.Sprintf("%s-gpu-test", s.Runtime.VM.KubeName),
			Namespace: "default",
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{
				{
					Name:  "gpu-test-container",
					Image: "mcr.microsoft.com/azuredocs/samples-tf-mnist-demo:gpu",
					Args: []string{
						"--max-steps", "1",
					},
					Resources: corev1.ResourceRequirements{
						Limits: corev1.ResourceList{
							corev1.ResourceName(resourceName): resource.MustParse(fmt.Sprintf("%d", gpuCount)),
						},
					},
				},
			},
			NodeSelector: map[string]string{
				"kubernetes.io/hostname": s.Runtime.VM.KubeName,
			},
		},
	}

	ValidatePodRunning(ctx, s, pod)

	s.T.Logf("GPU workload is schedulable and runs successfully")
}

// ValidatePubkeySSHDisabled validates that SSH with private key authentication is disabled by checking sshd_config
func ValidatePubkeySSHDisabled(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Part 1. Use VMSS RunCommand to check sshd_config directly on the node
	resp, err := RunCommand(ctx, s, `#!/bin/bash
# Check if PubkeyAuthentication is disabled in sshd_config
if grep -q "^PubkeyAuthentication no" /etc/ssh/sshd_config; then
    echo "SUCCESS: PubkeyAuthentication is disabled"
    exit 0
else
    echo "FAILED: PubkeyAuthentication is not properly disabled"
    echo "Current sshd_config content related to PubkeyAuthentication:"
    grep -i "PubkeyAuthentication" /etc/ssh/sshd_config || echo "No PubkeyAuthentication setting found"
    exit 1
fi`)
	require.NoError(s.T, err, "Failed to run command to check sshd_config")
	stdout := lo.FromPtr(resp.Output)
	s.T.Logf("Run command stdout: %s\nstderr: %s", stdout, lo.FromPtr(resp.Error))

	// Check if the command execution was successful by looking for our success message in the output
	if !strings.Contains(stdout, "SUCCESS: PubkeyAuthentication is disabled") {
		s.T.Fatalf("PubkeyAuthentication is not properly disabled. stdout: %s", stdout)
	}

	// Part 2. Check cannot SSH with private key (expect failure)
	err = validateSSHConnectivity(ctx, s)
	require.Error(s.T, err, "Expected SSH connection with private key to fail, but it succeeded")
	if !strings.Contains(err.Error(), "Permission denied") {
		s.T.Fatalf("Expected permission denied error, but got: %v", err)
	}

	s.T.Logf("PubkeyAuthentication is properly disabled as expected")
}

// ValidateSSHServiceDisabled validates that the SSH daemon service is disabled and stopped on the node
func ValidateSSHServiceDisabled(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Use VMSS RunCommand to check SSH service status directly on the node
	// Ubuntu uses 'ssh' as service name, while AzureLinux and Mariner use 'sshd'
	resp, err := RunCommand(ctx, s, `#!/bin/bash
# Determine the correct SSH service name based on the distro
# Ubuntu uses 'ssh', AzureLinux and Mariner use 'sshd'
if [ -f /etc/os-release ]; then
    . /etc/os-release
    if [[ "$ID" == "ubuntu" ]]; then
        SSH_SERVICE="ssh"
    else
        SSH_SERVICE="sshd"
    fi
else
    # Default to sshd if we can't determine the OS
    SSH_SERVICE="sshd"
fi

echo "Detected SSH service name: $SSH_SERVICE"

# Check SSH service status
status_output=$(systemctl status "$SSH_SERVICE" 2>&1)
echo "SSH service status output:"
echo "$status_output"

# Check if the service is inactive (dead) and disabled
if echo "$status_output" | grep -q "Active: inactive (dead)"; then
    if echo "$status_output" | grep -q "Loaded:.*disabled"; then
        echo "SUCCESS: SSH service is disabled and stopped"
        exit 0
    else
        echo "FAILED: SSH service is inactive but not disabled"
        exit 1
    fi
else
    echo "FAILED: SSH service is not inactive"
    exit 1
fi`)
	require.NoError(s.T, err, "Failed to run command to check SSH service status")
	stdout := lo.FromPtr(resp.Output)
	s.T.Logf("Run command stdout: %s\nstderr: %s", stdout, lo.FromPtr(resp.Error))

	// Check if the command execution was successful by looking for our success message in the output
	if !strings.Contains(stdout, "SUCCESS: SSH service is disabled and stopped") {
		s.T.Fatalf("SSH service is not properly disabled and stopped. stdout: %s", stdout)
	}

	s.T.Logf("SSH service is properly disabled and stopped as expected")
}

func ValidateNvidiaDCGMExporterSystemDServiceRunning(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Verify nvidia-dcgm service is running
		"systemctl is-active nvidia-dcgm",
		// Verify nvidia-dcgm-exporter service is running
		"systemctl is-active nvidia-dcgm-exporter",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Nvidia DCGM Exporter service validation failed")
}

func ValidateNvidiaDCGMExporterIsScrapable(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Check if nvidia-dcgm-exporter is scrapable on port 19400
		"curl -f http://localhost:19400/metrics",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Nvidia DCGM Exporter is not scrapable on port 19400")
}

func ValidateNvidiaDCGMExporterScrapeCommonMetric(ctx context.Context, s *Scenario, metric string) {
	s.T.Helper()
	command := []string{
		"set -ex",
		// Verify the most universal GPU metric is present
		"curl -s http://localhost:19400/metrics | grep -q '" + metric + "'",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "Nvidia DCGM Exporter is not returning "+metric)
}

func ValidateMIGModeEnabled(ctx context.Context, s *Scenario) {
	s.T.Helper()
	s.T.Logf("validating that MIG mode is enabled")

	command := []string{
		"set -ex",
		// Grep to verify it contains 'Enabled' - this will fail if MIG is disabled
		"sudo nvidia-smi --query-gpu=mig.mode.current --format=csv,noheader | grep -i 'Enabled'",
	}
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG mode is not enabled")

	stdout := strings.TrimSpace(execResult.stdout)
	s.T.Logf("MIG mode status: %s", stdout)
	require.Contains(s.T, stdout, "Enabled", "expected MIG mode to be enabled, but got: %s", stdout)
	s.T.Logf("MIG mode is enabled")
}

func ValidateMIGInstancesCreated(ctx context.Context, s *Scenario, migProfile string) {
	s.T.Helper()
	s.T.Logf("validating that MIG instances are created with profile %s", migProfile)

	command := []string{
		"set -ex",
		// List MIG devices using nvidia-smi
		"sudo nvidia-smi mig -lgi",
		// Ensure the output contains the expected MIG profile (will fail if "No MIG-enabled devices found")
		"sudo nvidia-smi mig -lgi | grep -v 'No MIG-enabled devices found' | grep -q '" + migProfile + "'",
	}
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "MIG instances with profile "+migProfile+" were not found")

	stdout := execResult.stdout
	require.Contains(s.T, stdout, migProfile, "expected to find MIG profile %s in output, but did not.\nOutput:\n%s", migProfile, stdout)
	require.NotContains(s.T, stdout, "No MIG-enabled devices found", "no MIG devices were created.\nOutput:\n%s", stdout)
	s.T.Logf("MIG instances with profile %s are created", migProfile)
}

// ValidateIPTablesCompatibleWithCiliumEBPF validates that all iptables rules in each table match the provided patterns which are accounted for
// when eBPF host routing is enabled.
func ValidateIPTablesCompatibleWithCiliumEBPF(ctx context.Context, s *Scenario) {
	s.T.Helper()
	tablePatterns, globalPatterns := getIPTablesRulesCompatibleWithEBPFHostRouting()
	tables := []string{"filter", "mangle", "nat", "raw", "security"}
	success := true

	for _, table := range tables {
		// Get the rules for this table
		command := fmt.Sprintf("sudo iptables -t %s -S", table)
		execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, command, 0, fmt.Sprintf("failed to get iptables rules for table %s", table))

		stdout := execResult.stdout
		rules := strings.Split(strings.TrimSpace(stdout), "\n")

		// Get patterns for this table
		patterns := tablePatterns[table]
		if patterns == nil {
			patterns = []string{}
		}

		// Combine with global patterns
		allPatterns := append([]string{}, globalPatterns...)
		allPatterns = append(allPatterns, patterns...)

		// Check each rule
		for _, rule := range rules {
			rule = strings.TrimSpace(rule)
			if rule == "" {
				continue
			}

			matched := false
			for _, pattern := range allPatterns {
				pattern = strings.TrimSpace(pattern)
				if pattern == "" {
					continue
				}

				// Try regex match
				matched, _ = regexp.MatchString(pattern, rule)
				if matched {
					break
				}

				// Also try exact match for non-regex patterns
				if strings.Contains(rule, pattern) {
					matched = true
					break
				}
			}

			if !matched {
				s.T.Logf("Rule in table %s did not match any pattern: %s", table, rule)
				success = false
			}
		}
	}

	require.True(
		s.T,
		success,
		"Rules found that do not match any of the given patterns. See previous log lines for details. "+
			"This may indicate an unsupported iptables rule when eBPF host routing is enabled. "+
			"Contact acndp@microsoft.com for details.",
	)
}

// ValidateAppArmorBasic validates that AppArmor is running without requiring aa-status
func ValidateAppArmorBasic(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Check if AppArmor module is enabled in the kernel
	command := []string{
		"set -ex",
		"cat /sys/module/apparmor/parameters/enabled",
	}
	execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to check AppArmor kernel parameter")
	stdout := strings.TrimSpace(execResult.stdout)
	require.Equal(s.T, "Y", stdout, "expected AppArmor to be enabled in kernel")

	// Check if apparmor.service is active
	command = []string{
		"set -ex",
		"systemctl is-active apparmor.service",
	}
	execResult = execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "apparmor.service is not active")
	stdout = strings.TrimSpace(execResult.stdout)
	require.Equal(s.T, "active", stdout, "expected apparmor.service to be active")

	// Check if AppArmor is enforcing by checking current process profile
	command = []string{
		"set -ex",
		"cat /proc/self/attr/apparmor/current",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to check AppArmor current profile")
	// Any output indicates AppArmor is active (profile will be shown)
}

func truncatePodName(t testing.TB, pod *corev1.Pod) {
	name := pod.Name
	if len(pod.Name) < 63 {
		return
	}
	pod.Name = pod.Name[:63]
	pod.Name = strings.TrimRight(pod.Name, "-")
	t.Logf("truncated pod name %q to %q", name, pod.Name)
}

// ValidateNodeHasLabel checks if the node has the expected label with the expected value
func ValidateNodeHasLabel(ctx context.Context, s *Scenario, labelKey, expectedValue string) {
	s.T.Helper()
	node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{})
	require.NoError(s.T, err, "failed to get node %q", s.Runtime.VM.KubeName)

	actualValue, exists := node.Labels[labelKey]
	require.True(s.T, exists, "expected node %q to have label %q, but it was not found", s.Runtime.VM.KubeName, labelKey)
	require.Equal(s.T, expectedValue, actualValue, "expected node %q label %q to have value %q, but got %q", s.Runtime.VM.KubeName, labelKey, expectedValue, actualValue)
}

// ValidateScriptlessCSECmd checks if the node has scriptless cmd correctly enabled
func ValidateScriptlessCSECmd(ctx context.Context, s *Scenario) {
	nbc := s.Runtime.NBC
	if nbc != nil && s.VHD.SupportsScriptless() && (nbc.EnableScriptlessCSECmd || nbc.EnableScriptlessNBCCSECmd) {
		ValidateFileExists(ctx, s, "/opt/azure/containers/scriptless-cse-overrides.txt")
	}
}

// ValidateScriptlessNBCCSECmd checks if the node has scriptless NBCCSECmd correctly enabled
func ValidateScriptlessNBCCSECmd(ctx context.Context, s *Scenario) {
	nbc := s.Runtime.NBC
	if nbc != nil && nbc.EnableScriptlessNBCCSECmd && s.VHD.SupportsScriptless() {
		fileNameToCheck := "/opt/azure/containers/aks-node-controller-nbc-cmd.sh"
		if enableScriptlessCompilation(s) {
			fileNameToCheck = "/opt/azure/containers/aks-node-controller-nbc-cmd-hack.sh"
		}
		ValidateFileExists(ctx, s, fileNameToCheck)
		ValidateFileHasContent(ctx, s, "/var/log/azure/aks-node-controller.log", "Using NBC command for scriptless phase 2")
	}
}

// ValidateScriptlessPhase3 validates that there are not diffs between ANC generated cse cmd NBC cse cmd vars
func ValidateScriptlessPhase3(ctx context.Context, s *Scenario) {
	s.T.Helper()
	if s.Runtime.AKSNodeConfig != nil && s.Runtime.NBC.EnableScriptlessNBCCSECmd {
		logFile := "/var/log/azure/aks-node-controller.log"
		if !fileHasContent(ctx, s, logFile, "env compare: no differences found between provision-config and nbc-cmd env vars") {
			// Grep for all env-compare diff markers to show what's different.
			diffCmd := "sudo grep -E 'differs|only-in-pc|only-in-nbc|env var differences' " + logFile + " || true"
			result := execScriptOnVMForScenarioValidateExitCode(ctx, s, diffCmd, 0, "could not grep for differences in aks-node-controller.log")
			s.T.Fatalf("expected no env var differences between provision-config and nbc-cmd, but found differences:\n%s", result.stdout)
		}
	}
}

// ValidateStaleCachedKubeBinariesRemoved validates that stale versioned kube binaries (e.g. kubelet-1.29.0, kubectl-1.29.0)
// have been removed from /opt/bin/ after the correct version is installed.
func ValidateStaleCachedKubeBinariesRemoved(ctx context.Context, s *Scenario) {
	s.T.Helper()
	// List any remaining versioned kubelet/kubectl binaries in /opt/bin/
	cmd := `find /opt/bin -maxdepth 1 \( -name "kubelet-*" -o -name "kubectl-*" \) -type f 2>/dev/null`
	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0, "could not list stale cached binaries")
	staleFiles := strings.TrimSpace(result.stdout)
	if staleFiles != "" {
		s.T.Fatalf("expected no stale cached binaries in /opt/bin/, but found:\n%s", staleFiles)
	}
}

// ValidateRxBufferDefault validates rx buffer config using default values based on VM's CPU count
func ValidateRxBufferDefault(ctx context.Context, s *Scenario) {
	s.T.Helper()

	// Query the VM's actual CPU count using nproc
	cpuCountCmd := "nproc"
	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cpuCountCmd, 0, "could not get CPU count from VM")
	vmCPUCount := strings.TrimSpace(result.stdout)

	// Parse CPU count
	cpuCount, err := strconv.Atoi(vmCPUCount)
	require.NoError(s.T, err, "failed to parse CPU count: %s", vmCPUCount)

	// Determine expected rx based on VM's CPU count (matching configure-azure-network.sh logic)
	expectedRx := "1024"
	if cpuCount >= 4 {
		expectedRx = "2048"
	}

	s.T.Logf("VM has %d CPUs, expecting rx buffer size: %s", cpuCount, expectedRx)

	customNicConfig := map[string]string{
		"rx": expectedRx,
	}

	// Validate files exist
	ValidateAzureNetworkFiles(ctx, s)

	// Validate network interface settings match expected default
	ValidateNetworkInterfaceConfig(ctx, s, customNicConfig)
}

// ValidateKernelLogs checks kernel logs for critical errors across multiple categories:
// - Kernel panics/crashes (panic, oops, call trace, BUG, etc.)
// - CPU lockups/stalls (soft/hard lockup, RCU stall, hung task, watchdog)
// - Memory issues (OOM killer, page allocation failure, memory corruption)
// - I/O and filesystem errors (I/O error, filesystem errors, nvme/ata/scsi errors)
func ValidateKernelLogs(ctx context.Context, s *Scenario) {
	s.T.Helper()

	if s.VHD != nil && s.VHD.SkipOldVHDValidations {
		return
	}

	type categoryPattern struct {
		pattern string
		exclude string // optional pattern to exclude false positives
	}
	// sr[0-9] is the virtual CD-ROM drive on Azure VMs. This error occurs when the VM tries to read from an empty virtual optical drive, which is normal and expected.
	// "Shutdown timeout set to" is an informational message from the NVMe driver during initialization, not an error.
	ioFSExclude := `sr[0-9]|Shutdown timeout set to`
	if s.VHD != nil && s.VHD.OS == config.OSACL {
		// ACL-only: exclude benign BTRFS udev race warnings ("duplicate device") and loop device I/O errors
		// from sysext squashfs read-ahead overshooting the backing file boundary.
		ioFSExclude += `|duplicate device|loop[0-9]`
	}

	patterns := map[string]categoryPattern{
		"PANIC/CRASH": {
			pattern: `(kernel: )?(panic|oops|call trace|backtrace|general protection fault|BUG:|RIP:)`,
			// exclude boot parameters like "panic=-1" and dm-verity's "panic-on-corruption" (used by ACL for verified boot)
			exclude: `panic[-=]`,
		},
		"LOCKUP/STALL": {pattern: `(soft|hard) lockup|rcu.*(stall|detected stalls)|hung task|watchdog.*(detected|stuck)`},
		"MEMORY":       {pattern: `oom[- ]killer|Out of memory:|page allocation failure|memory corruption`},
		"IO/FS": {
			pattern: `I/O error|read-only file system|EXT[2-4]-fs error|XFS.*(ERROR|error|corruption)|BTRFS.*(error|warning)|nvme .* (timeout|reset)|ata[0-9].*(failed|error|reset)|scsi.*(error|failed)`,
			exclude: ioFSExclude,
		},
	}

	// Collect all issues first before potentially failing
	issuesFound := make(map[string]string)
	for category, cp := range patterns {
		var command []string
		if cp.exclude != "" {
			command = []string{
				"set -e",
				"output=$(sudo dmesg)",
				fmt.Sprintf("echo \"$output\" | grep -iE '%s' | grep -ivE '%s' || true", cp.pattern, cp.exclude),
			}
		} else {
			command = []string{
				"set -e",
				"output=$(sudo dmesg)",
				fmt.Sprintf("echo \"$output\" | grep -iE '%s' || true", cp.pattern),
			}
		}
		execResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0, "failed to retrieve kernel logs")

		stdout := strings.TrimSpace(execResult.stdout)
		if stdout != "" {
			issuesFound[category] = stdout
		}
	}

	// If issues found, write the full kernel dump to a file for debugging
	if len(issuesFound) > 0 {
		// Get full kernel log dump and write to file
		fullDmesgResult := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo dmesg", 0, "failed to retrieve full kernel logs")
		logFileName := "kernel-log.txt"
		if err := writeToFile(s.T, logFileName, fullDmesgResult.stdout); err != nil {
			s.T.Logf("Warning: failed to write kernel log to file: %v", err)
		} else {
			s.T.Logf("Full kernel log written to: %s/%s", testDir(s.T), logFileName)
		}

		// Log each category of issues found
		var summary strings.Builder
		summary.WriteString("Critical kernel issues detected:\n")
		for category, issues := range issuesFound {
			summary.WriteString(fmt.Sprintf("\n[%s]:\n%s\n", category, issues))
		}
		s.T.Fatalf("%s", summary.String())
	}

	s.T.Logf("No critical kernel issues found")
}

// ValidateWaagentLog checks /var/log/waagent.log for expected agent behavior:
// - AutoUpdate is disabled as expected
// - The correct version is running as ExtHandler
// - No errors from ExtHandler
// Skipped on Flatcar and OSGuard VHDs which manage WALinuxAgent independently.
func ValidateWaagentLog(ctx context.Context, s *Scenario) {
	s.T.Helper()

	if s.VHD.Flatcar || strings.Contains(string(s.VHD.Distro), "osguard") || s.VHD.SkipOldVHDValidations {
		s.T.Logf("Skipping waagent log validation: not applicable for %s", s.VHD.Distro)
		return
	}

	versions := components.GetExpectedPackageVersions("walinuxagent", "default", "current")
	if len(versions) == 0 || versions[0] == "<SKIP>" {
		s.T.Log("Skipping waagent log validation: no walinuxagent version in components.json")
		return
	}
	expectedVersion := versions[0]

	const waagentLogFile = "/var/log/waagent.log"

	logContents := execScriptOnVMForScenarioValidateExitCode(ctx, s,
		"sudo cat "+waagentLogFile, 0,
		"could not read waagent log").stdout

	// 1. Verify AutoUpdate is disabled
	require.Contains(s.T, logContents, "AutoUpdate.UpdateToLatestVersion is set to False, not processing the operation",
		"waagent.log should confirm AutoUpdate.UpdateToLatestVersion is set to False")

	// 2. Verify the correct version is running as ExtHandler (PID varies)
	expectedRunningPattern := fmt.Sprintf("ExtHandler WALinuxAgent-%s running as process", expectedVersion)
	require.Contains(s.T, logContents, expectedRunningPattern,
		"waagent.log should confirm WALinuxAgent-%s is running as ExtHandler", expectedVersion)

	// 3. Check for ExtHandler errors
	// On Ubuntu 22.04 FIPS VHDs, waagent logs "Cannot convert PFX to PEM" because
	// of a known bug with VMSS that fails to propagate the FIPS additionalCapabilities.
	// Until the VMSS bug is fixed, skip the "Cannot convert PFX to PEM" errors.
	// TODO: Remove the conditional exclusion once the underlying VMSS issue is resolved.
	isUbuntu2204FIPS := s.VHD == config.VHDUbuntu2204FIPSContainerd ||
		s.VHD == config.VHDUbuntu2204Gen2FIPSContainerd ||
		s.VHD == config.VHDUbuntu2204Gen2FIPSTLContainerd
	grepCmd := fmt.Sprintf("sudo grep 'ERROR ExtHandler' %s || true", waagentLogFile)
	if isUbuntu2204FIPS {
		grepCmd = fmt.Sprintf("sudo grep 'ERROR ExtHandler' %s | grep -v 'Cannot convert PFX to PEM' || true", waagentLogFile)
	}
	extHandlerErrors := execScriptOnVMForScenarioValidateExitCode(ctx, s,
		strings.Join([]string{
			"set -e",
			grepCmd,
		}, "\n"), 0,
		"failed to scan waagent log for ExtHandler errors")

	errOutput := strings.TrimSpace(extHandlerErrors.stdout)
	if errOutput != "" {
		logFileName := "waagent-exthandler-errors.log"
		if err := writeToFile(s.T, logFileName, logContents); err != nil {
			s.T.Logf("Warning: failed to write waagent log to file: %v", err)
		} else {
			s.T.Logf("Full waagent log written to: %s/%s", testDir(s.T), logFileName)
		}
		s.T.Fatalf("ExtHandler errors found in waagent.log:\n%s", errOutput)
	}

	s.T.Logf("waagent.log validation passed: WALinuxAgent-%s running correctly with no ExtHandler errors", expectedVersion)
}

// ValidateCollectWindowsLogsScript runs c:\k\debug\collect-windows-logs.ps1 on the node
// and verifies that a zip archive was produced by the script.
func ValidateCollectWindowsLogsScript(ctx context.Context, s *Scenario) {
	s.T.Helper()
	command := []string{
		"$ErrorActionPreference = \"Stop\"",
		"cd c:/",
		"mkdir logs-for-test",
		"cd logs-for-test",
		"$startTime = Get-Date",
		"Remove-Item \"*_logs.zip\" -ErrorAction SilentlyContinue",
		"$ErrorActionPreference = \"SilentlyContinue\"",
		"& c:\\k\\debug\\collect-windows-logs.ps1",
		"$ErrorActionPreference = \"Stop\"",
		"$zipFile = Get-ChildItem -Filter \"*_logs.zip\" | Sort-Object LastWriteTime -Descending | Select-Object -First 1",
		"if (-not $zipFile) { throw \"collect-windows-logs.ps1 did not create a zip file\" }",
		"Write-Host \"Zip file created: $($zipFile.FullName) (Size: $($zipFile.Length) bytes)\"",
	}
	execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(command, "\n"), 0,
		"collect-windows-logs.ps1 failed or did not produce a zip file")
}

// ValidateVulnerableKernelModulesDisabled verifies that kernel modules with known LPE
// vulnerabilities (CVE-2026-31431 / DirtyFrag / Fragnesia: algif_aead, esp4, esp6, rxrpc)
// are handled correctly per OS:
//
//   - Ubuntu / Mariner: full check — modprobe config entries are present, modules are
//     NOT loaded, and modprobe refuses to load them.
//   - AzureLinux 3.0: assert ABSENCE of the four modprobe blacklist entries. AzL3 is
//     descoped from the mitigation because kernel 6.6.139.1-1.azl3 and later fix all
//     three CVEs upstream, AND customer workloads on AzL3 require those modules (the
//     blacklist actively blocks legitimate use cases). Newly-built AzL3 VHDs therefore
//     no longer ship the modprobe-CIS.conf entries, and E2E runs against freshly-built
//     VHDs. See https://github.com/Azure/AKS/issues/5753.
//
// To add a new CVE mitigation, append the module name to BOTH lists below —
// the AzureLinux 3.0 absence-check list AND the default presence + load-refusal list.
func ValidateVulnerableKernelModulesDisabled(ctx context.Context, s *Scenario) {
	s.T.Helper()

	if s.VHD.Flatcar && s.VHD.OS != config.OSACL {
		s.T.Log("Skipping vulnerable kernel module validation: not applicable for Flatcar")
		return
	}

	// AzureLinux 3.0 (regular, NOT OSGuard): kernel 6.6.139.1-1.azl3+ supersedes the modprobe
	// blacklist and the bake-in has been removed because customers need those modules. Assert
	// the blacklist entries are NOT present on freshly-built AzL3 VHDs. AzureLinux OSGuard is
	// intentionally kept in-scope (falls through to the full presence + load-refusal check below).
	if s.VHD.OS == config.OSAzureLinux && !s.VHD.Distro.IsAzureLinuxOSGuardDistro() && s.VHD.Distro != datamodel.AKSAzureLinuxV2Gen2 {
		script := strings.Join([]string{
			`failed=0`,
			`for mod in algif_aead esp4 esp6 rxrpc; do`,
			`  if grep -qsE "^(install ${mod} /bin/false|blacklist ${mod})" /etc/modprobe.d/*.conf 2>/dev/null; then`,
			`    echo "FAIL: ${mod} blacklist entry unexpectedly present on AzureLinux 3.0 (bake-in removed; kernel 6.6.139.1-1.azl3+ supersedes)"`,
			`    failed=1`,
			`  else`,
			`    echo "PASS: ${mod} blacklist correctly absent on AzureLinux 3.0"`,
			`  fi`,
			`done`,
			`exit $failed`,
		}, "\n")
		execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
			"AzureLinux 3.0 modprobe blacklist should be absent (kernel fix 6.6.139.1-1.azl3+ supersedes; bake-in removed; no `install` or `blacklist` directive should remain)")
		return
	}

	script := strings.Join([]string{
		`failed=0`,
		`for mod in algif_aead esp4 esp6 rxrpc; do`,
		`  if ! grep -qsE "^install ${mod} /bin/false" /etc/modprobe.d/*.conf 2>/dev/null; then`,
		`    echo "FAIL: ${mod} disable rule not found in /etc/modprobe.d/*.conf"`,
		`    failed=1`,
		`  else`,
		`    echo "PASS: modprobe config blocks ${mod}"`,
		`  fi`,
		`  if grep -qE "^${mod} " /proc/modules 2>/dev/null; then`,
		`    echo "FAIL: ${mod} module is loaded"`,
		`    failed=1`,
		`  else`,
		`    echo "PASS: ${mod} module is not loaded"`,
		`  fi`,
		`  if sudo modprobe "${mod}" 2>/dev/null; then`,
		`    echo "FAIL: modprobe ${mod} succeeded, should be blocked"`,
		`    sudo modprobe -r "${mod}" 2>/dev/null || true`,
		`    failed=1`,
		`  else`,
		`    echo "PASS: modprobe ${mod} correctly refused"`,
		`  fi`,
		`done`,
		`exit $failed`,
	}, "\n")

	execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0,
		"Vulnerable kernel module mitigation validation failed (algif_aead/esp4/esp6/rxrpc)")
}

// resolveSecondaryNICName discovers the kernel interface name of the secondary NIC
// (IMDS interface index 1) by matching its MAC address against /sys/class/net/*/address.
// This avoids hardcoding "eth1" which can be wrong when SR-IOV VFs or predictable
// naming (ens*/enP*) are in use.
func resolveSecondaryNICName(ctx context.Context, s *Scenario) string {
	s.T.Helper()
	// Get the secondary NIC's MAC from IMDS, then look it up in sysfs.
	// -sf makes curl fail with non-zero exit on HTTP errors (403/404) instead
	// of silently returning the error body as the "MAC".
	// Skip SR-IOV VFs (enslaved interfaces with /sys/class/net/<iface>/master)
	// which share the MAC of their master but don't hold an IP address.
	// Exit 1 if no matching interface is found rather than falling back to a
	// hardcoded name that could target a VF or wrong interface.
	cmd := `mac=$(curl -sf -H "Metadata:true" "http://169.254.169.254/metadata/instance/network/interface/1/macAddress?api-version=2021-02-01&format=text") || { echo "IMDS MAC lookup failed" >&2; exit 1; }; mac_lower=$(echo "$mac" | sed 's/\(..\)/\1:/g; s/:$//' | tr '[:upper:]' '[:lower:]'); for f in /sys/class/net/*/address; do d=$(dirname "$f"); [ -e "$d/master" ] && continue; if [ "$(cat "$f" 2>/dev/null)" = "$mac_lower" ]; then basename "$d"; exit 0; fi; done; echo "no interface found for MAC $mac_lower" >&2; exit 1`
	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0,
		"failed to resolve secondary NIC interface name")
	ifaceName := strings.TrimSpace(result.stdout)
	require.NotEmpty(s.T, ifaceName, "resolved secondary NIC name should not be empty")
	return ifaceName
}

// ValidateSecondaryNICUp checks that the given network interface is UP and has an IPv4 address.
func ValidateSecondaryNICUp(ctx context.Context, s *Scenario, ifaceName string) {
	s.T.Helper()
	cmd := fmt.Sprintf("ip addr show %s", ifaceName)
	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0,
		fmt.Sprintf("failed to get interface info for %s", ifaceName))
	require.Contains(s.T, result.stdout, "state UP",
		"expected interface %s to be UP, got:\n%s", ifaceName, result.stdout)
	require.Contains(s.T, result.stdout, "inet ",
		"expected interface %s to have an IPv4 address, got:\n%s", ifaceName, result.stdout)
}

// ValidateSecondaryNICDualStack checks that the given network interface is UP and has both IPv4 and IPv6 addresses.
func ValidateSecondaryNICDualStack(ctx context.Context, s *Scenario, ifaceName string) {
	s.T.Helper()
	cmd := fmt.Sprintf("ip addr show %s", ifaceName)
	result := execScriptOnVMForScenarioValidateExitCode(ctx, s, cmd, 0,
		fmt.Sprintf("failed to get interface info for %s", ifaceName))
	require.Contains(s.T, result.stdout, "state UP",
		"expected interface %s to be UP, got:\n%s", ifaceName, result.stdout)
	require.Contains(s.T, result.stdout, "inet ",
		"expected interface %s to have an IPv4 address, got:\n%s", ifaceName, result.stdout)
	require.Contains(s.T, result.stdout, "inet6 ",
		"expected interface %s to have an IPv6 address, got:\n%s", ifaceName, result.stdout)
	require.Contains(s.T, result.stdout, "scope global",
		"expected interface %s to have a global IPv6 address (not just link-local), got:\n%s", ifaceName, result.stdout)
}