diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index fa899c4d366..2561d5419e5 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -1334,4 +1334,5 @@ stages: - template: ./templates/e2e-template.yaml parameters: name: All Linux + IgnoreScenariosWithMissingVhd: true diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index 48921c5272f..1af50d9386e 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -138,4 +138,6 @@ stages: VHD_BUILD_ID: $(Build.BuildId) jobs: - template: ./templates/e2e-template.yaml + parameters: + IgnoreScenariosWithMissingVhd: true diff --git a/.pipelines/e2e-windows.yaml b/.pipelines/e2e-windows.yaml index 52d0fa17f93..3921664b160 100644 --- a/.pipelines/e2e-windows.yaml +++ b/.pipelines/e2e-windows.yaml @@ -36,3 +36,4 @@ jobs: - template: ./templates/e2e-template.yaml parameters: name: Windows Tests + IgnoreScenariosWithMissingVhd: false diff --git a/.pipelines/e2e.yaml b/.pipelines/e2e.yaml index 677987904c8..713adbf6b42 100644 --- a/.pipelines/e2e.yaml +++ b/.pipelines/e2e.yaml @@ -34,3 +34,4 @@ jobs: - template: ./templates/e2e-template.yaml parameters: name: Linux Tests + IgnoreScenariosWithMissingVhd: false \ No newline at end of file diff --git a/.pipelines/templates/.build-and-test-windows-vhd-template.yaml b/.pipelines/templates/.build-and-test-windows-vhd-template.yaml index 5054d865e46..8291ef213f3 100644 --- a/.pipelines/templates/.build-and-test-windows-vhd-template.yaml +++ b/.pipelines/templates/.build-and-test-windows-vhd-template.yaml @@ -95,4 +95,5 @@ stages: - template: ./e2e-template.yaml parameters: name: For image ${{ parameters.imageName }} + IgnoreScenariosWithMissingVhd: false diff --git a/.pipelines/templates/e2e-template.yaml b/.pipelines/templates/e2e-template.yaml index 8c5637f8aba..9fe9e9e9b78 100644 --- a/.pipelines/templates/e2e-template.yaml +++ b/.pipelines/templates/e2e-template.yaml @@ -3,6 +3,9 @@ parameters: type: string displayName: Additional name for the pipeline step default: "" + - name: IgnoreScenariosWithMissingVhd + type: boolean + displayName: Ignore tests where the VHD can't be loaded jobs: - job: e2e @@ -14,6 +17,13 @@ jobs: timeoutInMinutes: 90 displayName: Run AgentBaker E2E ${{parameters.name}} steps: + - checkout: self + # s is the default path for repositories - if we don't set this when using multiple repsositories, then it is the repo name + # "s/AgentBaker", but most of the pipeline variables still refer to "s" and many of the scripts below fail. + path: s + fetchTags: false + fetchDepth: 1 + - bash: | set -ex az login --identity --username $(E2E_AGENT_IDENTITY_ID) @@ -28,16 +38,18 @@ jobs: echo "##vso[task.setvariable variable=LOGGING_DIR]$LOGGING_DIR" displayName: Set logging directory - bash: | - set -ex + set -e export PATH="/usr/local/go/bin:$PATH" go version echo "VHD_BUILD_ID=$VHD_BUILD_ID" - + echo "IGNORE_SCENARIOS_WITH_MISSING_VHD: $IGNORE_SCENARIOS_WITH_MISSING_VHD" + echo "LOGGING_DIR: $LOGGING_DIR" + if [ -n "${VHD_BUILD_ID}" ]; then + echo "VHD_BUILD_ID is specified (${VHD_BUILD_ID}). Running tests using VHDs from that build" export SIG_VERSION_TAG_NAME=buildId export SIG_VERSION_TAG_VALUE=$VHD_BUILD_ID - export IGNORE_SCENARIOS_WITH_MISSING_VHD=true else echo "VHD_BUILD_ID is not specified. Running tests with default SIG version tag selectors." fi @@ -50,6 +62,7 @@ jobs: env: VHD_BUILD_ID: $(VHD_BUILD_ID) LOGGING_DIR: $(LOGGING_DIR) + IGNORE_SCENARIOS_WITH_MISSING_VHD: ${{parameters.IgnoreScenariosWithMissingVhd}} # Without this step next step may fail on successful run - bash: | set -ex diff --git a/e2e/config/azure.go b/e2e/config/azure.go index b29172d4cf5..e23266eb6e0 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -386,6 +386,7 @@ func (a *AzureClient) LatestSIGImageVersionByTag(ctx context.Context, t *testing } versions := page.Value for _, version := range versions { + t.Logf("Found image version: %s", version.Tags) // skip images tagged with the no-selection tag, indicating they // shouldn't be selected dynmically for running abe2e scenarios if _, ok := version.Tags[noSelectionTagName]; ok { diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index b8300f1f51f..d94d14d8795 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -144,7 +144,6 @@ var ( OS: "windows", Arch: "amd64", Distro: datamodel.AKSWindows2019Containerd, - Latest: true, Gallery: windowsGallery, } @@ -153,7 +152,6 @@ var ( OS: "windows", Arch: "amd64", Distro: datamodel.AKSWindows2022Containerd, - Latest: true, Gallery: windowsGallery, } @@ -162,7 +160,6 @@ var ( OS: OSWindows, Arch: "amd64", Distro: datamodel.AKSWindows2022ContainerdGen2, - Latest: true, Gallery: windowsGallery, } @@ -171,7 +168,6 @@ var ( OS: OSWindows, Arch: "amd64", Distro: datamodel.AKSWindows23H2, - Latest: true, Gallery: windowsGallery, } @@ -180,7 +176,6 @@ var ( OS: OSWindows, Arch: "amd64", Distro: datamodel.AKSWindows23H2Gen2, - Latest: true, Gallery: windowsGallery, } @@ -189,7 +184,6 @@ var ( OS: OSWindows, Arch: "amd64", Distro: datamodel.AKSWindows2025, - Latest: true, Gallery: windowsGallery, } @@ -198,7 +192,6 @@ var ( OS: OSWindows, Arch: "amd64", Distro: datamodel.AKSWindows2025Gen2, - Latest: true, Gallery: windowsGallery, } ) @@ -212,7 +205,6 @@ type Image struct { OS OS Version string Gallery *Gallery - Latest bool // a hack to get the latest version of the image for windows, currently windows images are not tagged vhd VHDResourceID vhdOnce sync.Once @@ -227,16 +219,19 @@ func (i *Image) String() string { func (i *Image) VHDResourceID(ctx context.Context, t *testing.T) (VHDResourceID, error) { i.vhdOnce.Do(func() { switch { - case i.Latest: - i.vhd, i.vhdErr = Azure.LatestSIGImageVersionByTag(ctx, t, i, "", "") case i.Version != "": i.vhd, i.vhdErr = Azure.EnsureSIGImageVersion(ctx, t, i) + t.Logf("got version vid %s: %s", i.Version, i.vhd) default: i.vhd, i.vhdErr = Azure.LatestSIGImageVersionByTag(ctx, t, i, Config.SIGVersionTagName, Config.SIGVersionTagValue) + t.Logf("got version by tag %s=%s: %s", Config.SIGVersionTagName, Config.SIGVersionTagValue, i.vhd) } if i.vhdErr != nil { i.vhdErr = fmt.Errorf("img: %s, tag %s=%s, err %w", i.Name, Config.SIGVersionTagName, Config.SIGVersionTagValue, i.vhdErr) - t.Logf("failed to find the latest image version for %s", i.vhdErr) + t.Logf("Failed to find the image. Sub=%s rg=%s gallary=%s version for with err %s", i.Gallery.SubscriptionID, i.Gallery.ResourceGroupName, i.Gallery.Name, i.vhdErr) + } else { + t.Logf("Found the image. Sub=%s rg=%s gallary=%s version for with id %s", i.Gallery.SubscriptionID, i.Gallery.ResourceGroupName, i.Gallery.Name, i.vhd.Short()) + } }) return i.vhd, i.vhdErr diff --git a/e2e/scenario_helpers_test.go b/e2e/scenario_helpers_test.go index 589f305e72b..883e61b5735 100644 --- a/e2e/scenario_helpers_test.go +++ b/e2e/scenario_helpers_test.go @@ -85,6 +85,7 @@ func RunScenario(t *testing.T, s *Scenario) { s.T = t t.Parallel() ctx := newTestCtx(t) + maybeSkipScenario(ctx, t, s) cluster, err := s.Config.Cluster(ctx, s.T) require.NoError(s.T, err) diff --git a/e2e/scenario_win_test.go b/e2e/scenario_win_test.go index f4694c3a9fb..de9602f3975 100644 --- a/e2e/scenario_win_test.go +++ b/e2e/scenario_win_test.go @@ -28,6 +28,7 @@ func Test_Windows2019Containerd(t *testing.T) { ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) ValidateCiliumIsNotRunningWindows(ctx, s) + ValidateFileHasContent(ctx, s, "/k/test.txt", "this is a test file") }, }, }) @@ -48,6 +49,7 @@ func Test_Windows2022Containerd(t *testing.T) { ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) ValidateCiliumIsNotRunningWindows(ctx, s) + ValidateFileHasContent(ctx, s, "/k/test.txt", "this is a test file") }, }, }) @@ -68,6 +70,7 @@ func Test_Windows2022ContainerdGen2(t *testing.T) { ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) ValidateCiliumIsNotRunningWindows(ctx, s) + ValidateFileHasContent(ctx, s, "/k/test.txt", "this is a test file") }, }, }) @@ -88,6 +91,7 @@ func Test_Windows23H2(t *testing.T) { ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) ValidateCiliumIsNotRunningWindows(ctx, s) + ValidateFileHasContent(ctx, s, "/k/test.txt", "this is a test file") }, }, }) @@ -108,6 +112,7 @@ func Test_Windows23H2Gen2(t *testing.T) { ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote") ValidateWindowsProcessHasCliArguments(ctx, s, "kubelet.exe", []string{"--rotate-certificates=true", "--client-ca-file=c:\\k\\ca.crt"}) ValidateCiliumIsNotRunningWindows(ctx, s) + ValidateFileHasContent(ctx, s, "/k/test.txt", "this is a test file") }, }, }) diff --git a/e2e/types.go b/e2e/types.go index c764d3e114e..c826d35a39a 100644 --- a/e2e/types.go +++ b/e2e/types.go @@ -162,6 +162,8 @@ func (s *Scenario) PrepareVMSSModel(ctx context.Context, t *testing.T, vmss *arm require.NotNil(t, vmss, "input VirtualMachineScaleSet") require.NotNil(t, vmss.Properties, "input VirtualMachineScaleSet.Properties") + s.T.Logf("got vhd resource id %s", resourceID) + if s.VMConfigMutator != nil { s.VMConfigMutator(vmss) } diff --git a/e2e/validators.go b/e2e/validators.go index 6f6e9b21c32..a0d9ac6253f 100644 --- a/e2e/validators.go +++ b/e2e/validators.go @@ -81,9 +81,11 @@ func ValidateNonEmptyDirectory(ctx context.Context, s *Scenario, dirName string) func ValidateFileHasContent(ctx context.Context, s *Scenario, fileName string, contents string) { if s.VHD.OS == config.OSWindows { steps := []string{ + "$ErrorActionPreference = \"Stop\"", fmt.Sprintf("dir %[1]s", fileName), fmt.Sprintf("Get-Content %[1]s", fileName), - fmt.Sprintf("if (Select-String -Path %s -Pattern \"%s\" -SimpleMatch -Quiet) { return 1 } else { return 0 }", fileName, contents), + fmt.Sprintf("if ( -not ( Test-Path -Path %s ) ) { exit 2 }", fileName), + fmt.Sprintf("if (Select-String -Path %s -Pattern \"%s\" -SimpleMatch -Quiet) { exit 0 } else { exit 1 }", fileName, contents), } execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate file has contents - might mean file does not have contents, might mean something went wrong") @@ -102,14 +104,26 @@ func ValidateFileHasContent(ctx context.Context, s *Scenario, fileName string, c func ValidateFileExcludesContent(ctx context.Context, s *Scenario, fileName string, contents string) { require.NotEqual(s.T, "", contents, "Test setup failure: Can't validate that a file excludes an empty string. Filename: %s", fileName) - steps := []string{ - "set -ex", - fmt.Sprintf("test -f %[1]s || exit 0", fileName), - fmt.Sprintf("ls -la %[1]s", fileName), - fmt.Sprintf("sudo cat %[1]s", fileName), - fmt.Sprintf("(sudo cat %[1]s | grep -q -v -F -e %[2]q)", fileName, contents), + if s.VHD.OS == config.OSWindows { + steps := []string{ + "$ErrorActionPreference = \"Stop\"", + fmt.Sprintf("dir %[1]s", fileName), + fmt.Sprintf("Get-Content %[1]s", fileName), + fmt.Sprintf("if ( -not ( Test-Path -Path %s ) ) { exit 2 }", fileName), + fmt.Sprintf("if (Select-String -Path %s -Pattern \"%s\" -SimpleMatch -Quiet) { exit 1 } else { exit 0 }", fileName, contents), + } + + execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate file has contents - might mean file does not have contents, might mean something went wrong") + } else { + steps := []string{ + "set -ex", + fmt.Sprintf("test -f %[1]s || exit 0", fileName), + fmt.Sprintf("ls -la %[1]s", fileName), + fmt.Sprintf("sudo cat %[1]s", fileName), + fmt.Sprintf("(sudo cat %[1]s | grep -q -v -F -e %[2]q)", fileName, contents), + } + execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate file excludes contents - might mean file does have contents, might mean something went wrong") } - execScriptOnVMForScenarioValidateExitCode(ctx, s, strings.Join(steps, "\n"), 0, "could not validate file excludes contents - might mean file does have contents, might mean something went wrong") } func ServiceCanRestartValidator(ctx context.Context, s *Scenario, serviceName string, restartTimeoutInSeconds int) { diff --git a/e2e/vmss.go b/e2e/vmss.go index 18a6283b296..2823ca9cf37 100644 --- a/e2e/vmss.go +++ b/e2e/vmss.go @@ -52,6 +52,16 @@ func createVMSS(ctx context.Context, s *Scenario) *armcompute.VirtualMachineScal customData = nodeBootstrapping.CustomData } + if nodeBootstrapping != nil && nodeBootstrapping.SigImageConfig != nil { + s.T.Logf("vhd sub id %s", config.Config.SubscriptionID) + s.T.Logf("vhd rgr id %s", config.ResourceGroupName) + s.T.Logf("vhd galler %s", config.Config.GalleryNameWindows) + s.T.Logf("vhd versio %s", nodeBootstrapping.SigImageConfig.Version) + s.T.Logf("vhd defini %s", nodeBootstrapping.SigImageConfig.Definition) + } else { + s.T.Logf("nbc or sigimageconfig is nil") + } + model := getBaseVMSSModel(s, customData, cse) if s.Tags.NonAnonymousACR { // add acr pull identity diff --git a/vhdbuilder/packer/windows/configure-windows-vhd.ps1 b/vhdbuilder/packer/windows/configure-windows-vhd.ps1 index e69880723c2..2f666fae792 100644 --- a/vhdbuilder/packer/windows/configure-windows-vhd.ps1 +++ b/vhdbuilder/packer/windows/configure-windows-vhd.ps1 @@ -903,6 +903,9 @@ try Clear-TempFolder Log-VHDFreeSize Test-AzureExtensions + + Write-Output "creating test.txt file" + Write-Output "this is a test file" > "c:\k\test.txt" } default { Write-Log "Unable to determine provisiong phase... exiting"