From 571fd375987105af99ec003165b29654b2cc0d2d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 26 Sep 2025 10:24:22 +0200 Subject: [PATCH 1/4] Run test template tests with Arcade --- build/RunTestTemplateTests.ps1 | 46 ---------------------- eng/pipelines/templates/jobs/sdk-build.yml | 15 +++++-- 2 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 build/RunTestTemplateTests.ps1 diff --git a/build/RunTestTemplateTests.ps1 b/build/RunTestTemplateTests.ps1 deleted file mode 100644 index 5fe903b77f34..000000000000 --- a/build/RunTestTemplateTests.ps1 +++ /dev/null @@ -1,46 +0,0 @@ -<# -.SYNOPSIS - Runs Microsoft.TestTemplates.Acceptance.Tests.dll in the dogfood environment. -.DESCRIPTION - This script enters the dogfood environment and runs the RunTestTemplateTests tests. -#> -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $configuration = "Release" -) - -function Run-TestTemplateTests { - $ErrorActionPreference = 'Stop' - $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') - $classNameFilter = "--filter" - $filterValue = "FullyQualifiedName~Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests" - $TestDll = Join-Path $RepoRoot "artifacts\bin\dotnet-new.IntegrationTests\$configuration\dotnet-new.IntegrationTests.dll" - - # Check if the test DLL exists - if (-not (Test-Path $TestDll)) { - Write-Error "Test DLL not found at: $TestDll" - return 1 - } - - Write-Host "Running tests for test templates in the dogfood environment..." -ForegroundColor Cyan - - # Call dogfood.ps1 directly instead of through dogfood.cmd to avoid the -NoExit parameter - $dogfoodPs1 = Join-Path $RepoRoot "eng\dogfood.ps1" - - Write-Host "Executing: dotnet test $TestDll via dogfood environment" -ForegroundColor Gray - # Pass the command directly to the dogfood.ps1 script - & $dogfoodPs1 -configuration $configuration -command @("dotnet", "test", $TestDll, $classNameFilter, $filterValue) - - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-Error "Tests failed with exit code: $exitCode" - } else { - Write-Host "Tests completed successfully!" -ForegroundColor Green - } - - return $exitCode -} - -# Execute the function using Invoke-Command -$exitCode = Invoke-Command -ScriptBlock ${function:Run-TestTemplateTests} -exit $exitCode diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 202d480959af..2cc7d6d38049 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -105,10 +105,6 @@ jobs: BuildConfig: $(buildConfiguration) TestFullMSBuild: ${{ parameters.testFullMSBuild }} - - ${{ if eq(parameters.targetArchitecture, 'x64') }}: - - powershell: build/RunTestTemplateTests.ps1 - displayName: 🟣 Run Test Templates Tests - - ${{ else }}: - script: | source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh @@ -135,6 +131,17 @@ jobs: OPENSSL_ENABLE_SHA1_SIGNATURES: 1 ############### TESTING ############### + - ${{ if eq(parameters.pool.os, 'windows') }}: + - ${{ if eq(parameters.targetArchitecture, 'x64') }}: + - powershell: eng/common/build.ps1 + -restore -test -ci -prepareMachine -nativeToolsOnMachine + -configuration $(buildConfiguration) + /p:Projects="test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj" + ${{ parameters.runtimeSourceProperties }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/TestTemplateTests.binlog + displayName: 🟣 Run Test Templates Tests + condition: succeeded() + - ${{ if eq(parameters.runTests, true) }}: - ${{ if eq(parameters.runAoTTests, true) }}: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 From ac67de50eed900336989bf42a7b7da7218db854b Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Fri, 26 Sep 2025 10:51:20 +0200 Subject: [PATCH 2/4] Update sdk-build.yml --- eng/pipelines/templates/jobs/sdk-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 2cc7d6d38049..a1d2212e8b0a 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -136,7 +136,7 @@ jobs: - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) - /p:Projects="test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj" + /p:Projects="$(Build.SourcesDirectory)/test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj" ${{ parameters.runtimeSourceProperties }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/TestTemplateTests.binlog displayName: 🟣 Run Test Templates Tests From 7e865a27c68517292643385a8e122011f238373c Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Fri, 26 Sep 2025 12:01:17 +0200 Subject: [PATCH 3/4] Only test templates --- eng/pipelines/templates/jobs/sdk-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index a1d2212e8b0a..49df93cd2b54 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -137,6 +137,7 @@ jobs: -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) /p:Projects="$(Build.SourcesDirectory)/test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj" + /p:TestRunnerAdditionalArguments="-class Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests" ${{ parameters.runtimeSourceProperties }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/TestTemplateTests.binlog displayName: 🟣 Run Test Templates Tests From 537aa4779996a347381d79e2c0b1157db26fefdf Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 26 Sep 2025 15:44:03 -0700 Subject: [PATCH 4/4] Added job for TestTemplatesCG to run only on official builds. Cleaned up invocation of populateInternalRuntimeVariables. Fixed Arcade template folder name paths. Minor formatting changes. --- .vsts-ci.yml | 15 ++++++++++ .../dotnet-format-integration.yml | 18 ++++++++--- eng/pipelines/templates/jobs/sdk-build.yml | 30 ++++++++----------- .../templates/jobs/sdk-job-matrix.yml | 1 - 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index cdee394e48f5..82dded316160 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -102,6 +102,7 @@ extends: oneESCompat: templateFolderName: templates-official publishTaskPrefix: 1ES. + populateInternalRuntimeVariables: true runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) locBranch: release/10.0.1xx # WORKAROUND: BinSkim requires the folder exist prior to scanning. @@ -141,6 +142,14 @@ extends: _SignType: real dependsOn: Official_windows_x64 downloadManifestMsiPackages: true + ### TestTemplatesCG ### + # Note: This job is only used to allow the test templates to be built locally on the agent as opposed to Helix. + # The tests acquire the templates' PackageReferences from NuGet, which allows them to be scanned by CG (component governance). + # CG is only ran internally, so this job makes sense to only run alongside of the official jobs. + - categoryName: TestTemplatesCG + testProjects: $(Build.SourcesDirectory)/test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj + testRunnerAdditionalArguments: -class Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests + publishXunitResults: true ############### LINUX ############### - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self @@ -153,6 +162,7 @@ extends: oneESCompat: templateFolderName: templates-official publishTaskPrefix: 1ES. + populateInternalRuntimeVariables: true runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}: timeoutInMinutes: 90 @@ -235,6 +245,7 @@ extends: oneESCompat: templateFolderName: templates-official publishTaskPrefix: 1ES. + populateInternalRuntimeVariables: true runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}: timeoutInMinutes: 90 @@ -260,6 +271,8 @@ extends: vmImage: macOS-latest os: macOS helixTargetQueue: osx.13.arm64 + populateInternalRuntimeVariables: true + runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) macOSJobParameterSets: - categoryName: TestBuild targetArchitecture: arm64 @@ -270,7 +283,9 @@ extends: - template: /eng/dotnet-format/dotnet-format-integration.yml@self parameters: oneESCompat: + templateFolderName: templates-official publishTaskPrefix: 1ES. + populateInternalRuntimeVariables: true runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) ############### PUBLISH STAGE ############### diff --git a/eng/dotnet-format/dotnet-format-integration.yml b/eng/dotnet-format/dotnet-format-integration.yml index 1fa8bbb8ad9f..25c17948bf59 100644 --- a/eng/dotnet-format/dotnet-format-integration.yml +++ b/eng/dotnet-format/dotnet-format-integration.yml @@ -2,6 +2,7 @@ parameters: - name: oneESCompat type: object default: + templateFolderName: templates publishTaskPrefix: '' - name: TestArguments @@ -63,6 +64,9 @@ parameters: - name: runtimeSourceProperties type: string default: '' +- name: populateInternalRuntimeVariables + type: boolean + default: false jobs: - job: Formatting_Check @@ -77,8 +81,11 @@ jobs: os: windows timeoutInMinutes: ${{ parameters.timeoutInMinutes }} steps: - - template: /eng/common/templates/steps/enable-internal-runtimes.yml - - template: /eng/common/templates/steps/enable-internal-sources.yml + - ${{ if eq(parameters.populateInternalRuntimeVariables, true) }}: + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-runtimes.yml - script: .\restore.cmd ${{ parameters.runtimeSourceProperties }} displayName: 🟣 Restore dependencies - script: | @@ -106,8 +113,11 @@ jobs: os: windows timeoutInMinutes: ${{ parameters.timeoutInMinutes }} steps: - - template: /eng/common/templates/steps/enable-internal-runtimes.yml - - template: /eng/common/templates/steps/enable-internal-sources.yml + - ${{ if eq(parameters.populateInternalRuntimeVariables, true) }}: + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-runtimes.yml - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'prepare' -runtimeSourceProperties '${{ parameters.runtimeSourceProperties }}' displayName: 🟣 Prepare ${{ testArgs._repoName }} for formatting diff --git a/eng/pipelines/templates/jobs/sdk-build.yml b/eng/pipelines/templates/jobs/sdk-build.yml index 49df93cd2b54..a9fc3537bfcb 100644 --- a/eng/pipelines/templates/jobs/sdk-build.yml +++ b/eng/pipelines/templates/jobs/sdk-build.yml @@ -12,7 +12,9 @@ parameters: testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj publishRetryConfig: false publishXunitResults: false + testRunnerAdditionalArguments: '' enableSbom: true + populateInternalRuntimeVariables: false timeoutInMinutes: 150 ### ENV VARS ### testFullMSBuild: false @@ -71,11 +73,11 @@ jobs: targetPath: $(Build.SourcesDirectory)/eng/BuildConfiguration artifactName: BuildConfiguration - # Populate internal runtime variables. - - template: /eng/common/templates/steps/enable-internal-sources.yml - parameters: - legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) - - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - ${{ if eq(parameters.populateInternalRuntimeVariables, true) }}: + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.downloadManifestMsiPackages, true) }}: - task: DownloadBuildArtifacts@1 @@ -84,7 +86,7 @@ jobs: downloadPath: $(Build.SourcesDirectory)/artifacts/downloaded-manifest-msi-packages itemPattern: '**/*Manifest-*.Msi.*.nupkg' checkDownloadedFiles: true - displayName: Download Manifest msi packages + displayName: 🟣 Download Manifest MSI Packages ############### BUILDING ############### - ${{ if eq(parameters.pool.os, 'windows') }}: @@ -131,30 +133,20 @@ jobs: OPENSSL_ENABLE_SHA1_SIGNATURES: 1 ############### TESTING ############### - - ${{ if eq(parameters.pool.os, 'windows') }}: - - ${{ if eq(parameters.targetArchitecture, 'x64') }}: - - powershell: eng/common/build.ps1 - -restore -test -ci -prepareMachine -nativeToolsOnMachine - -configuration $(buildConfiguration) - /p:Projects="$(Build.SourcesDirectory)/test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj" - /p:TestRunnerAdditionalArguments="-class Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests" - ${{ parameters.runtimeSourceProperties }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(buildConfiguration)/TestTemplateTests.binlog - displayName: 🟣 Run Test Templates Tests - condition: succeeded() - - ${{ if eq(parameters.runTests, true) }}: - ${{ if eq(parameters.runAoTTests, true) }}: # For the reason this is here, see: https://github.com/dotnet/sdk/issues/22655 - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update workingDirectory: $(Build.SourcesDirectory)/artifacts/bin displayName: 🟣 Install wasm-tools Workload + # For the /p:Projects syntax for PowerShell, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1146466335 - ${{ if eq(parameters.pool.os, 'windows') }}: - powershell: eng/common/build.ps1 -restore -test -ci -prepareMachine -nativeToolsOnMachine -configuration $(buildConfiguration) /p:Projects=\`"${{ replace(parameters.testProjects, ';', '`;') }}\`" + /p:TestRunnerAdditionalArguments="${{ parameters.testRunnerAdditionalArguments }}" /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.runtimeSourceProperties }} /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }} @@ -167,6 +159,7 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) RunAoTTests: ${{ parameters.runAoTTests }} TestFullMSBuild: ${{ parameters.testFullMSBuild }} + - ${{ else }}: # For the /p:Projects syntax for Bash, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034 # The /p:CustomHelixTargetQueue syntax is: @ @@ -175,6 +168,7 @@ jobs: -restore -test -ci -prepareMachine -configuration $(buildConfiguration) '/p:Projects="${{ parameters.testProjects }}"' + /p:TestRunnerAdditionalArguments="${{ parameters.testRunnerAdditionalArguments }}" /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:TargetRid=${{ parameters.runtimeIdentifier }} ${{ parameters.osProperties }} diff --git a/eng/pipelines/templates/jobs/sdk-job-matrix.yml b/eng/pipelines/templates/jobs/sdk-job-matrix.yml index 8e9000f3c6cc..870f20d8f88b 100644 --- a/eng/pipelines/templates/jobs/sdk-job-matrix.yml +++ b/eng/pipelines/templates/jobs/sdk-job-matrix.yml @@ -31,7 +31,6 @@ parameters: container: azureLinux30Amd64 helixTargetContainer: $(helixTargetContainerPrefix)ubuntu-24.04-helix-amd64 osProperties: /p:OSName=linux /p:BuildSdkDeb=true - runTests: true # Helix is hanging on this job using the container. See: https://github.com/dotnet/dnceng/issues/6000 disableJob: true - categoryName: TemplateEngine