diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 9676702f28387f..d583af24d16e8c 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -25,6 +25,12 @@ "commands": [ "slngen" ] + }, + "microsoft.dotnet.helix.jobmonitor": { + "version": "11.0.0-beta.26256.1", + "commands": [ + "dotnet-helix-job-monitor" + ] } } } diff --git a/NuGet.config b/NuGet.config index 8d2d351914d014..6a926020b7209e 100644 --- a/NuGet.config +++ b/NuGet.config @@ -22,6 +22,8 @@ + + diff --git a/eng/Version.Details.props b/eng/Version.Details.props index afdfd5818acf34..c6e5969fe192df 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -5,32 +5,33 @@ This file should be imported by eng/Versions.props --> + + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 11.0.0-beta.26256.1 + 3.2.2-beta.26256.1 + 2.9.3-beta.26256.1 + 11.0.0-beta.26256.1 5.7.0-1.26211.102 5.7.0-1.26211.102 5.7.0-1.26211.102 11.0.100-preview.4.26211.102 11.0.100-preview.4.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 0.11.5-preview.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 11.0.0-beta.26211.102 - 3.2.2-beta.26211.102 - 2.9.3-beta.26211.102 - 11.0.0-beta.26211.102 5.7.0-1.26211.102 11.0.0-preview.4.26211.102 11.0.100-preview.4.26211.102 @@ -117,12 +118,7 @@ This file should be imported by eng/Versions.props - - $(MicrosoftCodeAnalysisPackageVersion) - $(MicrosoftCodeAnalysisAnalyzersPackageVersion) - $(MicrosoftCodeAnalysisCSharpPackageVersion) - $(MicrosoftCodeAnalysisNetAnalyzersPackageVersion) - $(MicrosoftDotNetApiCompatTaskPackageVersion) + $(MicrosoftDotNetArcadeSdkPackageVersion) $(MicrosoftDotNetBuildTasksArchivesPackageVersion) $(MicrosoftDotNetBuildTasksFeedPackageVersion) @@ -131,7 +127,6 @@ This file should be imported by eng/Versions.props $(MicrosoftDotNetBuildTasksTargetFrameworkPackageVersion) $(MicrosoftDotNetBuildTasksTemplatingPackageVersion) $(MicrosoftDotNetBuildTasksWorkloadsPackageVersion) - $(MicrosoftDotNetCecilPackageVersion) $(MicrosoftDotNetCodeAnalysisPackageVersion) $(MicrosoftDotNetGenAPIPackageVersion) $(MicrosoftDotNetGenFacadesPackageVersion) @@ -143,6 +138,13 @@ This file should be imported by eng/Versions.props $(MicrosoftDotNetXUnitAssertPackageVersion) $(MicrosoftDotNetXUnitConsoleRunnerPackageVersion) $(MicrosoftDotNetXUnitExtensionsPackageVersion) + + $(MicrosoftCodeAnalysisPackageVersion) + $(MicrosoftCodeAnalysisAnalyzersPackageVersion) + $(MicrosoftCodeAnalysisCSharpPackageVersion) + $(MicrosoftCodeAnalysisNetAnalyzersPackageVersion) + $(MicrosoftDotNetApiCompatTaskPackageVersion) + $(MicrosoftDotNetCecilPackageVersion) $(MicrosoftNetCompilersToolsetPackageVersion) $(MicrosoftNETSdkILPackageVersion) $(MicrosoftNETWorkloadEmscriptenCurrentManifest110100TransportPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ee7fa3d9101fb4..62c256d201986c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -51,77 +51,77 @@ - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad https://github.com/dotnet/runtime-assets @@ -299,9 +299,9 @@ https://github.com/dotnet/xharness 0668c80ec27851f3c7f1b3e4536110a1d39af587 - - https://github.com/dotnet/dotnet - 36afe73557f5f93cd7bc827cb644a3ff018eca0b + + https://dev.azure.com/dnceng/internal/_git/dotnet-arcade + 8f3edeb4bc5045b77e659f758af96c1921cc70ad https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml new file mode 100644 index 00000000000000..8e69ba8ff7a2d8 --- /dev/null +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -0,0 +1,235 @@ +parameters: +# Pool override. When empty the template selects a default azurelinux pool based on the team project. +- name: pool + type: object + default: {} + +# NuGet package id of the Helix job monitor tool. +- name: toolPackageId + type: string + default: Microsoft.DotNet.Helix.JobMonitor + +# Console command exposed by the installed tool package. +- name: toolCommand + type: string + default: dotnet-helix-job-monitor + +# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the +# default code path the version is taken from the consuming repo's .config/dotnet-tools.json. +- name: toolVersion + type: string + default: '' + +# Optional NuGet feed used as an additional source when installing the tool. Only honored +# when 'toolNupkgArtifactName' is set; in the default code path the tool is restored from +# the consuming repo's .config/dotnet-tools.json manifest and no extra feeds are consulted. +- name: toolSource + type: string + default: '' + +# Base URI for the Helix service (--helix-base-uri). +- name: helixBaseUri + type: string + default: https://helix.dot.net/ + +# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable. +- name: helixAccessToken + type: string + default: '' + +# Polling interval in seconds (--polling-interval-seconds). +- name: pollingIntervalSeconds + type: number + default: 30 + +# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes. +- name: timeoutInMinutes + type: number + default: 360 + +# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization. +# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty. +- name: organization + type: string + default: '' + +# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository. +# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty. +- name: repository + type: string + default: '' + +# Pull request number being built (--pr-number). Defaults to SYSTEM_PULLREQUEST_PULLREQUESTNUMBER +# when empty. +- name: prNumber + type: string + default: '' + +# Optional dependency list for the generated job. +- name: dependsOn + type: object + default: [] + +# Optional condition for the generated job. +- name: condition + type: string + default: '' + +# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool +# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into +# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is +# primarily intended for the Arcade repository itself, where the Helix job monitor tool is +# built in the same pipeline that runs this template. +# +# When this parameter is empty (the default), the consuming repository must declare the tool +# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template +# will check out the repo and run 'dotnet tool restore' to install the version pinned there. +- name: toolNupkgArtifactName + type: string + default: '' + +# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults +# to the standard Arcade non-shipping packages location for a Release build (relative to the +# pipeline artifact root, which is itself the build's 'artifacts' directory). +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + +jobs: +- job: HelixJobMonitor + displayName: Monitor Helix Jobs + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(length(parameters.dependsOn), 0) }}: + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64.open + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64 + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Helix Job Monitor artifact + inputs: + buildType: current + artifactName: ${{ parameters.toolNupkgArtifactName }} + itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg' + targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg + + - bash: | + set -euo pipefail + + toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool" + mkdir -p "$toolPath" + + packageId='${{ parameters.toolPackageId }}' + toolVersion='${{ parameters.toolVersion }}' + nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}' + nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath" + + if [ ! -d "$nupkgDir" ]; then + echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2 + exit 1 + fi + + nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1) + if [ -z "$nupkg" ]; then + echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2 + exit 1 + fi + + # Derive the version from the nupkg filename so the local package is selected + # deterministically instead of resolving against any other configured feed. + nupkgBase=$(basename "$nupkg" .nupkg) + derivedVersion="${nupkgBase#${packageId}.}" + if [ -z "$toolVersion" ]; then + toolVersion="$derivedVersion" + fi + + echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + + # Create a minimal NuGet.config that only references the local nupkg directory. + # This avoids conflicts with the repo's package source mapping which blocks --add-source. + toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config" + printf '\n\n \n \n \n \n\n' "$nupkgDir" > "$toolNugetConfig" + + pushd "$(Build.SourcesDirectory)" > /dev/null + ./eng/common/dotnet.sh tool install \ + --tool-path "$toolPath" "$packageId" \ + --version "$toolVersion" \ + --configfile "$toolNugetConfig" + + # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec. + toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1) + toolDll="${toolDll%.deps.json}.dll" + if [ ! -f "$toolDll" ]; then + echo "Could not find tool DLL in '$toolPath/.store'." >&2 + exit 1 + fi + + echo "Tool DLL: $toolDll" + echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll" + displayName: Install Helix Job Monitor + + - ${{ else }}: + - bash: ./eng/common/dotnet.sh tool restore + displayName: Restore Helix Job Monitor + + - bash: | + set -euo pipefail + + toolArgs=( + --helix-base-uri '${{ parameters.helixBaseUri }}' + --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' + --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 2))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. + --attempt '$(System.JobAttempt)' + --stage-name '$(System.StageName)' + ) + + organization='${{ parameters.organization }}' + repository='${{ parameters.repository }}' + prNumber='${{ parameters.prNumber }}' + + # Fall back to Azure DevOps-provided environment variables when the caller did not + # supply organization / repository / pr-number explicitly. BUILD_REPOSITORY_NAME is + # typically 'owner/repo' for GitHub-backed builds. + if [ -z "$organization" ] || [ -z "$repository" ]; then + buildRepoName="${BUILD_REPOSITORY_NAME:-}" + if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then + repoOwner="${buildRepoName%%/*}" + repoName="${buildRepoName#*/}" + if [ -z "$organization" ]; then organization="$repoOwner"; fi + if [ -z "$repository" ]; then repository="$repoName"; fi + fi + fi + + if [ -z "$prNumber" ]; then + prNumber="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}" + fi + + if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi + if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + if [ -n "$prNumber" ]; then toolArgs+=( --pr-number "$prNumber" ); fi + + if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then + # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet. + export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet" + ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}" + else + # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it + # through the manifest from the repo root. + pushd "$BUILD_SOURCESDIRECTORY" > /dev/null + trap 'popd > /dev/null' EXIT + ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}" + fi + displayName: Monitor Helix Jobs + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 99d6dfe82dde38..ff2dfdb4a5bf60 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -225,19 +225,13 @@ elseif(ILLUMOS) locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) elseif(HAIKU) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") - if ($ENV{CCC_CC} MATCHES ".*gcc.*") - set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin") - locate_toolchain_exec(gcc CMAKE_C_COMPILER) - locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) - else() - set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") - set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") - set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/cross-tools-x86_64") - endif() + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) # let CMake set up the correct search paths include(Platform/Haiku) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 6710ffb884bb23..c96f5018fe43a8 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -545,6 +545,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { + # keep this in sync with the VSWhereVersion in DefaultVersions.props $vswhereVersion = '3.1.7' } @@ -679,17 +680,7 @@ function GetNuGetPackageCachePath() { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject([string]$taskName) { - $toolsetDir = Split-Path (InitializeToolset) -Parent - $proj = Join-Path $toolsetDir "$taskName.proj" - if (Test-Path $proj) { - return $proj - } - # TODO: Remove this fallback once all supported versions use the new layout. - $legacyProj = Join-Path $toolsetDir "SdkTasks\$taskName.proj" - if (Test-Path $legacyProj) { - return $legacyProj - } - throw "Unable to find $taskName.proj in toolset at: $toolsetDir" + return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj" } function InitializeNativeTools() { @@ -726,18 +717,13 @@ function InitializeToolset() { $nugetCache = GetNuGetPackageCachePath $toolsetVersion = Read-ArcadeSdkVersion - $toolsetToolsDir = Join-Path $ToolsetDir $toolsetVersion - - # Check if the toolset has already been extracted - $toolsetBuildProj = $null - $buildProjPath = Join-Path $toolsetToolsDir 'Build.proj' - - if (Test-Path $buildProjPath) { - $toolsetBuildProj = $buildProjPath - } + $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" - if ($toolsetBuildProj -ne $null) { - return $global:_InitializeToolset = $toolsetBuildProj + if (Test-Path $toolsetLocationFile) { + $path = Get-Content $toolsetLocationFile -TotalCount 1 + if (Test-Path $path) { + return $global:_InitializeToolset = $path + } } if (-not $restore) { @@ -745,40 +731,21 @@ function InitializeToolset() { ExitWithExitCode 1 } - $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--prerelease", "--output", "$nugetCache") - if ($env:NUGET_CONFIG) { - $downloadArgs += "--configfile" - $downloadArgs += $env:NUGET_CONFIG - } - DotNet @downloadArgs + $buildTool = InitializeBuildTool - $packageDir = Join-Path $nugetCache (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) - $packageToolsetDir = Join-Path $packageDir 'toolset' - $packageToolsDir = Join-Path $packageDir 'tools' + $proj = Join-Path $ToolsetDir 'restore.proj' + $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' } - # TODO: Remove the tools/ check once all supported versions have the toolset folder. - if (!(Test-Path $packageToolsetDir) -and !(Test-Path $packageToolsDir)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Arcade SDK package does not contain a toolset or tools folder: $packageDir" - ExitWithExitCode 3 - } + '' | Set-Content $proj - New-Item -ItemType Directory -Path $toolsetToolsDir -Force | Out-Null + MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - # Copy toolset if present at the package root (new layout), otherwise fall back to tools - if (Test-Path $packageToolsetDir) { - Copy-Item -Path "$packageToolsetDir\*" -Destination $toolsetToolsDir -Recurse -Force - } else { - # TODO: Remove this fallback once all supported versions have the toolset folder. - Copy-Item -Path "$packageToolsDir\*" -Destination $toolsetToolsDir -Recurse -Force + $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 + if (!(Test-Path $path)) { + throw "Invalid toolset path: $path" } - if (Test-Path $buildProjPath) { - $toolsetBuildProj = $buildProjPath - } else { - throw "Unable to find Build.proj in toolset at: $toolsetToolsDir" - } - - return $global:_InitializeToolset = $toolsetBuildProj + return $global:_InitializeToolset = $path } function ExitWithExitCode([int] $exitCode) { @@ -839,40 +806,6 @@ function MSBuild() { MSBuild-Core @args } -# -# Executes a dotnet command with arguments passed to the function. -# Terminates the script if the command fails. -# -function DotNet() { - $dotnetRoot = InitializeDotNetCli -install:$restore - $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - - $cmdArgs = "" - foreach ($arg in $args) { - if ($null -ne $arg -and $arg.Trim() -ne "") { - if ($arg.EndsWith('\')) { - $arg = $arg + "\" - } - $cmdArgs += " `"$arg`"" - } - } - - $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs" - - $exitCode = Exec-Process $dotnetPath $cmdArgs - - if ($exitCode -ne 0) { - Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red - - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { - Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed." - ExitWithExitCode 0 - } else { - ExitWithExitCode $exitCode - } - } -} - # # Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. # The arguments are automatically quoted. @@ -909,7 +842,7 @@ function MSBuild-Core() { $cmdArgs += ' /p:TreatWarningsAsErrors=false' } - if ($warnAsError -and $warnNotAsError) { + if ($warnNotAsError) { $cmdArgs += " /warnnotaserror:$warnNotAsError /p:AdditionalWarningsNotAsErrors=$warnNotAsError" } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index d2339eb21d5945..a6e0ed594fda17 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -407,18 +407,15 @@ function InitializeToolset { ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion - local toolset_tools_dir="$toolset_dir/$toolset_version" + local toolset_location_file="$toolset_dir/$toolset_version.txt" - # Check if the toolset has already been extracted - local toolset_build_proj="" - if [[ -a "$toolset_tools_dir/Build.proj" ]]; then - toolset_build_proj="$toolset_tools_dir/Build.proj" - fi - - if [[ -n "$toolset_build_proj" ]]; then - # return value - _InitializeToolset="$toolset_build_proj" - return + if [[ -a "$toolset_location_file" ]]; then + local path=`cat "$toolset_location_file"` + if [[ -a "$path" ]]; then + # return value + _InitializeToolset="$path" + return + fi fi if [[ "$restore" != true ]]; then @@ -426,34 +423,20 @@ function InitializeToolset { ExitWithExitCode 2 fi - local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--prerelease" "--output" "$_GetNuGetPackageCachePath") - if [[ -n "${NUGET_CONFIG:-}" ]]; then - download_args+=("--configfile" "$NUGET_CONFIG") - fi - DotNet "${download_args[@]}" - - local package_dir="$_GetNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" + local proj="$toolset_dir/restore.proj" - # TODO: Remove the tools/ check once all supported versions have the toolset folder. - if [[ ! -d "$package_dir/toolset" && ! -d "$package_dir/tools" ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset or tools folder: $package_dir" - ExitWithExitCode 3 + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:$log_dir/ToolsetRestore.binlog" fi - mkdir -p "$toolset_tools_dir" + echo '' > "$proj" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" - # Copy toolset if present at the package root (new layout), otherwise fall back to tools - if [[ -d "$package_dir/toolset" ]]; then - cp -r "$package_dir/toolset/." "$toolset_tools_dir" - else - # TODO: Remove this fallback once all supported versions have the toolset folder. - cp -r "$package_dir/tools/." "$toolset_tools_dir" - fi + local toolset_build_proj=`cat "$toolset_location_file"` - if [[ -a "$toolset_tools_dir/Build.proj" ]]; then - toolset_build_proj="$toolset_tools_dir/Build.proj" - else - Write-PipelineTelemetryError -category 'Build' "Unable to find Build.proj in toolset at: $toolset_tools_dir" + if [[ ! -a "$toolset_build_proj" ]]; then + Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -475,26 +458,6 @@ function StopProcesses { return 0 } -function DotNet { - InitializeDotNetCli $restore - - local dotnet_path="$_InitializeDotNetCli/dotnet" - - export ARCADE_BUILD_TOOL_COMMAND="$dotnet_path $@" - - "$dotnet_path" "$@" || { - local exit_code=$? - echo "dotnet command failed with exit code $exit_code. Check errors above." - - if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$from_vmr" != true ]]; then - Write-PipelineSetResult -result "Failed" -message "dotnet command execution failed." - ExitWithExitCode 0 - else - ExitWithExitCode $exit_code - fi - } -} - function MSBuild { local args=( "$@" ) if [[ "$pipelines_log" == true ]]; then @@ -571,7 +534,7 @@ function MSBuild-Core { fi local warnnotaserror_switch="" - if [[ -n "$warn_not_as_error" && "$warn_as_error" == true ]]; then + if [[ -n "$warn_not_as_error" ]]; then warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" fi @@ -592,22 +555,8 @@ function GetDarc { # Returns a full path to an Arcade SDK task project file. function GetSdkTaskProject { - local taskName=$1 - local toolsetDir - toolsetDir="$(dirname "$_InitializeToolset")" - local proj="$toolsetDir/$taskName.proj" - if [[ -a "$proj" ]]; then - echo "$proj" - return - fi - # TODO: Remove this fallback once all supported versions use the new layout. - local legacyProj="$toolsetDir/SdkTasks/$taskName.proj" - if [[ -a "$legacyProj" ]]; then - echo "$legacyProj" - return - fi - Write-PipelineTelemetryError -category 'Build' "Unable to find $taskName.proj in toolset at: $toolsetDir" - ExitWithExitCode 3 + taskName=$1 + echo "$(dirname $_InitializeToolset)/SdkTasks/$taskName.proj" } ResolvePath "${BASH_SOURCE[0]}" diff --git a/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml b/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml index f88692190cc221..7f22df6c95545d 100644 --- a/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml +++ b/eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml @@ -13,6 +13,7 @@ parameters: nativeAotTest: false runtimeFlavor: 'mono' runtimeVariant: 'monointerpreter' + useHelixMonitor: false llvmAotStepContainer: '' scenarios: - normal @@ -105,6 +106,7 @@ steps: helixSource: $(_HelixSource) ${{ if ne(parameters.readyToRun, true) }}: helixType: 'test/functional/cli/' + useHelixMonitor: ${{ parameters.useHelixMonitor }} helixQueues: ${{ parameters.helixQueues }} # This tests whether an array is empty diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index 9e66956a3c2547..e959eba200b4cf 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -12,6 +12,7 @@ parameters: helixSource: '' helixQueues: '' helixType: '' + useHelixMonitor: false msbuildParallelism: '/maxcpucount' scenarios: '' timeoutPerTestCollectionInMinutes: '' @@ -42,7 +43,16 @@ steps: - template: send-to-helix-inner-step.yml parameters: osGroup: ${{ parameters.osGroup }} - sendParams: ${{ parameters.helixProjectArguments }} ${{ parameters.msbuildParallelism }} /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog /p:TargetArchitecture=${{ parameters.archType }} /p:TargetOS=${{ parameters.osGroup }} /p:TargetOSSubgroup=${{ parameters.osSubgroup }} /p:Configuration=${{ parameters.buildConfig }} ${{ parameters.extraHelixArguments }} + sendParams: > + ${{ parameters.helixProjectArguments }} + ${{ parameters.msbuildParallelism }} + /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog + /p:TargetArchitecture=${{ parameters.archType }} + /p:TargetOS=${{ parameters.osGroup }} + /p:TargetOSSubgroup=${{ parameters.osSubgroup }} + /p:Configuration=${{ parameters.buildConfig }} + /p:EnableHelixJobMonitor=${{ parameters.useHelixMonitor }} + ${{ parameters.extraHelixArguments }} condition: and(succeeded(), ${{ parameters.condition }}) shouldContinueOnError: ${{ parameters.shouldContinueOnError }} displayName: ${{ parameters.displayName }} diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 03b3cf1ebaffca..ab8f1f02c6e324 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -12,39 +12,41 @@ parameters: condition: always() extraHelixArguments: '' shouldContinueOnError: false + useHelixMonitor: false scenarios: '' SuperPmiCollect: '' SuperPmiCollectionType: '' SuperPmiCollectionName: '' steps: - - script: $(_msbuildCommand) $(_warnAsErrorParamHelixOverride) -restore - $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - /p:TargetArchitecture=${{ parameters.archType }} - /p:TargetRuntimeIdentifier=${{ parameters.targetRid }} - /p:Configuration=${{ parameters.buildConfig }} - /p:TargetOS=${{ parameters.osGroup }} - /p:MonoForceInterpreter=${{ parameters.interpreter }} - /p:TestScope=${{ parameters.testScope }} - /p:TestRunNamePrefixSuffix=${{ parameters.testRunNamePrefixSuffix }} - /p:HelixBuild=$(Build.BuildNumber) - ${{ parameters.extraHelixArguments }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog - displayName: Send to Helix - condition: and(succeeded(), ${{ parameters.condition }}) - continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops - _Scenarios: ${{ join(',', parameters.scenarios) }} # Pass scenarios to MSBuild as env var to avoid need of escaping comma separated list - _SuperPmiCollect: ${{ parameters.SuperPmiCollect }} - _SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} - _SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} +- script: $(_msbuildCommand) $(_warnAsErrorParamHelixOverride) -restore + $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj + /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} + /p:TargetArchitecture=${{ parameters.archType }} + /p:TargetRuntimeIdentifier=${{ parameters.targetRid }} + /p:Configuration=${{ parameters.buildConfig }} + /p:TargetOS=${{ parameters.osGroup }} + /p:MonoForceInterpreter=${{ parameters.interpreter }} + /p:TestScope=${{ parameters.testScope }} + /p:TestRunNamePrefixSuffix=${{ parameters.testRunNamePrefixSuffix }} + /p:HelixBuild=$(Build.BuildNumber) + /p:EnableHelixJobMonitor=${{ parameters.useHelixMonitor }} + ${{ parameters.extraHelixArguments }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + displayName: Send to Helix + condition: and(succeeded(), ${{ parameters.condition }}) + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops + _Scenarios: ${{ join(',', parameters.scenarios) }} # Pass scenarios to MSBuild as env var to avoid need of escaping comma separated list + _SuperPmiCollect: ${{ parameters.SuperPmiCollect }} + _SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} + _SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: ${{ replace(lower(join('+', parameters.helixQueues)), '.open', '') }} - Creator: '' - ${{ if eq(variables['System.TeamProject'], 'public') }}: - HelixTargetQueues: ${{ join('+', parameters.helixQueues) }} - Creator: ${{ parameters.creator }} + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: ${{ replace(lower(join('+', parameters.helixQueues)), '.open', '') }} + Creator: '' + ${{ if eq(variables['System.TeamProject'], 'public') }}: + HelixTargetQueues: ${{ join('+', parameters.helixQueues) }} + Creator: ${{ parameters.creator }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index a1eb3eec103e5d..646ef4dee8bfd8 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -797,6 +797,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + useHelixMonitor: true condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -1093,6 +1094,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_$(_BuildConfig) + useHelixMonitor: true condition: >- or( eq(variables['librariesContainsChange'], true), @@ -1135,6 +1137,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + useHelixMonitor: true condition: >- or( eq(variables['librariesContainsChange'], true), @@ -1179,6 +1182,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_$(_BuildConfig) + useHelixMonitor: true extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true condition: >- or( @@ -1223,6 +1227,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + useHelixMonitor: true extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true condition: >- or( @@ -1267,6 +1272,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + useHelixMonitor: true extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true condition: >- or( @@ -1310,6 +1316,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_$(_BuildConfig) + useHelixMonitor: true condition: >- or( eq(variables['librariesContainsChange'], true), @@ -1352,6 +1359,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + useHelixMonitor: true extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true condition: >- or( @@ -1394,6 +1402,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + useHelixMonitor: true extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true condition: >- or( @@ -1517,6 +1526,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: NET481_$(_BuildConfig) + useHelixMonitor: true extraHelixArguments: /p:BuildTargetFramework=net481 condition: >- or( @@ -1675,6 +1685,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_Release + useHelixMonitor: true extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -1711,6 +1722,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_Release + useHelixMonitor: true extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: @@ -1749,6 +1761,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_Release + useHelixMonitor: true extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # @@ -1788,6 +1801,7 @@ extends: creator: dotnet-bot llvmAotStepContainer: linux_x64_llvmaot testRunNamePrefixSuffix: Mono_Release + useHelixMonitor: true extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -2039,6 +2053,7 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_Interpreter_$(_BuildConfig) + useHelixMonitor: true interpreter: true condition: >- or( @@ -2071,8 +2086,14 @@ extends: parameters: creator: dotnet-bot testRunNamePrefixSuffix: Mono_Minijit_$(_BuildConfig) + useHelixMonitor: true condition: >- or( eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) + + - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + helixAccessToken: $(HelixApiAccessToken) + timeoutInMinutes: 540 diff --git a/global.json b/global.json index 2a02cd5952b45b..714d2487b678d7 100644 --- a/global.json +++ b/global.json @@ -13,9 +13,9 @@ "dotnet": "11.0.100-preview.3.26170.106" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26211.102", - "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26211.102", - "Microsoft.DotNet.SharedFramework.Sdk": "11.0.0-beta.26211.102", + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26256.1", + "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26256.1", + "Microsoft.DotNet.SharedFramework.Sdk": "11.0.0-beta.26256.1", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "11.0.0-preview.4.26211.102"