From 978a6dc612e3ccbeb5222df24971f0ba70518aff Mon Sep 17 00:00:00 2001 From: wtgodbe Date: Mon, 28 Aug 2017 13:14:52 -0700 Subject: [PATCH] Proof-of-concept of using new Publish infra in coreclr --- ...tNet-CoreClr-Trusted-Linux-Crossbuild.json | 40 ++++++- .../DotNet-CoreClr-Trusted-Linux.json | 40 ++++++- buildpipeline/DotNet-CoreClr-Trusted-Mac.json | 40 ++++++- .../DotNet-CoreClr-Trusted-Windows-x86.json | 22 +++- .../DotNet-CoreClr-Trusted-Windows.json | 22 +++- .../DotNet-Trusted-Publish-Symbols.json | 2 +- buildpipeline/DotNet-Trusted-Publish.json | 101 ++++++++++++++++-- .../Dotnet-CoreClr-Trusted-BuildTests.json | 43 +------- config.json | 32 +++++- init-tools.cmd | 8 +- init-tools.msbuild | 22 +++- init-tools.sh | 6 +- publish-packages.cmd | 3 + publish-packages.sh | 4 +- src/publish.proj | 87 ++++++++++----- src/syncAzure.proj | 10 +- sync.cmd | 6 +- tests/dir.props | 2 +- 18 files changed, 380 insertions(+), 110 deletions(-) diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Linux-Crossbuild.json b/buildpipeline/DotNet-CoreClr-Trusted-Linux-Crossbuild.json index 11f8deda8a00..f6e108a20d84 100644 --- a/buildpipeline/DotNet-CoreClr-Trusted-Linux-Crossbuild.json +++ b/buildpipeline/DotNet-CoreClr-Trusted-Linux-Crossbuild.json @@ -184,7 +184,7 @@ "enabled": true, "continueOnError": false, "alwaysRun": false, - "displayName": "Run publish-packages.sh", + "displayName": "Publish packages", "timeoutInMinutes": 0, "task": { "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", @@ -193,7 +193,43 @@ }, "inputs": { "filename": "docker", - "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(Label) -distroRid=$(Rid) -- /p:OverwriteOnPublish=true", + "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishPackages -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/pkg /p:PublishFlatContainer=false /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish symbol packages", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "docker", + "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishSymbols -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/symbolpkg /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish test native binaries", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "docker", + "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishTestNativeBins -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/TestNativeBins/$(Rid) /p:OverwriteOnPublish=true", "workingFolder": "", "failOnStandardError": "false" } diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Linux.json b/buildpipeline/DotNet-CoreClr-Trusted-Linux.json index c237b54ef4c9..9ee71268fad2 100644 --- a/buildpipeline/DotNet-CoreClr-Trusted-Linux.json +++ b/buildpipeline/DotNet-CoreClr-Trusted-Linux.json @@ -148,7 +148,7 @@ "enabled": true, "continueOnError": false, "alwaysRun": false, - "displayName": "Run publish-packages.sh", + "displayName": "Publish packages", "timeoutInMinutes": 0, "task": { "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", @@ -157,7 +157,43 @@ }, "inputs": { "filename": "docker", - "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Platform) -Container=$(Label) -distroRid=$(Rid) -- /p:OverwriteOnPublish=true", + "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Platform) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishPackages -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/pkg /p:PublishFlatContainer=false /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish symbol packages", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "docker", + "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Platform) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishSymbols -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/symbolpkg /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish test native binaries", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "docker", + "arguments": "run --rm $(DockerCommonRunArgs) ./publish-packages.sh -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Platform) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishTestNativeBins -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/TestNativeBins/$(Rid) /p:OverwriteOnPublish=true", "workingFolder": "", "failOnStandardError": "false" } diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Mac.json b/buildpipeline/DotNet-CoreClr-Trusted-Mac.json index b96ebb5bbfe2..e9a6f5419efa 100644 --- a/buildpipeline/DotNet-CoreClr-Trusted-Mac.json +++ b/buildpipeline/DotNet-CoreClr-Trusted-Mac.json @@ -76,7 +76,7 @@ "enabled": true, "continueOnError": false, "alwaysRun": false, - "displayName": "Run $(Agent.BuildDirectory)/s/publish-packages.sh", + "displayName": "Publish packages", "timeoutInMinutes": 0, "task": { "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", @@ -85,7 +85,43 @@ }, "inputs": { "filename": "$(Agent.BuildDirectory)/s/publish-packages.sh", - "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(Label) -distroRid=$(Rid) -- /p:OverwriteOnPublish=true", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishPackages -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/pkg /p:PublishFlatContainer=false /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish symbol packages", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "$(Agent.BuildDirectory)/s/publish-packages.sh", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishSymbols -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/symbolpkg /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish test native binaries", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "$(Agent.BuildDirectory)/s/publish-packages.sh", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildType=$(PB_BuildType) -BuildArch=$(Architecture) -Container=$(PB_ContainerName) -distroRid=$(Rid) -PublishSymbols -PublishTestNativeBins -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/TestNativeBins/$(Rid) /p:OverwriteOnPublish=true", "workingFolder": "", "failOnStandardError": "false" } diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Windows-x86.json b/buildpipeline/DotNet-CoreClr-Trusted-Windows-x86.json index f941575fe1d2..1ec087b0d6dc 100644 --- a/buildpipeline/DotNet-CoreClr-Trusted-Windows-x86.json +++ b/buildpipeline/DotNet-CoreClr-Trusted-Windows-x86.json @@ -141,7 +141,7 @@ "enabled": true, "continueOnError": false, "alwaysRun": false, - "displayName": "Run publish-packages.cmd", + "displayName": "Publish packages", "timeoutInMinutes": 0, "task": { "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", @@ -150,7 +150,25 @@ }, "inputs": { "filename": "publish-packages.cmd", - "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildArch=$(Architecture) -BuildType=$(PB_BuildType) -- /p:ContainerName=$(Label) /p:OverwriteOnPublish=true", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildArch=$(Architecture) -BuildType=$(PB_BuildType) -Container=$(PB_ContainerName) -PublishPackages -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/pkg /p:PublishFlatContainer=false /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish symbol packages", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "publish-packages.cmd", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildArch=$(Architecture) -BuildType=$(PB_BuildType) -Container=$(PB_ContainerName) -PublishSymbols -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/symbolpkg /p:OverwriteOnPublish=true", "workingFolder": "", "failOnStandardError": "false" } diff --git a/buildpipeline/DotNet-CoreClr-Trusted-Windows.json b/buildpipeline/DotNet-CoreClr-Trusted-Windows.json index 6997893c17bf..f80b5fef8d19 100644 --- a/buildpipeline/DotNet-CoreClr-Trusted-Windows.json +++ b/buildpipeline/DotNet-CoreClr-Trusted-Windows.json @@ -141,7 +141,7 @@ "enabled": true, "continueOnError": false, "alwaysRun": false, - "displayName": "Run publish-packages.cmd", + "displayName": "Publish packages", "timeoutInMinutes": 0, "task": { "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", @@ -150,7 +150,25 @@ }, "inputs": { "filename": "publish-packages.cmd", - "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildArch=$(Architecture) -BuildType=$(PB_BuildType) -- /p:ContainerName=$(Label) /p:OverwriteOnPublish=true", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildArch=$(Architecture) -BuildType=$(PB_BuildType) -Container=$(PB_ContainerName) -PublishPackages -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/pkg /p:PublishFlatContainer=false /p:OverwriteOnPublish=true", + "workingFolder": "", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Publish symbol packages", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "publish-packages.cmd", + "arguments": "-AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -BuildArch=$(Architecture) -BuildType=$(PB_BuildType) -Container=$(PB_ContainerName) -PublishSymbols -- /p:RelativePath=$(PB_BlobNamePrefix)/$(PB_BuildType)/symbolpkg /p:OverwriteOnPublish=true", "workingFolder": "", "failOnStandardError": "false" } diff --git a/buildpipeline/DotNet-Trusted-Publish-Symbols.json b/buildpipeline/DotNet-Trusted-Publish-Symbols.json index a85b3f96e924..e6c6970f6d7c 100644 --- a/buildpipeline/DotNet-Trusted-Publish-Symbols.json +++ b/buildpipeline/DotNet-Trusted-Publish-Symbols.json @@ -178,7 +178,7 @@ "allowOverride": true }, "AzureContainerSymbolPackageGlob": { - "value": "symbolpkg\\*.nupkg", + "value": "symbolpkg\\*\\*\\*.nupkg", "allowOverride": true }, "PB_AzureContainerSymbolPackageGlob": { diff --git a/buildpipeline/DotNet-Trusted-Publish.json b/buildpipeline/DotNet-Trusted-Publish.json index 93c1bbbe737c..63fade85ca44 100644 --- a/buildpipeline/DotNet-Trusted-Publish.json +++ b/buildpipeline/DotNet-Trusted-Publish.json @@ -82,19 +82,17 @@ "enabled": true, "continueOnError": false, "alwaysRun": false, - "displayName": "sync -ab", + "displayName": "Sync packages", "timeoutInMinutes": 0, "task": { - "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", "versionSpec": "1.*", "definitionType": "task" }, "inputs": { - "scriptType": "inlineScript", - "scriptName": "", - "arguments": "$(CloudDropAccountName) $(CloudDropAccessToken) $(Label)", - "inlineScript": "param($account, $token, $container)\nif ($env:UseLegacyBuildScripts -eq \"true\")\n{\n .\\sync.cmd /ab /p:CloudDropAccountName=$account /p:CloudDropAccessToken=$token /p:ContainerName=$container\n}\nelse\n{\n# .\\sync.cmd -ab \"-AzureAccount=$account\" \"-AzureToken=$token\" \"-Container=$container\"\n .\\init-tools.cmd\n msbuild src\\syncAzure.proj /p:CloudDropAccountName=$account /p:CloudDropAccessToken=$token /p:ContainerName=$container\n}", - "workingFolder": "$(Pipeline.SourcesDirectory)", + "filename": "sync.cmd", + "arguments": "-ab -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(PB_ContainerName) -BlobNamePrefix=$(PB_BlobNamePrefix)", + "workingFolder": "", "failOnStandardError": "false" } }, @@ -215,6 +213,82 @@ "failOnStandardError": "true" } }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Packages -> Azure", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "msbuild", + "arguments": "publish.proj /t:PublishPackages /p:CloudDropAccessToken=$(AzureBlobFeedAccessToken) /p:CloudDropAccountName=$(AzureBlobFeedAccountName) /p:ContainerName=$(AzureBlobFeedContainerName) /p:__PublishPackages=true /p:OverwriteOnPublish=true /p:PackagesPattern=../packages/AzureTransfer/Release/pkg/*/*/*.nupkg /p:RelativePath=packages /p:__BuildType=$(ConfigurationGroup) /p:OfficialPublish=true /p:PublishFlatContainer=false /fileloggerparameters:Verbosity=diag;LogFile=publishpkg.log", + "workingFolder": "src", + "failOnStandardError": "false" + } + }, + { + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Symbol Packages -> Azure", + "timeoutInMinutes": 0, + "task": { + "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "filename": "msbuild", + "arguments": "publish.proj /t:PublishSymbolPackages /p:CloudDropAccessToken=$(AzureBlobFeedAccessToken) /p:CloudDropAccountName=$(AzureBlobFeedAccountName) /p:ContainerName=$(AzureBlobFeedContainerName) /p:__PublishSymbols=true /p:OverwriteOnPublish=true /p:SymbolsPackagesPattern=../packages/AzureTransfer/Release/symbolpkg/*/*/*.nupkg /p:RelativePath=symbols /p:__BuildType=$(ConfigurationGroup) /p:OfficialPublish=true /p:PublishFlatContainer=true /fileloggerparameters:Verbosity=diag;LogFile=publishpkg.log", + "workingFolder": "src", + "failOnStandardError": "false" + } + }, + { + "enabled": false, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Create VSTS auth NuGet.Config", + "timeoutInMinutes": 0, + "task": { + "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "scriptType": "inlineScript", + "scriptName": "", + "arguments": "$(VstsAuthedNuGetConfigPath) $(VstsFeedUrl) $(VstsPat)", + "inlineScript": "param($path, $url, $pat)\n\nSet-Content $path @\"\n\n\n \n \n \n \n \n \n \n \n \n\n\"@", + "workingFolder": "", + "failOnStandardError": "true" + } + }, + { + "enabled": false, + "continueOnError": false, + "alwaysRun": false, + "displayName": "packages -> VSTS", + "timeoutInMinutes": 0, + "task": { + "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "scriptType": "inlineScript", + "scriptName": "", + "arguments": "", + "inlineScript": "if ($env:ConfigurationGroup -ne \"Release\") { exit }\n& $env:CustomNuGetPath push $env:Pipeline.SourcesDirectory\\packages\\AzureTransfer\\$env:ConfigurationGroup\\$env:AzureContainerPackageGlob placeholderapikey -Source vsts-dotnet-core -ConfigFile $env:VstsAuthedNuGetConfigPath -Timeout 3600", + "workingFolder": "", + "failOnStandardError": "true" + } + }, { "enabled": true, "continueOnError": false, @@ -398,6 +472,19 @@ "allowOverride": true, "isSecret": true }, + "AzureBlobFeedAccountName": { + "value": "dotnetfeed", + "allowOverride": true + }, + "AzureBlobFeedAccessToken": { + "value": null, + "allowOverride": true, + "isSecret": true + }, + "AzureBlobFeedContainerName": { + "value": "dotnet-core", + "allowOverride": true + }, "OfficialBuildId": { "value": "$(Build.BuildNumber)", "allowOverride": true diff --git a/buildpipeline/Dotnet-CoreClr-Trusted-BuildTests.json b/buildpipeline/Dotnet-CoreClr-Trusted-BuildTests.json index ebb1f6e3f003..2cb18d8936c0 100644 --- a/buildpipeline/Dotnet-CoreClr-Trusted-BuildTests.json +++ b/buildpipeline/Dotnet-CoreClr-Trusted-BuildTests.json @@ -50,43 +50,7 @@ }, "inputs": { "filename": "sync.cmd", - "arguments": "-n -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(Label)-$(Rid) -RuntimeId=$(Rid)", - "workingFolder": "", - "failOnStandardError": "false" - } - }, - { - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "Sync packages", - "timeoutInMinutes": 0, - "task": { - "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "versionSpec": "1.*", - "definitionType": "task" - }, - "inputs": { - "filename": "sync.cmd", - "arguments": "-ab -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(Label) -BlobNamePrefix=$(PB_BuildType)/pkg/Microsoft", - "workingFolder": "", - "failOnStandardError": "false" - } - }, - { - "enabled": true, - "continueOnError": false, - "alwaysRun": false, - "displayName": "Sync runtime packages", - "timeoutInMinutes": 0, - "task": { - "id": "d9bafed4-0b18-4f58-968d-86655b4d2ce9", - "versionSpec": "1.*", - "definitionType": "task" - }, - "inputs": { - "filename": "sync.cmd", - "arguments": "-ab -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(Label) -BlobNamePrefix=$(PB_BuildType)/pkg/runtime", + "arguments": "-ab -AzureAccount=$(CloudDropAccountName) -AzureToken=$(CloudDropAccessToken) -Container=$(PB_ContainerName) -RuntimeId=$(Rid) -BlobNamePrefix=$(PB_BlobNamePrefix)/$(PB_BuildType)/TestNativeBins/$(Rid) -- /p:DownloadDirectory=$(Build.SourcesDirectory)/packages/TestNativeBins/$(Rid)", "workingFolder": "", "failOnStandardError": "false" } @@ -122,7 +86,7 @@ }, "inputs": { "filename": "build-test.cmd", - "arguments": "$(PB_BuildType) $(Architecture) buildagainstpackages runtimeid $(Rid) $(TargetsNonWindowsArg)$(CrossgenArg)-OfficialBuildId=$(OfficialBuildId) -OverwriteCoreClrPackageVersion -Priority=$(Priority)", + "arguments": "$(PB_BuildType) $(Architecture) buildagainstpackages runtimeid $(Rid) $(TargetsNonWindowsArg)$(CrossgenArg)-OfficialBuildId=$(OfficialBuildId) -OverwriteCoreClrPackageVersion -Priority=$(Priority) -- /p:IntermediateAzureFeed=$(IntermediateAzureFeed)", "workingFolder": "", "failOnStandardError": "false" } @@ -280,6 +244,9 @@ "TestContainerSuffix": { "value": "linux" }, + "IntermediateAzureFeed": { + "value": "https://dotnetbuildoutput.blob.core.windows.net/$(PB_ContainerName)/$(PB_BlobNamePrefix)$(PB_BuildType)/pkg/index.json", + }, "RuntimeIDArg": { "value": "runtimeid linux-x64" }, diff --git a/config.json b/config.json index 8f3a58b8ec61..5322daa48e66 100644 --- a/config.json +++ b/config.json @@ -12,6 +12,18 @@ "values": [], "defaultValue": "" }, + "__PublishPackages": { + "description": "Specifies whether to publish product packages.", + "valueType": "property", + "values": [ true, false ], + "defaultValue": true + }, + "__PublishSymbols": { + "description": "Specifies whether to publish symbol packages.", + "valueType": "property", + "values": [ true, false ], + "defaultValue": true + }, "BuildNumberMajor": { "description": "Product build major number.", "valueType": "property", @@ -24,7 +36,7 @@ "values": [], "defaultValue": "" }, - "__Container": { + "ContainerName": { "description": "Container name for Azure upload.", "valueType": "property", "values": [], @@ -540,7 +552,7 @@ "container": { "description": "Container name to download from in Azure Blob storage.", "settings": { - "__Container": "default" + "ContainerName": "default" } }, "blobNamePrefix": { @@ -598,7 +610,7 @@ "container": { "description": "Container name to upload into in Azure Blob storage.", "settings": { - "__Container": "default" + "ContainerName": "default" } }, "buildArch": { @@ -630,7 +642,19 @@ "settings": { "PublishTestNativeBins": "default" } - } + }, + "PublishPackages": { + "description": "Publishes product packages.", + "settings": { + "__PublishPackages": "default" + } + }, + "PublishSymbols": { + "description": "Publishes symbol packages.", + "settings": { + "__PublishSymbols": "default" + } + }, }, "defaultValues": { "toolName": "msbuild", diff --git a/init-tools.cmd b/init-tools.cmd index 5aa2ef5ecfd0..235f64532ede 100644 --- a/init-tools.cmd +++ b/init-tools.cmd @@ -54,8 +54,8 @@ if NOT exist "%DOTNET_LOCAL_PATH%" ( if exist "%BUILD_TOOLS_PATH%" goto :afterbuildtoolsrestore echo Restoring BuildTools version %BUILDTOOLS_VERSION%... -echo Running: "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages %PACKAGES_DIR% --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% >> "%INIT_TOOLS_LOG%" -call "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages %PACKAGES_DIR% --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% >> "%INIT_TOOLS_LOG%" +echo Running: "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages %PACKAGES_DIR% --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" +call "%DOTNET_CMD%" restore "%INIT_TOOLS_RESTORE_PROJECT%" --no-cache --packages %PACKAGES_DIR% --source "%BUILDTOOLS_SOURCE%" /p:BuildToolsPackageVersion=%BUILDTOOLS_VERSION% /p:ToolsDir=%TOOLRUNTIME_DIR% >> "%INIT_TOOLS_LOG%" if NOT exist "%BUILD_TOOLS_PATH%init-tools.cmd" ( echo ERROR: Could not restore build tools correctly. 1>&2 goto :error @@ -75,7 +75,7 @@ if not [%INIT_TOOLS_ERRORLEVEL%]==[0] ( :: Create semaphore file echo Done initializing tools. if NOT exist "%BUILD_TOOLS_SEMAPHORE_DIR%" ( - mkdir "%BUILD_TOOLS_SEMAPHORE_DIR%" + mkdir "%BUILD_TOOLS_SEMAPHORE_DIR%" ) echo Init-Tools.cmd completed for BuildTools Version: %BUILDTOOLS_VERSION% > "%BUILD_TOOLS_SEMAPHORE%" exit /b 0 @@ -83,4 +83,4 @@ exit /b 0 :error echo Please check the detailed log that follows. 1>&2 type "%INIT_TOOLS_LOG%" 1>&2 -exit /b 1 +exit /b 1 \ No newline at end of file diff --git a/init-tools.msbuild b/init-tools.msbuild index ff2a9c3cb4e1..c8eaf702c7a6 100644 --- a/init-tools.msbuild +++ b/init-tools.msbuild @@ -3,14 +3,30 @@ netcoreapp1.0 false true - $(MSBuildThisFileDirectory)Tools/$(BuildToolsPackageVersion) + $(ToolsDir)/$(BuildToolsPackageVersion) + 1.0.0-prerelease-02111-01 + microsoft.dotnet.build.tasks.feed + $(PackageTargetFallback);netframework - + $(BuildToolsPackageVersion) - + 1.0.4-prerelease + + $(FeedTasksVersion) + + + + + + + + + \ No newline at end of file diff --git a/init-tools.sh b/init-tools.sh index f7f5cba7906c..00b096199e82 100755 --- a/init-tools.sh +++ b/init-tools.sh @@ -119,8 +119,8 @@ if [ ! -e $__INIT_TOOLS_DONE_MARKER ]; then else if [ ! -e $__BUILD_TOOLS_PATH ]; then echo "Restoring BuildTools version $__BUILD_TOOLS_PACKAGE_VERSION..." - echo "Running: $__DOTNET_CMD restore \"$__INIT_TOOLS_RESTORE_PROJECT\" --no-cache --packages $__PACKAGES_DIR --source $__BUILDTOOLS_SOURCE /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION" >> $__init_tools_log - $__DOTNET_CMD restore "$__INIT_TOOLS_RESTORE_PROJECT" --no-cache --packages $__PACKAGES_DIR --source $__BUILDTOOLS_SOURCE /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION >> $__init_tools_log + echo "Running: $__DOTNET_CMD restore \"$__INIT_TOOLS_RESTORE_PROJECT\" --no-cache --packages $__PACKAGES_DIR --source $__BUILDTOOLS_SOURCE /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION /p:ToolsDir=$__TOOLRUNTIME_DIR" >> $__init_tools_log + $__DOTNET_CMD restore "$__INIT_TOOLS_RESTORE_PROJECT" --no-cache --packages $__PACKAGES_DIR --source $__BUILDTOOLS_SOURCE /p:BuildToolsPackageVersion=$__BUILD_TOOLS_PACKAGE_VERSION /p:ToolsDir=$__TOOLRUNTIME_DIR >> $__init_tools_log if [ ! -e "$__BUILD_TOOLS_PATH/init-tools.sh" ]; then echo "ERROR: Could not restore build tools correctly." 1>&2 display_error_message @@ -153,4 +153,4 @@ if [ ! -e $__INIT_TOOLS_DONE_MARKER ]; then echo "Done initializing tools." else echo "Tools are already initialized" -fi +fi \ No newline at end of file diff --git a/publish-packages.cmd b/publish-packages.cmd index d23ff2373fb1..3dd89ceecf4f 100644 --- a/publish-packages.cmd +++ b/publish-packages.cmd @@ -18,6 +18,9 @@ echo -AzureAccount="account name" echo -AzureToken="access token" echo -BuildType="Configuration" echo -BuildArch="Architecture" +echo For publishing to Azure, one of the following properties is required. +echo -PublishPackages Pass this switch to publish product packages +echo -PublishSymbols Pass this switch to publish symbol packages echo To specify the name of the container to publish into, use the following property: echo -Container="container name" echo Architecture can be x64, x86, arm, or arm64 diff --git a/publish-packages.sh b/publish-packages.sh index 0827a659440d..1d0f13b2961e 100755 --- a/publish-packages.sh +++ b/publish-packages.sh @@ -8,6 +8,9 @@ usage() echo " -AzureToken=\"access token\"" echo " -BuildType=\"Configuration\"" echo " -BuildArch=\"Architecture\"" + echo "For publishing to Azure, one of the following properties is required." + echo " -PublishPackages Pass this switch to publish product packages" + echo " -PublishSymbols Pass this switch to publish symbol packages" echo "To specify the name of the container to publish into, use the following property:" echo " -Container=\"container name\"" echo "To specify the OS you're building for, use the following property:" @@ -20,7 +23,6 @@ usage() working_tree_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" $working_tree_root/run.sh publish-packages -BuildOS $* -$working_tree_root/run.sh publish-packages -BuildOS -PublishTestNativeBins $* if [ $? -ne 0 ] then echo "ERROR: An error occurred while publishing packages; see $working_tree_root/publish-packages.log for more details. There may have been networking problems, so please try again in a few minutes." diff --git a/src/publish.proj b/src/publish.proj index c6c2149ba3c0..9def32ab0c50 100644 --- a/src/publish.proj +++ b/src/publish.proj @@ -1,41 +1,76 @@ - - - - - + + $(ToolsDir)Microsoft.Dotnet.Build.Tasks.Feed/net46/ + $(ToolsDir)microsoft.dotnet.build.tasks.feed/netstandard1.5/ + + + + + + $(PackagesBinDir)pkg\*.nupkg + $(OutputPath)\bin\** + $(PackagesBinDir)symbolpkg\*.nupkg + true + + + + $(ContainerName.Replace(".","-")) + $(CloudDropAccountName) + $(CloudDropAccessToken) + + + + + $(__BuildType)/pkg + https://$(AccountName).blob.core.windows.net/$(ContainerName)/$(RelativePath)/index.json + - + + - + + + + + + $(__BuildType)/symbolpkg + https://$(AccountName).blob.core.windows.net/$(ContainerName)/$(RelativePath)/index.json + - - $(__BuildType)/%(RecursiveDir)%(Filename)%(Extension) - $(__BuildType)/%(RecursiveDir)%(Filename)%(Extension) - + + - + - - $(PackagesBinDir)**\*.nupkg - $(OutputPath)\bin\** - - - + - coreclr-$(PreReleaseLabel)-$(BuildNumberMajor)-$(BuildNumberMinor) - coreclr-$(PreReleaseLabel)-$(BuildNumberMajor)-$(BuildNumberMinor)-$(__DistroRid)-$(__BuildArch) - $(__Container) - $(__Container)-$(__DistroRid)-$(__BuildArch) - $(ContainerName.Replace(".","-")) + $(__BuildType)/TestNativeBins + https://$(AccountName).blob.core.windows.net/$(ContainerName)/$(RelativePath)/index.json + + + + + - + + \ No newline at end of file diff --git a/src/syncAzure.proj b/src/syncAzure.proj index 14baa2622e2b..c5360d77f5a9 100644 --- a/src/syncAzure.proj +++ b/src/syncAzure.proj @@ -3,13 +3,9 @@ - coreclr-$(PreReleaseLabel) - $(ContainerNamePrefix)-$(BuildNumberMajor)-$(BuildNumberMinor) - $(__Container) $(ContainerName.Replace(".","-")) $(__BlobNamePrefix) - $(PackagesDir)AzureTransfer - $(PackagesDir)TestNativeBins\$(RuntimeId) + $(PackagesDir)AzureTransfer @@ -17,10 +13,10 @@ - + - + \ No newline at end of file diff --git a/sync.cmd b/sync.cmd index 218fb9de8e1f..949f1605f9d3 100644 --- a/sync.cmd +++ b/sync.cmd @@ -28,11 +28,7 @@ echo To download from a specific container, specify: echo -Container="container name" echo To download blobs starting with a specific prefix, specify: echo -BlobNamePrefix="Blob name prefix" -echo -n - Downloads test native binaries from the specified container -echo The following properties are required: -echo -AzureAccount="Account name" -echo -AzureToken="Access token" -echo -Container="Container name" +echo To specify which RID you are downloading binaries for (optional): echo -RuntimeId="RID" (Needs to match what's in the container) echo. echo. diff --git a/tests/dir.props b/tests/dir.props index fc32d87e38f3..c17c9eff96c2 100644 --- a/tests/dir.props +++ b/tests/dir.props @@ -89,7 +89,7 @@ - +