Windows release build cleanup and PyManager support (#236)

Author: zooba

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
       - id: check-merge-conflict
       - id: check-toml
       - id: check-yaml
-        exclude: windows-release/azure-pipelines.yml
+        exclude: windows-release/(azure-pipelines|msi-steps).yml
       - id: debug-statements
       - id: end-of-file-fixer
       - id: forbid-submodules

pyproject.toml

@@ -16,5 +16,6 @@ exclude = [
   "^tests/test_release_tag.py$",
   "^tests/test_run_release.py$",
   "^tests/test_sbom.py$",
+  "^windows-release/merge-and-upload.py$",
   "^windows-release/purge.py$",
 ]

sbom.py

@@ -696,7 +696,7 @@ def create_sbom_for_windows_artifact(
     artifact_path: str, cpython_source_dir: Path | str
 ) -> SBOM:
     artifact_name = os.path.basename(artifact_path)
-    if m := re.match(pat := r"^python-([0-9abrc.]+)(?:-|\.exe|\.zip)", artifact_name):
+    if m := re.match(pat := r"^python-([0-9abrc.]+)t?(?:-|\.exe|\.zip)", artifact_name):
         cpython_version = m.group(1)
     else:
         raise ValueError(f"Invalid {artifact_name=}, expected {pat!r}")
@@ -731,8 +731,8 @@ def create_sbom_for_windows_artifact(
     sbom_cpython_package_spdx_id = spdx_id("SPDXRef-PACKAGE-cpython")
 
     # The Windows embed artifacts don't contain pip/ensurepip,
-    # but the MSI artifacts do. Add pip for MSI installers.
-    if artifact_name.endswith(".exe"):
+    # but the others do.
+    if "-embed" not in artifact_name:
 
         # Find the pip wheel in ensurepip in the source code
         for pathname in os.listdir(cpython_source_dir / "Lib/ensurepip/_bundled"):

windows-release/azure-pipelines.yml

@@ -11,6 +11,7 @@ parameters:
   - 'Yhg1s'
   - 'pablogsal'
   - 'ambv'
+  - 'zooba'
   - '(Other)'
 - name: GitRemote_Other
   displayName: "If Other, specify Git remote"
@@ -39,7 +40,7 @@ parameters:
 - name: SigningDescription
   displayName: "Signature description"
   type: string
-  default: 'Built: $(Build.BuildNumber)'
+  default: '(default)'
 - name: DoGPG
   displayName: "Include GPG signatures (3.13 and earlier)"
   type: boolean
@@ -52,14 +53,6 @@ parameters:
   displayName: "Publish ARM64 build (3.11 and later)"
   type: boolean
   default: true
-# Because there is no ARM64 Tcl/Tk pre-3.11, we need a separate option
-# to keep those builds working when the files are going to be absent.
-# Eventually when we stop releasing anything that old, we can drop this
-# argument (and make it implicitly always 'true')
-- name: ARM64TclTk
-  displayName: "Use Tcl/Tk for ARM64 (3.11 and later)"
-  type: boolean
-  default: true
 - name: DoPGO
   displayName: "Run PGO"
   type: boolean
@@ -88,15 +81,32 @@ parameters:
   displayName: "Produce EXE/MSI installer"
   type: boolean
   default: true
+- name: DoPyManager
+  displayName: "Produce PyManager package"
+  type: boolean
+  default: true
 - name: BuildToPublish
-  displayName: "Build number to publish (0 to skip)"
-  type: number
-  default: '0'
+  displayName: "Republish a build (select in Resources)"
+  type: string
+  default: current
+  values: ['current', 'build_to_publish']
+- name: BuildToPackage
+  displayName: "Repackage and publish a build (select in Resources)"
+  type: string
+  default: current
+  values: ['current', 'build_to_package']
 - name: SignNuget
   displayName: "Enable Nuget signing (not recommended right now)"
   type: boolean
   default: false
 
+resources:
+  pipelines:
+  - pipeline: build_to_publish
+    source: 'Windows-Release'
+  - pipeline: build_to_package
+    source: 'Windows-Release'
+
 variables:
   ${{ if ne(parameters.GitRemote, '(Other)') }}:
     GitRemote: ${{ parameters.GitRemote }}
@@ -105,22 +115,12 @@ variables:
   SourceTag: ${{ parameters.SourceTag }}
   ${{ if ne(parameters.SourceCommit, 'empty') }}:
     SourceCommit: ${{ parameters.SourceCommit }}
-  ${{ else }}:
-    SourceCommit: ''
-  ${{ if ne(parameters.SigningCertificate, 'Unsigned') }}:
-    SigningCertificate: ${{ parameters.SigningCertificate }}
-  SigningDescription: ${{ parameters.SigningDescription }}
   ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}:
     IsRealSigned: true
   ${{ else }}:
     IsRealSigned: false
-  DoFreethreaded: ${{ parameters.DoFreethreaded }}
-  DoLayout: ${{ parameters.DoLayout }}
-  DoMSIX: ${{ parameters.DoMSIX }}
-  DoNuget: ${{ parameters.DoNuget }}
-  DoEmbed: ${{ parameters.DoEmbed }}
-  DoMSI: ${{ parameters.DoMSI }}
-  DoPublish: ${{ parameters.DoPublish }}
+  ${{ if ne(parameters.SigningDescription, '(default)') }}:
+    SigningDescription: ${{ parameters.SigningDescription }}
   PublishARM64: ${{ parameters.DoARM64 }}
 # QUEUE TIME VARIABLES
 #  PyDotOrgUsername: ''
@@ -130,144 +130,133 @@ trigger: none
 pr: none
 
 stages:
-- ${{ if eq(parameters.BuildToPublish, '0') }}:
+- ${{ if and(eq(parameters.BuildToPublish, 'current'), eq(parameters.BuildToPackage, 'current')) }}:
   - stage: Build
     displayName: Build binaries
     jobs:
     - template: stage-build.yml
       parameters:
-        ARM64TclTk: ${{ parameters.ARM64TclTk }}
         DoFreethreaded: ${{ parameters.DoFreethreaded }}
         DoPGO: ${{ parameters.DoPGO }}
         DoPGOARM64: ${{ parameters.DoPGOARM64 }}
+        ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
+          ToBeSigned: true
 
   - stage: Sign
     displayName: Sign binaries
     dependsOn: Build
     jobs:
     - template: stage-sign.yml
       parameters:
-        ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
-          SigningCertificate: ${{ parameters.SigningCertificate }}
+        SigningCertificate: ${{ parameters.SigningCertificate }}
         DoFreethreaded: ${{ parameters.DoFreethreaded }}
 
+- ${{ if eq(parameters.BuildToPublish, 'current') }}:
   - stage: Layout
-    displayName: Generate layouts
-    dependsOn: Sign
+    ${{ if eq(parameters.BuildToPackage, 'current') }}:
+      displayName: Generate layouts
+      dependsOn: Sign
+    ${{ else }}:
+      displayName: Generate layouts from prior build
+      dependsOn: []
     jobs:
     - template: stage-layout-full.yml
       parameters:
-        ARM64TclTk: ${{ parameters.ARM64TclTk }}
+        BuildToPackage: ${{ parameters.BuildToPackage }}
         DoFreethreaded: ${{ parameters.DoFreethreaded }}
-    - template: stage-layout-embed.yml
-    - template: stage-layout-nuget.yml
+    - template: stage-layout-symbols.yml
       parameters:
+        BuildToPackage: ${{ parameters.BuildToPackage }}
         DoFreethreaded: ${{ parameters.DoFreethreaded }}
+    - ${{ if eq(parameters.DoEmbed, 'true') }}:
+      - template: stage-layout-embed.yml
+        parameters:
+          BuildToPackage: ${{ parameters.BuildToPackage }}
+    - ${{ if eq(parameters.DoNuget, 'true') }}:
+      - template: stage-layout-nuget.yml
+        parameters:
+          BuildToPackage: ${{ parameters.BuildToPackage }}
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
+    - ${{ if eq(parameters.DoMSIX, 'true') }}:
+      - template: stage-layout-msix.yml
+        parameters:
+          BuildToPackage: ${{ parameters.BuildToPackage }}
+    - ${{ if eq(parameters.DoPyManager, 'true') }}:
+      - template: stage-layout-pymanager.yml
+        parameters:
+          BuildToPackage: ${{ parameters.BuildToPackage }}
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
+          DoEmbed: ${{ parameters.DoEmbed }}
 
   - stage: Pack
     dependsOn: Layout
+    displayName: Pack
     jobs:
-    - template: stage-pack-nuget.yml
-      parameters:
-        ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
-          ${{ if eq(parameters.SignNuget, 'true') }}:
-            SigningCertificate: ${{ parameters.SigningCertificate }}
-        DoFreethreaded: ${{ parameters.DoFreethreaded }}
-
-  - stage: Test
-    dependsOn: Pack
-    jobs:
-    - template: stage-test-embed.yml
-    - template: stage-test-nuget.yml
-      parameters:
-        DoFreethreaded: ${{ parameters.DoFreethreaded }}
-
-  - ${{ if eq(parameters.DoMSIX, 'true') }}:
-    - stage: Layout_MSIX
-      displayName: Generate MSIX layouts
-      dependsOn: Sign
-      jobs:
-      - template: stage-layout-msix.yml
+    #- ${{ if eq(parameters.DoEmbed, 'true') }}:
+    #  - template: stage-pack-embed.yml
+    - ${{ if eq(parameters.DoMSI, 'true') }}:
+      - template: stage-msi.yml
         parameters:
-          ARM64TclTk: ${{ parameters.ARM64TclTk }}
-
-    - stage: Pack_MSIX
-      displayName: Package MSIX
-      dependsOn: Layout_MSIX
-      jobs:
+          BuildToPackage: ${{ parameters.BuildToPackage }}
+          DoARM64: ${{ parameters.DoARM64}}
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
+    - ${{ if eq(parameters.DoMSIX, 'true') }}:
       - template: stage-pack-msix.yml
         parameters:
           ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
             SigningCertificate: ${{ parameters.SigningCertificate }}
-
-  - ${{ if eq(parameters.DoMSI, 'true') }}:
-    - stage: Build_MSI
-      displayName: Build MSI installer
-      dependsOn: Sign
-      jobs:
-      - template: stage-msi.yml
+    - ${{ if eq(parameters.DoNuget, 'true') }}:
+      - template: stage-pack-nuget.yml
         parameters:
-          ARM64TclTk: ${{ parameters.ARM64TclTk }}
-          ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
-            SigningCertificate: ${{ parameters.SigningCertificate }}
-          DoARM64: ${{ parameters.DoARM64}}
+          ${{ if eq(parameters.SignNuget, 'true') }}:
+            ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
+              SigningCertificate: ${{ parameters.SigningCertificate }}
           DoFreethreaded: ${{ parameters.DoFreethreaded }}
-
-    - stage: Test_MSI
-      displayName: Test MSI installer
-      dependsOn: Build_MSI
-      jobs:
-      - template: stage-test-msi.yml
+    - ${{ if eq(parameters.DoPyManager, 'true') }}:
+      - template: stage-pack-pymanager.yml
         parameters:
           DoFreethreaded: ${{ parameters.DoFreethreaded }}
+          DoEmbed: ${{ parameters.DoEmbed }}
 
-  - ${{ if eq(parameters.DoPublish, 'true') }}:
+  - stage: Test
+    dependsOn: Pack
+    jobs:
+    - ${{ if eq(parameters.DoEmbed, 'true') }}:
+      - template: stage-test-embed.yml
     - ${{ if eq(parameters.DoMSI, 'true') }}:
-      - stage: PublishPyDotOrg
-        displayName: Publish to python.org
-        dependsOn: ['Test_MSI', 'Test']
-        jobs:
-        - template: stage-publish-pythonorg.yml
-          parameters:
-            IncludeGPG: ${{ parameters.DoGPG }}
-
+      - template: stage-test-msi.yml
+        parameters:
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
     - ${{ if eq(parameters.DoNuget, 'true') }}:
-      - stage: PublishNuget
-        displayName: Publish to nuget.org
-        ${{ if eq(parameters.DoMSI, 'true') }}:
-          dependsOn: ['Test_MSI', 'Test']
-        ${{ else }}:
-          dependsOn: 'Test'
-        jobs:
-        - template: stage-publish-nugetorg.yml
-
-    - ${{ if eq(parameters.DoMSIX, 'true') }}:
-      - stage: PublishStore
-        displayName: Publish to Store
-        ${{ if eq(parameters.DoMSI, 'true') }}:
-          dependsOn: ['Test_MSI', 'Pack_MSIX']
-        ${{ else }}:
-          dependsOn: 'Pack_MSIX'
-        jobs:
-        - template: stage-publish-store.yml
+      - template: stage-test-nuget.yml
+        parameters:
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
+    - ${{ if eq(parameters.DoPyManager, 'true') }}:
+      - template: stage-test-pymanager.yml
+        parameters:
+          DoEmbed: ${{ parameters.DoEmbed }}
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
 
-- ${{ else }}:
-  - stage: PublishExisting
-    displayName: Publish existing build
-    dependsOn: []
+- ${{ if eq(parameters.DoPublish, 'true') }}:
+  - stage: Publish
+    displayName: Publish
+    dependsOn:
+    - ${{ if eq(parameters.BuildToPublish, 'current') }}:
+      - Test
     jobs:
+    - ${{ if eq(parameters.DoPyManager, 'true') }}:
+      - template: stage-publish-pymanager.yml
+        parameters:
+          BuildToPublish: ${{ parameters.BuildToPublish }}
+          DoEmbed: ${{ parameters.DoEmbed }}
+          DoFreethreaded: ${{ parameters.DoFreethreaded }}
     - ${{ if eq(parameters.DoMSI, 'true') }}:
       - template: stage-publish-pythonorg.yml
         parameters:
           BuildToPublish: ${{ parameters.BuildToPublish }}
+          DoEmbed: ${{ parameters.DoEmbed }}
           IncludeGPG: ${{ parameters.DoGPG }}
-
     - ${{ if eq(parameters.DoNuget, 'true') }}:
       - template: stage-publish-nugetorg.yml
         parameters:
           BuildToPublish: ${{ parameters.BuildToPublish }}
-
-    - ${{ if eq(parameters.DoMSIX, 'true') }}:
-      - template: stage-publish-store.yml
-        parameters:
-          BuildToPublish: ${{ parameters.BuildToPublish }}