Merge main into release/6.2 (#214)

**Explanation**: This updates `release/6.2` branch of the generator used
when building 6.2 Swift 6.2 toolchain with recent changes. Most notably:
* Embedded Swift SDK for WASI within existing Swift SDK for WASI
artifact bundles, thanks to
swiftlang/swift-package-manager#8638
* Bump to Swift 5.9 in `// swift-tools-version`
* Bump to generating with Swift 6.1 release tag by default (as opposed
to 6.0 previously)
* Support for Debian 11 and Debian 12 together with support for
downloading and unpacking Debian packages as dependencies.
* Additional test suite additions that cover these changes.

**Scope**: Isolated to cross-compilation and only impacts existing
Ubuntu toolchain job that generates Swift SDK for WASI.
**Risk**: Low, most of the changes LoC-wise were incubated on the `main`
branch for a month.
**Testing**: Expanded test suite, corresponding toolchain builds are
passing
https://ci.swift.org/job/oss-swift-pr-test-crosscompile-wasm-ubuntu-20_04/
**Issue**: rdar://151475169
**Reviewers**: @kateinoigakukun @euanh

---------

Co-authored-by: Jesse L. Zamora <[email protected]>
Co-authored-by: Euan Harris <[email protected]>

Author: 3 people

.swift-format

@@ -11,7 +11,7 @@
   "lineBreakBeforeControlFlowKeywords" : false,
   "lineBreakBeforeEachArgument" : true,
   "lineBreakBeforeEachGenericRequirement" : false,
-  "lineLength" : 100,
+  "lineLength" : 120,
   "maximumBlankLines" : 1,
   "multiElementCollectionTrailingCommas" : true,
   "noAssignmentInExpressions" : {

CONTRIBUTING.md

@@ -58,29 +58,6 @@ We require that your commit messages match our template. The easiest way to do t
 
     git config commit.template Utilities/git.commit.template
 
-### Run `./Utilities/soundness.sh`
-
-The scripts directory contains a [`soundness.sh` script](https://github.com/apple/swift-sdk-generator/blob/main/Utilities/soundness.sh) 
-that enforces additional checks, like license headers and formatting style.
-
-Please make sure to run `./Utilities/soundness.sh` before pushing a change upstream, otherwise it is likely the PR validation will fail
-on minor changes such as formatting issues.
-
-For frequent contributors, we recommend adding the script as a [git pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks), which you can do via executing the following command in the project root directory: 
-
-```bash
-cat << EOF > .git/hooks/pre-push
-
-if [[ -f "Utilities/soundness.sh" ]]; then
-  Utilities/soundness.sh
-fi
-EOF
-```
-
-Which makes the script execute, and only allow the `git push` to complete if the check has passed.
-
-In the case of formatting issues, you can then `git add` the formatting changes, and attempt the push again. 
-
 ## How to contribute your work
 
-Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review.
+Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review.

Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 5.8
+// swift-tools-version: 5.9
 // The swift-tools-version declares the minimum version of Swift required to build this package.
 
 import PackageDescription

README.md

@@ -47,14 +47,14 @@ The generator also allows cross-compiling between any Linux distributions offici
 | macOS (arm64)  | ✅ macOS 13.0+            | ❌                         |
 | macOS (x86_64) | ✅ macOS 13.0+[^1]        | ❌                         |
 | Ubuntu         | ✅ 20.04+                 | ✅ 20.04+                  |
-| RHEL           | ✅ Fedora 39[^2], UBI 9   | ✅ UBI 9                   |
+| Debian         | ✅ 11, 12[^2]             | ✅ 11, 12[^2]              |
+| RHEL           | ✅ Fedora 39, UBI 9       | ✅ Fedora 39, UBI 9[^3]    |
 | Amazon Linux 2 | ✅ Supported              | ✅ Supported[^3]           |
-| Debian 12      | ✅ Supported[^2]          | ✅ Supported[^2][^3]       |
 
 [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built
 from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also
 need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`).
-[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms.
+[^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible.
 [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138).
 
 ## How to use it

Sources/GeneratorCLI/GeneratorCLI.swift

@@ -114,7 +114,7 @@ extension GeneratorCLI {
     var swiftBranch: String? = nil
 
     @Option(help: "Version of Swift to supply in the bundle.")
-    var swiftVersion = "6.0.3-RELEASE"
+    var swiftVersion = "6.1-RELEASE"
 
     @Option(
       help: """
@@ -199,7 +199,8 @@ extension GeneratorCLI {
 
     @Option(
       help: """
-        Linux distribution to use if the target platform is Linux. Available options: `ubuntu`, `rhel`. Default is `ubuntu`.
+        Linux distribution to use if the target platform is Linux.
+        - Available options: `ubuntu`, `debian`, `rhel`. Default is `ubuntu`.
         """,
       transform: LinuxDistribution.Name.init(nameString:)
     )
@@ -208,8 +209,9 @@ extension GeneratorCLI {
     @Option(
       help: """
         Version of the Linux distribution used as a target platform.
-        Available options for Ubuntu: `20.04`, `22.04` (default when `--linux-distribution-name` is `ubuntu`), `24.04`.
-        Available options for RHEL: `ubi9` (default when `--linux-distribution-name` is `rhel`).
+        - Available options for Ubuntu: `20.04`, `22.04` (default when `--distribution-name` is `ubuntu`), `24.04`.
+        - Available options for Debian: `11`, `12` (default when `--distribution-name` is `debian`).
+        - Available options for RHEL: `ubi9` (default when `--distribution-name` is `rhel`).
         """
     )
     var linuxDistributionVersion: String?
@@ -239,6 +241,8 @@ extension GeneratorCLI {
         linuxDistributionDefaultVersion = "ubi9"
       case .ubuntu:
         linuxDistributionDefaultVersion = "22.04"
+      case .debian:
+        linuxDistributionDefaultVersion = "12"
       }
       let linuxDistributionVersion =
         self.linuxDistributionVersion ?? linuxDistributionDefaultVersion

Sources/Helpers/Vendor/QueryEngine/CacheKey.swift

@@ -182,4 +182,3 @@ extension Array: CacheKey where Element == FilePath.Component {
     map(\.string).joined(separator: "\n").hash(with: &hashFunction)
   }
 }
-

Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift

@@ -54,12 +54,12 @@ struct DownloadableArtifacts: Sendable {
 
     if hostTriple.os == .linux {
       // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts
-      let linuxArchSuffix =
+      let hostArchSuffix =
         hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : ""
       self.hostSwift = .init(
         remoteURL: versions.swiftDownloadURL(
-          subdirectory: "amazonlinux2\(linuxArchSuffix)",
-          platform: "amazonlinux2\(linuxArchSuffix)",
+          subdirectory: "amazonlinux2\(hostArchSuffix)",
+          platform: "amazonlinux2\(hostArchSuffix)",
           fileExtension: "tar.gz"
         ),
         localPath: paths.artifactsCachePath
@@ -97,7 +97,9 @@ struct DownloadableArtifacts: Sendable {
     self.targetSwift = .init(
       remoteURL: versions.swiftDownloadURL(),
       localPath: paths.artifactsCachePath
-        .appending("target_swift_\(versions.swiftVersion)_\(targetTriple.triple).tar.gz"),
+        .appending(
+          "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz"
+        ),
       isPrebuilt: true
     )
   }

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift

@@ -22,6 +22,7 @@ extension SwiftSDKGenerator {
     logger.info("Launching a container to extract the Swift SDK for the target triple...")
     try await withDockerContainer(fromImage: baseDockerImage) { containerID in
       try await inTemporaryDirectory { generator, _ in
+        let sdkLibPath = sdkDirPath.appending("lib")
         let sdkUsrPath = sdkDirPath.appending("usr")
         try await generator.createDirectoryIfNeeded(at: sdkUsrPath)
         try await generator.copyFromDockerContainer(
@@ -60,6 +61,15 @@ extension SwiftSDKGenerator {
             to: sdkUsrLib64Path
           )
           try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64")
+        } else if case let containerLib64 = FilePath("/lib64"),
+          try await generator.doesPathExist(containerLib64, inContainer: containerID)
+        {
+          let sdkLib64Path = sdkDirPath.appending("lib64")
+          try await generator.copyFromDockerContainer(
+            id: containerID,
+            from: containerLib64,
+            to: sdkLib64Path
+          )
         }
 
         let sdkUsrLibPath = sdkUsrPath.appending("lib")
@@ -72,13 +82,26 @@ extension SwiftSDKGenerator {
         // architecture-specific directories:
         //   https://wiki.ubuntu.com/MultiarchSpec
         // But not in all containers, so don't fail if it does not exist.
-        if case .ubuntu = targetDistribution {
-          subpaths += [("\(targetTriple.archName)-linux-gnu", false)]
+        if targetDistribution.name == .ubuntu || targetDistribution.name == .debian {
+          var archSubpath = "\(targetTriple.archName)-linux-gnu"
 
-          // Custom subpath for armv7
+          // armv7 with Debian uses a custom subpath for armhf
           if targetTriple.archName == "armv7" {
-            subpaths += [("arm-linux-gnueabihf", false)]
+            archSubpath = "arm-linux-gnueabihf"
           }
+
+          // Copy /lib/<archSubpath> for Debian 11
+          if case let .debian(debian) = targetDistribution, debian == .bullseye {
+            try await generator.createDirectoryIfNeeded(at: sdkLibPath)
+            try await generator.copyFromDockerContainer(
+              id: containerID,
+              from: FilePath("/lib").appending(archSubpath),
+              to: sdkLibPath.appending(archSubpath),
+              failIfNotExists: false
+            )
+          }
+
+          subpaths += [(archSubpath, false)]
         }
 
         for (subpath, failIfNotExists) in subpaths {
@@ -89,7 +112,11 @@ extension SwiftSDKGenerator {
             failIfNotExists: failIfNotExists
           )
         }
-        try await generator.createSymlink(at: sdkDirPath.appending("lib"), pointingTo: "usr/lib")
+
+        // Symlink if we do not have a /lib directory in the SDK
+        if await !generator.doesFileExist(at: sdkLibPath) {
+          try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib")
+        }
 
         // Look for 32-bit libraries to remove from RHEL-based distros
         // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up