Dynamically determine the swift compiler version. (#8707)

Prebuilts for macros and the automated downloading of SwiftSDKs need to
know the version of the compiler so we can fetch compatible binaries.
The swiftc compiler has a --print-target-info options which dumps out a
JSON structure that contains the compilerVersion. We already use info in
this structure to determine the hostTriple for the UserToolchain.

This adds the swiftCompilerVersion to UserToolchain that uses a couple
of regex's to pull out the Swift compiler version. This is then used by
the prebuilts feature instead of our current hardcodeing of the swift
toolchain version.

This also turns the prebuilts feature on by default which was supposed
to be done in the last update.

Author: dschaefer2

Sources/CoreCommands/Options.swift

@@ -198,7 +198,7 @@ public struct CachingOptions: ParsableArguments {
     @Flag(name: .customLong("experimental-prebuilts"),
           inversion: .prefixedEnableDisable,
           help: "Whether to use prebuilt swift-syntax libraries for macros.")
-    public var usePrebuilts: Bool = false
+    public var usePrebuilts: Bool = true
 
     /// Hidden option to override the prebuilts download location for testing
     @Option(

Sources/CoreCommands/SwiftCommandState.swift

@@ -47,6 +47,7 @@ import Bionic
 import class Basics.AsyncProcess
 import func TSCBasic.exec
 import class TSCBasic.FileLock
+import enum TSCBasic.JSON
 import protocol TSCBasic.OutputByteStream
 import enum TSCBasic.ProcessEnv
 import enum TSCBasic.ProcessLockError
@@ -287,6 +288,8 @@ public final class SwiftCommandState {
 
     private let hostTriple: Basics.Triple?
 
+    private let targetInfo: JSON?
+
     package var preferredBuildConfiguration = BuildConfiguration.debug
 
     /// Create an instance of this tool.
@@ -323,10 +326,12 @@ public final class SwiftCommandState {
         workspaceLoaderProvider: @escaping WorkspaceLoaderProvider,
         createPackagePath: Bool,
         hostTriple: Basics.Triple? = nil,
+        targetInfo: JSON? = nil,
         fileSystem: any FileSystem = localFileSystem,
         environment: Environment = .current
     ) throws {
         self.hostTriple = hostTriple
+        self.targetInfo = targetInfo
         self.fileSystem = fileSystem
         self.environment = environment
         // first, bootstrap the observability system
@@ -968,7 +973,11 @@ public final class SwiftCommandState {
         }
 
         return Result(catching: {
-            try UserToolchain(swiftSDK: swiftSDK, environment: self.environment, fileSystem: self.fileSystem)
+            try UserToolchain(
+                swiftSDK: swiftSDK,
+                environment: self.environment,
+                customTargetInfo: targetInfo,
+                fileSystem: self.fileSystem)
         })
     }()
 
@@ -983,6 +992,7 @@ public final class SwiftCommandState {
         return try UserToolchain(
             swiftSDK: hostSwiftSDK,
             environment: self.environment,
+            customTargetInfo: targetInfo,
             fileSystem: self.fileSystem
         )
     })

Sources/PackageModel/UserToolchain.swift

@@ -13,6 +13,7 @@
 import Basics
 import Foundation
 import TSCUtility
+import enum TSCBasic.JSON
 
 import class Basics.AsyncProcess
 
@@ -74,6 +75,9 @@ public final class UserToolchain: Toolchain {
 
     public let targetTriple: Basics.Triple
 
+    // A version string that can be used to identify the swift compiler version
+    public let swiftCompilerVersion: String?
+
     /// The list of CPU architectures to build for.
     public let architectures: [String]?
 
@@ -160,6 +164,75 @@ public final class UserToolchain: Toolchain {
         return try getTool(name, binDirectories: envSearchPaths, fileSystem: fileSystem)
     }
 
+    private static func getTargetInfo(swiftCompiler: AbsolutePath) throws -> JSON {
+        // Call the compiler to get the target info JSON.
+        let compilerOutput: String
+        do {
+            let result = try AsyncProcess.popen(args: swiftCompiler.pathString, "-print-target-info")
+            compilerOutput = try result.utf8Output().spm_chomp()
+        } catch {
+            throw InternalError(
+                "Failed to load target info (\(error.interpolationDescription))"
+            )
+        }
+        // Parse the compiler's JSON output.
+        do {
+            return try JSON(string: compilerOutput)
+        } catch {
+            throw InternalError(
+                "Failed to parse target info (\(error.interpolationDescription)).\nRaw compiler output: \(compilerOutput)"
+            )
+        }
+    }
+
+    private static func getHostTriple(targetInfo: JSON) throws -> Basics.Triple {
+        // Get the triple string from the target info.
+        let tripleString: String
+        do {
+            tripleString = try targetInfo.get("target").get("triple")
+        } catch {
+            throw InternalError(
+                "Target info does not contain a triple string (\(error.interpolationDescription)).\nTarget info: \(targetInfo)"
+            )
+        }
+
+        // Parse the triple string.
+        do {
+            return try Triple(tripleString)
+        } catch {
+            throw InternalError(
+                "Failed to parse triple string (\(error.interpolationDescription)).\nTriple string: \(tripleString)"
+            )
+        }
+    }
+
+    private static func computeSwiftCompilerVersion(targetInfo: JSON) -> String? {
+        // Get the compiler version from the target info
+        let compilerVersion: String
+        do {
+            compilerVersion = try targetInfo.get("compilerVersion")
+        } catch {
+            return nil
+        }
+
+        // Extract the swift version using regex from the description if available
+        do {
+            let regex = try Regex(#"\((swift(lang)?-[^ )]*)"#)
+            if let match = try regex.firstMatch(in: compilerVersion), match.count > 1, let substring = match[1].substring {
+                return String(substring)
+            }
+
+            let regex2 = try Regex(#"\(.*Swift (.*)[ )]"#)
+            if let match2 = try regex2.firstMatch(in: compilerVersion), match2.count > 1, let substring = match2[1].substring {
+                return "swift-\(substring)"
+            } else {
+                return nil
+            }
+        } catch {
+            return nil
+        }
+    }
+
     // MARK: - public API
 
     public static func determineLibrarian(
@@ -570,6 +643,7 @@ public final class UserToolchain: Toolchain {
         swiftSDK: SwiftSDK,
         environment: Environment = .current,
         searchStrategy: SearchStrategy = .default,
+        customTargetInfo: JSON? = nil,
         customLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation? = nil,
         customInstalledSwiftPMConfiguration: InstalledSwiftPMConfiguration? = nil,
         fileSystem: any FileSystem = localFileSystem
@@ -612,8 +686,14 @@ public final class UserToolchain: Toolchain {
                 default: InstalledSwiftPMConfiguration.default)
         }
 
-        // Use the triple from Swift SDK or compute the host triple using swiftc.
-        var triple = try swiftSDK.targetTriple ?? Triple.getHostTriple(usingSwiftCompiler: swiftCompilers.compile)
+        // targetInfo from the compiler
+        let targetInfo = try customTargetInfo ?? Self.getTargetInfo(swiftCompiler: swiftCompilers.compile)
+
+        // Get compiler version information from target info
+        self.swiftCompilerVersion = Self.computeSwiftCompilerVersion(targetInfo: targetInfo)
+
+        // Use the triple from Swift SDK or compute the host triple from the target info
+        var triple = try swiftSDK.targetTriple ?? Self.getHostTriple(targetInfo: targetInfo)
 
         // Change the triple to the specified arch if there's exactly one of them.
         // The Triple property is only looked at by the native build system currently.

Sources/Workspace/Workspace+Prebuilts.swift

@@ -127,19 +127,22 @@ extension Workspace {
 
     /// For simplified init in tests
     public struct CustomPrebuiltsManager {
+        let swiftVersion: String
         let httpClient: HTTPClient?
         let archiver: Archiver?
         let useCache: Bool?
         let hostPlatform: PrebuiltsManifest.Platform?
         let rootCertPath: AbsolutePath?
 
         public init(
+            swiftVersion: String,
             httpClient: HTTPClient? = .none,
             archiver: Archiver? = .none,
             useCache: Bool? = .none,
             hostPlatform: PrebuiltsManifest.Platform? = nil,
             rootCertPath: AbsolutePath? = nil
         ) {
+            self.swiftVersion = swiftVersion
             self.httpClient = httpClient
             self.archiver = archiver
             self.useCache = useCache
@@ -153,6 +156,7 @@ extension Workspace {
         public typealias Delegate = PrebuiltsManagerDelegate
 
         private let fileSystem: FileSystem
+        private let swiftVersion: String
         private let authorizationProvider: AuthorizationProvider?
         private let httpClient: HTTPClient
         private let archiver: Archiver
@@ -167,6 +171,7 @@ extension Workspace {
         init(
             fileSystem: FileSystem,
             hostPlatform: PrebuiltsManifest.Platform,
+            swiftCompilerVersion: String,
             authorizationProvider: AuthorizationProvider?,
             scratchPath: AbsolutePath,
             cachePath: AbsolutePath?,
@@ -178,6 +183,7 @@ extension Workspace {
         ) {
             self.fileSystem = fileSystem
             self.hostPlatform = hostPlatform
+            self.swiftVersion = swiftCompilerVersion
             self.authorizationProvider = authorizationProvider
             self.httpClient = customHTTPClient ?? HTTPClient()
 
@@ -222,9 +228,6 @@ extension Workspace {
 
         private let prebuiltPackages: [PrebuiltPackage]
 
-        // Version of the compiler we're building against
-        private let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)"
-
         fileprivate func findPrebuilts(packages: [PackageReference]) -> [PrebuiltPackage] {
             var prebuilts: [PrebuiltPackage] = []
             for packageRef in packages {
@@ -310,6 +313,11 @@ extension Workspace {
                 }
             }
 
+            // Skip prebuilts if this file exists.
+            if let cachePath, fileSystem.exists(cachePath.appending("noprebuilts")) {
+                return nil
+            }
+
             if fileSystem.exists(destination), let manifest = try? await loadManifest() {
                 return manifest
             } else if let cacheDest, fileSystem.exists(cacheDest) {

Sources/Workspace/Workspace.swift

@@ -568,7 +568,10 @@ public class Workspace {
         // register the binary artifacts downloader with the cancellation handler
         cancellator?.register(name: "binary artifacts downloads", handler: binaryArtifactsManager)
 
-        if configuration.usePrebuilts, let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform {
+        if configuration.usePrebuilts,
+           let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform,
+           let swiftCompilerVersion = hostToolchain.swiftCompilerVersion
+        {
             let rootCertPath: AbsolutePath?
             if let path = configuration.prebuiltsRootCertPath {
                 rootCertPath = try AbsolutePath(validating: path)
@@ -579,6 +582,7 @@ public class Workspace {
             let prebuiltsManager = PrebuiltsManager(
                 fileSystem: fileSystem,
                 hostPlatform: hostPlatform,
+                swiftCompilerVersion: customPrebuiltsManager?.swiftVersion ?? swiftCompilerVersion,
                 authorizationProvider: authorizationProvider,
                 scratchPath: location.prebuiltsDirectory,
                 cachePath: customPrebuiltsManager?.useCache == false || !configuration.sharedDependenciesCacheEnabled ? .none : location.sharedPrebuiltsCacheDirectory,

Sources/_InternalTestSupport/MockWorkspace.swift

@@ -21,8 +21,15 @@ import Workspace
 import XCTest
 
 import struct TSCUtility.Version
+import enum TSCBasic.JSON
 
 extension UserToolchain {
+    package static var mockTargetInfo: JSON {
+        JSON.dictionary([
+            "compilerVersion": .string("Apple Swift version 6.2-dev (LLVM 815013bbc318474, Swift 1459ecafa998782)")
+        ])
+    }
+
     package static func mockHostToolchain(
         _ fileSystem: InMemoryFileSystem,
         hostTriple: Triple = hostTriple
@@ -42,6 +49,7 @@ extension UserToolchain {
                 ),
                 useXcrun: true
             ),
+            customTargetInfo: Self.mockTargetInfo,
             fileSystem: fileSystem
         )
     }

Sources/swift-build-prebuilts/BuildPrebuilts.swift

@@ -19,6 +19,7 @@
 import ArgumentParser
 import Basics
 import Foundation
+import PackageModel
 import struct TSCBasic.ByteString
 import struct TSCBasic.SHA256
 import Workspace
@@ -168,7 +169,6 @@ var prebuiltRepos: IdentifiableSet<PrebuiltRepos> = [
     ),
 ]
 
-let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)"
 let dockerImageRoot = "swiftlang/swift:nightly-6.1-"
 
 @main
@@ -216,6 +216,21 @@ struct BuildPrebuilts: AsyncParsableCommand {
         }
     }
 
+    func computeSwiftVersion() throws -> String? {
+        let fileSystem = localFileSystem
+
+        let environment = Environment.current
+        let hostToolchain = try UserToolchain(
+            swiftSDK: SwiftSDK.hostSwiftSDK(
+                environment: environment,
+                fileSystem: fileSystem
+            ),
+            environment: environment
+        )
+
+        return hostToolchain.swiftCompilerVersion
+    }
+
     mutating func run() async throws {
         if build {
             try await build()
@@ -231,6 +246,11 @@ struct BuildPrebuilts: AsyncParsableCommand {
         let encoder = JSONEncoder()
         encoder.outputFormatting = .prettyPrinted
 
+        guard let swiftVersion = try computeSwiftVersion() else {
+            print("Unable to determine swift compiler version")
+            return
+        }
+
         print("Stage directory: \(stageDir)")
 
         let srcDir = stageDir.appending("src")
@@ -379,6 +399,11 @@ struct BuildPrebuilts: AsyncParsableCommand {
         encoder.outputFormatting = .prettyPrinted
         let decoder = JSONDecoder()
 
+        guard let swiftVersion = try computeSwiftVersion() else {
+            print("Unable to determine swift compiler version")
+            return
+        }
+
         for repo in prebuiltRepos.values {
             let prebuiltDir = stageDir.appending(repo.url.lastPathComponent)
             for version in repo.versions {