From 041b0532da3427befd68eb285c452adb6abdb1e0 Mon Sep 17 00:00:00 2001 From: Doug Schaefer <167107236+dschaefer2@users.noreply.github.com> Date: Thu, 22 May 2025 06:27:17 -0400 Subject: [PATCH] Dynamically determine the swift compiler version. (#8707) Prebuilts for macros and the automated downloading of SwiftSDKs need to know the version of the compiler so we can fetch compatible binaries. The swiftc compiler has a --print-target-info options which dumps out a JSON structure that contains the compilerVersion. We already use info in this structure to determine the hostTriple for the UserToolchain. This adds the swiftCompilerVersion to UserToolchain that uses a couple of regex's to pull out the Swift compiler version. This is then used by the prebuilts feature instead of our current hardcodeing of the swift toolchain version. This also turns the prebuilts feature on by default which was supposed to be done in the last update. --- Sources/CoreCommands/Options.swift | 2 +- Sources/CoreCommands/SwiftCommandState.swift | 12 ++- Sources/PackageModel/UserToolchain.swift | 84 ++++++++++++++++++- Sources/Workspace/Workspace+Prebuilts.swift | 14 +++- Sources/Workspace/Workspace.swift | 6 +- .../_InternalTestSupport/MockWorkspace.swift | 8 ++ .../BuildPrebuilts.swift | 27 +++++- Tests/BuildTests/BuildPlanTests.swift | 13 ++- .../SwiftCommandStateTests.swift | 1 + Tests/WorkspaceTests/PrebuiltsTests.swift | 13 ++- 10 files changed, 169 insertions(+), 11 deletions(-) diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index c892694f383..cdbf9089d54 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -198,7 +198,7 @@ public struct CachingOptions: ParsableArguments { @Flag(name: .customLong("experimental-prebuilts"), inversion: .prefixedEnableDisable, help: "Whether to use prebuilt swift-syntax libraries for macros.") - public var usePrebuilts: Bool = false + public var usePrebuilts: Bool = true /// Hidden option to override the prebuilts download location for testing @Option( diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 51d483ab980..29150247ecd 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -47,6 +47,7 @@ import Bionic import class Basics.AsyncProcess import func TSCBasic.exec import class TSCBasic.FileLock +import enum TSCBasic.JSON import protocol TSCBasic.OutputByteStream import enum TSCBasic.ProcessEnv import enum TSCBasic.ProcessLockError @@ -287,6 +288,8 @@ public final class SwiftCommandState { private let hostTriple: Basics.Triple? + private let targetInfo: JSON? + package var preferredBuildConfiguration = BuildConfiguration.debug /// Create an instance of this tool. @@ -323,10 +326,12 @@ public final class SwiftCommandState { workspaceLoaderProvider: @escaping WorkspaceLoaderProvider, createPackagePath: Bool, hostTriple: Basics.Triple? = nil, + targetInfo: JSON? = nil, fileSystem: any FileSystem = localFileSystem, environment: Environment = .current ) throws { self.hostTriple = hostTriple + self.targetInfo = targetInfo self.fileSystem = fileSystem self.environment = environment // first, bootstrap the observability system @@ -968,7 +973,11 @@ public final class SwiftCommandState { } return Result(catching: { - try UserToolchain(swiftSDK: swiftSDK, environment: self.environment, fileSystem: self.fileSystem) + try UserToolchain( + swiftSDK: swiftSDK, + environment: self.environment, + customTargetInfo: targetInfo, + fileSystem: self.fileSystem) }) }() @@ -983,6 +992,7 @@ public final class SwiftCommandState { return try UserToolchain( swiftSDK: hostSwiftSDK, environment: self.environment, + customTargetInfo: targetInfo, fileSystem: self.fileSystem ) }) diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 56bcf17b452..1ff61309ddd 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -13,6 +13,7 @@ import Basics import Foundation import TSCUtility +import enum TSCBasic.JSON import class Basics.AsyncProcess @@ -74,6 +75,9 @@ public final class UserToolchain: Toolchain { public let targetTriple: Basics.Triple + // A version string that can be used to identify the swift compiler version + public let swiftCompilerVersion: String? + /// The list of CPU architectures to build for. public let architectures: [String]? @@ -160,6 +164,75 @@ public final class UserToolchain: Toolchain { return try getTool(name, binDirectories: envSearchPaths, fileSystem: fileSystem) } + private static func getTargetInfo(swiftCompiler: AbsolutePath) throws -> JSON { + // Call the compiler to get the target info JSON. + let compilerOutput: String + do { + let result = try AsyncProcess.popen(args: swiftCompiler.pathString, "-print-target-info") + compilerOutput = try result.utf8Output().spm_chomp() + } catch { + throw InternalError( + "Failed to load target info (\(error.interpolationDescription))" + ) + } + // Parse the compiler's JSON output. + do { + return try JSON(string: compilerOutput) + } catch { + throw InternalError( + "Failed to parse target info (\(error.interpolationDescription)).\nRaw compiler output: \(compilerOutput)" + ) + } + } + + private static func getHostTriple(targetInfo: JSON) throws -> Basics.Triple { + // Get the triple string from the target info. + let tripleString: String + do { + tripleString = try targetInfo.get("target").get("triple") + } catch { + throw InternalError( + "Target info does not contain a triple string (\(error.interpolationDescription)).\nTarget info: \(targetInfo)" + ) + } + + // Parse the triple string. + do { + return try Triple(tripleString) + } catch { + throw InternalError( + "Failed to parse triple string (\(error.interpolationDescription)).\nTriple string: \(tripleString)" + ) + } + } + + private static func computeSwiftCompilerVersion(targetInfo: JSON) -> String? { + // Get the compiler version from the target info + let compilerVersion: String + do { + compilerVersion = try targetInfo.get("compilerVersion") + } catch { + return nil + } + + // Extract the swift version using regex from the description if available + do { + let regex = try Regex(#"\((swift(lang)?-[^ )]*)"#) + if let match = try regex.firstMatch(in: compilerVersion), match.count > 1, let substring = match[1].substring { + return String(substring) + } + + let regex2 = try Regex(#"\(.*Swift (.*)[ )]"#) + if let match2 = try regex2.firstMatch(in: compilerVersion), match2.count > 1, let substring = match2[1].substring { + return "swift-\(substring)" + } else { + return nil + } + } catch { + return nil + } + } + // MARK: - public API public static func determineLibrarian( @@ -570,6 +643,7 @@ public final class UserToolchain: Toolchain { swiftSDK: SwiftSDK, environment: Environment = .current, searchStrategy: SearchStrategy = .default, + customTargetInfo: JSON? = nil, customLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation? = nil, customInstalledSwiftPMConfiguration: InstalledSwiftPMConfiguration? = nil, fileSystem: any FileSystem = localFileSystem @@ -612,8 +686,14 @@ public final class UserToolchain: Toolchain { default: InstalledSwiftPMConfiguration.default) } - // Use the triple from Swift SDK or compute the host triple using swiftc. - var triple = try swiftSDK.targetTriple ?? Triple.getHostTriple(usingSwiftCompiler: swiftCompilers.compile) + // targetInfo from the compiler + let targetInfo = try customTargetInfo ?? Self.getTargetInfo(swiftCompiler: swiftCompilers.compile) + + // Get compiler version information from target info + self.swiftCompilerVersion = Self.computeSwiftCompilerVersion(targetInfo: targetInfo) + + // Use the triple from Swift SDK or compute the host triple from the target info + var triple = try swiftSDK.targetTriple ?? Self.getHostTriple(targetInfo: targetInfo) // Change the triple to the specified arch if there's exactly one of them. // The Triple property is only looked at by the native build system currently. diff --git a/Sources/Workspace/Workspace+Prebuilts.swift b/Sources/Workspace/Workspace+Prebuilts.swift index 9c16a5edb51..6d8d4c6cf13 100644 --- a/Sources/Workspace/Workspace+Prebuilts.swift +++ b/Sources/Workspace/Workspace+Prebuilts.swift @@ -127,6 +127,7 @@ extension Workspace { /// For simplified init in tests public struct CustomPrebuiltsManager { + let swiftVersion: String let httpClient: HTTPClient? let archiver: Archiver? let useCache: Bool? @@ -134,12 +135,14 @@ extension Workspace { let rootCertPath: AbsolutePath? public init( + swiftVersion: String, httpClient: HTTPClient? = .none, archiver: Archiver? = .none, useCache: Bool? = .none, hostPlatform: PrebuiltsManifest.Platform? = nil, rootCertPath: AbsolutePath? = nil ) { + self.swiftVersion = swiftVersion self.httpClient = httpClient self.archiver = archiver self.useCache = useCache @@ -153,6 +156,7 @@ extension Workspace { public typealias Delegate = PrebuiltsManagerDelegate private let fileSystem: FileSystem + private let swiftVersion: String private let authorizationProvider: AuthorizationProvider? private let httpClient: HTTPClient private let archiver: Archiver @@ -167,6 +171,7 @@ extension Workspace { init( fileSystem: FileSystem, hostPlatform: PrebuiltsManifest.Platform, + swiftCompilerVersion: String, authorizationProvider: AuthorizationProvider?, scratchPath: AbsolutePath, cachePath: AbsolutePath?, @@ -178,6 +183,7 @@ extension Workspace { ) { self.fileSystem = fileSystem self.hostPlatform = hostPlatform + self.swiftVersion = swiftCompilerVersion self.authorizationProvider = authorizationProvider self.httpClient = customHTTPClient ?? HTTPClient() @@ -222,9 +228,6 @@ extension Workspace { private let prebuiltPackages: [PrebuiltPackage] - // Version of the compiler we're building against - private let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" - fileprivate func findPrebuilts(packages: [PackageReference]) -> [PrebuiltPackage] { var prebuilts: [PrebuiltPackage] = [] for packageRef in packages { @@ -310,6 +313,11 @@ extension Workspace { } } + // Skip prebuilts if this file exists. + if let cachePath, fileSystem.exists(cachePath.appending("noprebuilts")) { + return nil + } + if fileSystem.exists(destination), let manifest = try? await loadManifest() { return manifest } else if let cacheDest, fileSystem.exists(cacheDest) { diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 116c8e44bb8..3c199714868 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -568,7 +568,10 @@ public class Workspace { // register the binary artifacts downloader with the cancellation handler cancellator?.register(name: "binary artifacts downloads", handler: binaryArtifactsManager) - if configuration.usePrebuilts, let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform { + if configuration.usePrebuilts, + let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform, + let swiftCompilerVersion = hostToolchain.swiftCompilerVersion + { let rootCertPath: AbsolutePath? if let path = configuration.prebuiltsRootCertPath { rootCertPath = try AbsolutePath(validating: path) @@ -579,6 +582,7 @@ public class Workspace { let prebuiltsManager = PrebuiltsManager( fileSystem: fileSystem, hostPlatform: hostPlatform, + swiftCompilerVersion: customPrebuiltsManager?.swiftVersion ?? swiftCompilerVersion, authorizationProvider: authorizationProvider, scratchPath: location.prebuiltsDirectory, cachePath: customPrebuiltsManager?.useCache == false || !configuration.sharedDependenciesCacheEnabled ? .none : location.sharedPrebuiltsCacheDirectory, diff --git a/Sources/_InternalTestSupport/MockWorkspace.swift b/Sources/_InternalTestSupport/MockWorkspace.swift index 441f06ddd8b..0a0871774cd 100644 --- a/Sources/_InternalTestSupport/MockWorkspace.swift +++ b/Sources/_InternalTestSupport/MockWorkspace.swift @@ -21,8 +21,15 @@ import Workspace import XCTest import struct TSCUtility.Version +import enum TSCBasic.JSON extension UserToolchain { + package static var mockTargetInfo: JSON { + JSON.dictionary([ + "compilerVersion": .string("Apple Swift version 6.2-dev (LLVM 815013bbc318474, Swift 1459ecafa998782)") + ]) + } + package static func mockHostToolchain( _ fileSystem: InMemoryFileSystem, hostTriple: Triple = hostTriple @@ -42,6 +49,7 @@ extension UserToolchain { ), useXcrun: true ), + customTargetInfo: Self.mockTargetInfo, fileSystem: fileSystem ) } diff --git a/Sources/swift-build-prebuilts/BuildPrebuilts.swift b/Sources/swift-build-prebuilts/BuildPrebuilts.swift index 4db1740dc17..40cb5541774 100644 --- a/Sources/swift-build-prebuilts/BuildPrebuilts.swift +++ b/Sources/swift-build-prebuilts/BuildPrebuilts.swift @@ -19,6 +19,7 @@ import ArgumentParser import Basics import Foundation +import PackageModel import struct TSCBasic.ByteString import struct TSCBasic.SHA256 import Workspace @@ -168,7 +169,6 @@ var prebuiltRepos: IdentifiableSet = [ ), ] -let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" let dockerImageRoot = "swiftlang/swift:nightly-6.1-" @main @@ -216,6 +216,21 @@ struct BuildPrebuilts: AsyncParsableCommand { } } + func computeSwiftVersion() throws -> String? { + let fileSystem = localFileSystem + + let environment = Environment.current + let hostToolchain = try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK( + environment: environment, + fileSystem: fileSystem + ), + environment: environment + ) + + return hostToolchain.swiftCompilerVersion + } + mutating func run() async throws { if build { try await build() @@ -231,6 +246,11 @@ struct BuildPrebuilts: AsyncParsableCommand { let encoder = JSONEncoder() encoder.outputFormatting = .prettyPrinted + guard let swiftVersion = try computeSwiftVersion() else { + print("Unable to determine swift compiler version") + return + } + print("Stage directory: \(stageDir)") let srcDir = stageDir.appending("src") @@ -379,6 +399,11 @@ struct BuildPrebuilts: AsyncParsableCommand { encoder.outputFormatting = .prettyPrinted let decoder = JSONDecoder() + guard let swiftVersion = try computeSwiftVersion() else { + print("Unable to determine swift compiler version") + return + } + for repo in prebuiltRepos.values { let prebuiltDir = stageDir.appending(repo.url.lastPathComponent) for version in repo.versions { diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 287890021f7..1806490b562 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4914,6 +4914,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fs ) let commonFlags = BuildFlags( @@ -5040,6 +5041,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fs ) @@ -5078,6 +5080,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { XCTAssertNoDiagnostics(observability.diagnostics) let result = try await BuildPlanResult(plan: mockBuildPlan( + triple: mockToolchain.targetTriple, toolchain: mockToolchain, graph: graph, commonFlags: .init(), @@ -5157,6 +5160,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fs ) @@ -5193,6 +5197,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { XCTAssertNoDiagnostics(observability.diagnostics) let result = try await BuildPlanResult(plan: mockBuildPlan( + triple: mockToolchain.targetTriple, toolchain: mockToolchain, graph: graph, commonFlags: .init(), @@ -5279,7 +5284,12 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { swiftStaticResourcesPath: "/usr/lib/swift_static/none" ) ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment, fileSystem: fileSystem) + let toolchain = try UserToolchain( + swiftSDK: swiftSDK, + environment: .mockEnvironment, + customTargetInfo: UserToolchain.mockTargetInfo, + fileSystem: fileSystem + ) let result = try await BuildPlanResult(plan: mockBuildPlan( triple: targetTriple, toolchain: toolchain, @@ -5445,6 +5455,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fileSystem ) let result = try await BuildPlanResult(plan: mockBuildPlan( diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index a4cb33a1e2a..5ee582d18bf 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -566,6 +566,7 @@ extension SwiftCommandState { }, createPackagePath: createPackagePath, hostTriple: .arm64Linux, + targetInfo: UserToolchain.mockTargetInfo, fileSystem: fileSystem, environment: environment ) diff --git a/Tests/WorkspaceTests/PrebuiltsTests.swift b/Tests/WorkspaceTests/PrebuiltsTests.swift index c36f73e0b37..0f672b4afea 100644 --- a/Tests/WorkspaceTests/PrebuiltsTests.swift +++ b/Tests/WorkspaceTests/PrebuiltsTests.swift @@ -205,11 +205,12 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, rootCertPath: rootCertPath - ) + ), ) try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in @@ -268,6 +269,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -372,6 +374,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -434,6 +437,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -486,6 +490,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, rootCertPath: rootCertPath @@ -551,6 +556,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .ubuntu_noble_x86_64, @@ -600,6 +606,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, rootCertPath: rootCertPath @@ -666,6 +673,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -727,6 +735,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -790,6 +799,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -846,6 +856,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver )