diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index c892694f383..cdbf9089d54 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -198,7 +198,7 @@ public struct CachingOptions: ParsableArguments { @Flag(name: .customLong("experimental-prebuilts"), inversion: .prefixedEnableDisable, help: "Whether to use prebuilt swift-syntax libraries for macros.") - public var usePrebuilts: Bool = false + public var usePrebuilts: Bool = true /// Hidden option to override the prebuilts download location for testing @Option( diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 51d483ab980..29150247ecd 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -47,6 +47,7 @@ import Bionic import class Basics.AsyncProcess import func TSCBasic.exec import class TSCBasic.FileLock +import enum TSCBasic.JSON import protocol TSCBasic.OutputByteStream import enum TSCBasic.ProcessEnv import enum TSCBasic.ProcessLockError @@ -287,6 +288,8 @@ public final class SwiftCommandState { private let hostTriple: Basics.Triple? + private let targetInfo: JSON? + package var preferredBuildConfiguration = BuildConfiguration.debug /// Create an instance of this tool. @@ -323,10 +326,12 @@ public final class SwiftCommandState { workspaceLoaderProvider: @escaping WorkspaceLoaderProvider, createPackagePath: Bool, hostTriple: Basics.Triple? = nil, + targetInfo: JSON? = nil, fileSystem: any FileSystem = localFileSystem, environment: Environment = .current ) throws { self.hostTriple = hostTriple + self.targetInfo = targetInfo self.fileSystem = fileSystem self.environment = environment // first, bootstrap the observability system @@ -968,7 +973,11 @@ public final class SwiftCommandState { } return Result(catching: { - try UserToolchain(swiftSDK: swiftSDK, environment: self.environment, fileSystem: self.fileSystem) + try UserToolchain( + swiftSDK: swiftSDK, + environment: self.environment, + customTargetInfo: targetInfo, + fileSystem: self.fileSystem) }) }() @@ -983,6 +992,7 @@ public final class SwiftCommandState { return try UserToolchain( swiftSDK: hostSwiftSDK, environment: self.environment, + customTargetInfo: targetInfo, fileSystem: self.fileSystem ) }) diff --git a/Sources/PackageModel/UserToolchain.swift b/Sources/PackageModel/UserToolchain.swift index 56bcf17b452..1ff61309ddd 100644 --- a/Sources/PackageModel/UserToolchain.swift +++ b/Sources/PackageModel/UserToolchain.swift @@ -13,6 +13,7 @@ import Basics import Foundation import TSCUtility +import enum TSCBasic.JSON import class Basics.AsyncProcess @@ -74,6 +75,9 @@ public final class UserToolchain: Toolchain { public let targetTriple: Basics.Triple + // A version string that can be used to identify the swift compiler version + public let swiftCompilerVersion: String? + /// The list of CPU architectures to build for. public let architectures: [String]? @@ -160,6 +164,75 @@ public final class UserToolchain: Toolchain { return try getTool(name, binDirectories: envSearchPaths, fileSystem: fileSystem) } + private static func getTargetInfo(swiftCompiler: AbsolutePath) throws -> JSON { + // Call the compiler to get the target info JSON. + let compilerOutput: String + do { + let result = try AsyncProcess.popen(args: swiftCompiler.pathString, "-print-target-info") + compilerOutput = try result.utf8Output().spm_chomp() + } catch { + throw InternalError( + "Failed to load target info (\(error.interpolationDescription))" + ) + } + // Parse the compiler's JSON output. + do { + return try JSON(string: compilerOutput) + } catch { + throw InternalError( + "Failed to parse target info (\(error.interpolationDescription)).\nRaw compiler output: \(compilerOutput)" + ) + } + } + + private static func getHostTriple(targetInfo: JSON) throws -> Basics.Triple { + // Get the triple string from the target info. + let tripleString: String + do { + tripleString = try targetInfo.get("target").get("triple") + } catch { + throw InternalError( + "Target info does not contain a triple string (\(error.interpolationDescription)).\nTarget info: \(targetInfo)" + ) + } + + // Parse the triple string. + do { + return try Triple(tripleString) + } catch { + throw InternalError( + "Failed to parse triple string (\(error.interpolationDescription)).\nTriple string: \(tripleString)" + ) + } + } + + private static func computeSwiftCompilerVersion(targetInfo: JSON) -> String? { + // Get the compiler version from the target info + let compilerVersion: String + do { + compilerVersion = try targetInfo.get("compilerVersion") + } catch { + return nil + } + + // Extract the swift version using regex from the description if available + do { + let regex = try Regex(#"\((swift(lang)?-[^ )]*)"#) + if let match = try regex.firstMatch(in: compilerVersion), match.count > 1, let substring = match[1].substring { + return String(substring) + } + + let regex2 = try Regex(#"\(.*Swift (.*)[ )]"#) + if let match2 = try regex2.firstMatch(in: compilerVersion), match2.count > 1, let substring = match2[1].substring { + return "swift-\(substring)" + } else { + return nil + } + } catch { + return nil + } + } + // MARK: - public API public static func determineLibrarian( @@ -570,6 +643,7 @@ public final class UserToolchain: Toolchain { swiftSDK: SwiftSDK, environment: Environment = .current, searchStrategy: SearchStrategy = .default, + customTargetInfo: JSON? = nil, customLibrariesLocation: ToolchainConfiguration.SwiftPMLibrariesLocation? = nil, customInstalledSwiftPMConfiguration: InstalledSwiftPMConfiguration? = nil, fileSystem: any FileSystem = localFileSystem @@ -612,8 +686,14 @@ public final class UserToolchain: Toolchain { default: InstalledSwiftPMConfiguration.default) } - // Use the triple from Swift SDK or compute the host triple using swiftc. - var triple = try swiftSDK.targetTriple ?? Triple.getHostTriple(usingSwiftCompiler: swiftCompilers.compile) + // targetInfo from the compiler + let targetInfo = try customTargetInfo ?? Self.getTargetInfo(swiftCompiler: swiftCompilers.compile) + + // Get compiler version information from target info + self.swiftCompilerVersion = Self.computeSwiftCompilerVersion(targetInfo: targetInfo) + + // Use the triple from Swift SDK or compute the host triple from the target info + var triple = try swiftSDK.targetTriple ?? Self.getHostTriple(targetInfo: targetInfo) // Change the triple to the specified arch if there's exactly one of them. // The Triple property is only looked at by the native build system currently. diff --git a/Sources/Workspace/Workspace+Prebuilts.swift b/Sources/Workspace/Workspace+Prebuilts.swift index 9c16a5edb51..6d8d4c6cf13 100644 --- a/Sources/Workspace/Workspace+Prebuilts.swift +++ b/Sources/Workspace/Workspace+Prebuilts.swift @@ -127,6 +127,7 @@ extension Workspace { /// For simplified init in tests public struct CustomPrebuiltsManager { + let swiftVersion: String let httpClient: HTTPClient? let archiver: Archiver? let useCache: Bool? @@ -134,12 +135,14 @@ extension Workspace { let rootCertPath: AbsolutePath? public init( + swiftVersion: String, httpClient: HTTPClient? = .none, archiver: Archiver? = .none, useCache: Bool? = .none, hostPlatform: PrebuiltsManifest.Platform? = nil, rootCertPath: AbsolutePath? = nil ) { + self.swiftVersion = swiftVersion self.httpClient = httpClient self.archiver = archiver self.useCache = useCache @@ -153,6 +156,7 @@ extension Workspace { public typealias Delegate = PrebuiltsManagerDelegate private let fileSystem: FileSystem + private let swiftVersion: String private let authorizationProvider: AuthorizationProvider? private let httpClient: HTTPClient private let archiver: Archiver @@ -167,6 +171,7 @@ extension Workspace { init( fileSystem: FileSystem, hostPlatform: PrebuiltsManifest.Platform, + swiftCompilerVersion: String, authorizationProvider: AuthorizationProvider?, scratchPath: AbsolutePath, cachePath: AbsolutePath?, @@ -178,6 +183,7 @@ extension Workspace { ) { self.fileSystem = fileSystem self.hostPlatform = hostPlatform + self.swiftVersion = swiftCompilerVersion self.authorizationProvider = authorizationProvider self.httpClient = customHTTPClient ?? HTTPClient() @@ -222,9 +228,6 @@ extension Workspace { private let prebuiltPackages: [PrebuiltPackage] - // Version of the compiler we're building against - private let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" - fileprivate func findPrebuilts(packages: [PackageReference]) -> [PrebuiltPackage] { var prebuilts: [PrebuiltPackage] = [] for packageRef in packages { @@ -310,6 +313,11 @@ extension Workspace { } } + // Skip prebuilts if this file exists. + if let cachePath, fileSystem.exists(cachePath.appending("noprebuilts")) { + return nil + } + if fileSystem.exists(destination), let manifest = try? await loadManifest() { return manifest } else if let cacheDest, fileSystem.exists(cacheDest) { diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 116c8e44bb8..3c199714868 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -568,7 +568,10 @@ public class Workspace { // register the binary artifacts downloader with the cancellation handler cancellator?.register(name: "binary artifacts downloads", handler: binaryArtifactsManager) - if configuration.usePrebuilts, let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform { + if configuration.usePrebuilts, + let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform, + let swiftCompilerVersion = hostToolchain.swiftCompilerVersion + { let rootCertPath: AbsolutePath? if let path = configuration.prebuiltsRootCertPath { rootCertPath = try AbsolutePath(validating: path) @@ -579,6 +582,7 @@ public class Workspace { let prebuiltsManager = PrebuiltsManager( fileSystem: fileSystem, hostPlatform: hostPlatform, + swiftCompilerVersion: customPrebuiltsManager?.swiftVersion ?? swiftCompilerVersion, authorizationProvider: authorizationProvider, scratchPath: location.prebuiltsDirectory, cachePath: customPrebuiltsManager?.useCache == false || !configuration.sharedDependenciesCacheEnabled ? .none : location.sharedPrebuiltsCacheDirectory, diff --git a/Sources/_InternalTestSupport/MockWorkspace.swift b/Sources/_InternalTestSupport/MockWorkspace.swift index 441f06ddd8b..0a0871774cd 100644 --- a/Sources/_InternalTestSupport/MockWorkspace.swift +++ b/Sources/_InternalTestSupport/MockWorkspace.swift @@ -21,8 +21,15 @@ import Workspace import XCTest import struct TSCUtility.Version +import enum TSCBasic.JSON extension UserToolchain { + package static var mockTargetInfo: JSON { + JSON.dictionary([ + "compilerVersion": .string("Apple Swift version 6.2-dev (LLVM 815013bbc318474, Swift 1459ecafa998782)") + ]) + } + package static func mockHostToolchain( _ fileSystem: InMemoryFileSystem, hostTriple: Triple = hostTriple @@ -42,6 +49,7 @@ extension UserToolchain { ), useXcrun: true ), + customTargetInfo: Self.mockTargetInfo, fileSystem: fileSystem ) } diff --git a/Sources/swift-build-prebuilts/BuildPrebuilts.swift b/Sources/swift-build-prebuilts/BuildPrebuilts.swift index 4db1740dc17..40cb5541774 100644 --- a/Sources/swift-build-prebuilts/BuildPrebuilts.swift +++ b/Sources/swift-build-prebuilts/BuildPrebuilts.swift @@ -19,6 +19,7 @@ import ArgumentParser import Basics import Foundation +import PackageModel import struct TSCBasic.ByteString import struct TSCBasic.SHA256 import Workspace @@ -168,7 +169,6 @@ var prebuiltRepos: IdentifiableSet = [ ), ] -let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" let dockerImageRoot = "swiftlang/swift:nightly-6.1-" @main @@ -216,6 +216,21 @@ struct BuildPrebuilts: AsyncParsableCommand { } } + func computeSwiftVersion() throws -> String? { + let fileSystem = localFileSystem + + let environment = Environment.current + let hostToolchain = try UserToolchain( + swiftSDK: SwiftSDK.hostSwiftSDK( + environment: environment, + fileSystem: fileSystem + ), + environment: environment + ) + + return hostToolchain.swiftCompilerVersion + } + mutating func run() async throws { if build { try await build() @@ -231,6 +246,11 @@ struct BuildPrebuilts: AsyncParsableCommand { let encoder = JSONEncoder() encoder.outputFormatting = .prettyPrinted + guard let swiftVersion = try computeSwiftVersion() else { + print("Unable to determine swift compiler version") + return + } + print("Stage directory: \(stageDir)") let srcDir = stageDir.appending("src") @@ -379,6 +399,11 @@ struct BuildPrebuilts: AsyncParsableCommand { encoder.outputFormatting = .prettyPrinted let decoder = JSONDecoder() + guard let swiftVersion = try computeSwiftVersion() else { + print("Unable to determine swift compiler version") + return + } + for repo in prebuiltRepos.values { let prebuiltDir = stageDir.appending(repo.url.lastPathComponent) for version in repo.versions { diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 287890021f7..1806490b562 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4914,6 +4914,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fs ) let commonFlags = BuildFlags( @@ -5040,6 +5041,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fs ) @@ -5078,6 +5080,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { XCTAssertNoDiagnostics(observability.diagnostics) let result = try await BuildPlanResult(plan: mockBuildPlan( + triple: mockToolchain.targetTriple, toolchain: mockToolchain, graph: graph, commonFlags: .init(), @@ -5157,6 +5160,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fs ) @@ -5193,6 +5197,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { XCTAssertNoDiagnostics(observability.diagnostics) let result = try await BuildPlanResult(plan: mockBuildPlan( + triple: mockToolchain.targetTriple, toolchain: mockToolchain, graph: graph, commonFlags: .init(), @@ -5279,7 +5284,12 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { swiftStaticResourcesPath: "/usr/lib/swift_static/none" ) ) - let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment, fileSystem: fileSystem) + let toolchain = try UserToolchain( + swiftSDK: swiftSDK, + environment: .mockEnvironment, + customTargetInfo: UserToolchain.mockTargetInfo, + fileSystem: fileSystem + ) let result = try await BuildPlanResult(plan: mockBuildPlan( triple: targetTriple, toolchain: toolchain, @@ -5445,6 +5455,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { ), useXcrun: true ), + customTargetInfo: UserToolchain.mockTargetInfo, fileSystem: fileSystem ) let result = try await BuildPlanResult(plan: mockBuildPlan( diff --git a/Tests/CommandsTests/SwiftCommandStateTests.swift b/Tests/CommandsTests/SwiftCommandStateTests.swift index a4cb33a1e2a..5ee582d18bf 100644 --- a/Tests/CommandsTests/SwiftCommandStateTests.swift +++ b/Tests/CommandsTests/SwiftCommandStateTests.swift @@ -566,6 +566,7 @@ extension SwiftCommandState { }, createPackagePath: createPackagePath, hostTriple: .arm64Linux, + targetInfo: UserToolchain.mockTargetInfo, fileSystem: fileSystem, environment: environment ) diff --git a/Tests/WorkspaceTests/PrebuiltsTests.swift b/Tests/WorkspaceTests/PrebuiltsTests.swift index c36f73e0b37..0f672b4afea 100644 --- a/Tests/WorkspaceTests/PrebuiltsTests.swift +++ b/Tests/WorkspaceTests/PrebuiltsTests.swift @@ -205,11 +205,12 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, rootCertPath: rootCertPath - ) + ), ) try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in @@ -268,6 +269,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -372,6 +374,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -434,6 +437,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -486,6 +490,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, rootCertPath: rootCertPath @@ -551,6 +556,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .ubuntu_noble_x86_64, @@ -600,6 +606,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, rootCertPath: rootCertPath @@ -666,6 +673,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -727,6 +735,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -790,6 +799,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver, hostPlatform: .macos_aarch64, @@ -846,6 +856,7 @@ final class PrebuiltsTests: XCTestCase { swiftSyntax ], prebuiltsManager: .init( + swiftVersion: swiftVersion, httpClient: httpClient, archiver: archiver )