Tests: Augment SwiftBuild build provider test coverage

Augment some test to run against the native and swiftbuild build
providers to gain extra confidence with the swift build integration.

Update Test helpers to use BuildConfiguration in production code instead
of configuration

Where applicable, create a test swift for the various build
configurations to allow to better skip test in specific scenarios.

Author: bkhouri

IntegrationTests/Tests/IntegrationTests/SwiftPMTests.swift

@@ -75,15 +75,6 @@ private struct SwiftPMTests {
     }
 
     @Test(
-        .requireThreadSafeWorkingDirectory,
-        arguments: [BuildSystemProvider.native]
-    )
-    func packageInitExecutable(_ buildSystemProvider: BuildSystemProvider) throws {
-        try _packageInitExecutable(buildSystemProvider)
-    }
-
-    @Test(
-        .skipHostOS(.windows),
         .requireThreadSafeWorkingDirectory,
         .bug(
             "https://github.com/swiftlang/swift-package-manager/issues/8416",
@@ -93,13 +84,9 @@ private struct SwiftPMTests {
             "https://github.com/swiftlang/swift-package-manager/issues/8514",
             "[Windows] Integration test SwiftPMTests.packageInitExecutable with --build-system swiftbuild is skipped"
         ),
-        arguments: [BuildSystemProvider.swiftbuild]
+        arguments: BuildSystemProvider.allCases
     )
-    func packageInitExecutablSkipWindows(_ buildSystemProvider: BuildSystemProvider) throws {
-        try _packageInitExecutable(buildSystemProvider)
-    }
-
-    private func _packageInitExecutable(_ buildSystemProvider: BuildSystemProvider) throws {
+    private func packageInitExecutable(_ buildSystemProvider: BuildSystemProvider) throws {
         try withTemporaryDirectory { tmpDir in
             let packagePath = tmpDir.appending(component: "foo")
             try localFileSystem.createDirectory(packagePath)
@@ -115,16 +102,16 @@ private struct SwiftPMTests {
                 #expect(!stderr.contains("error:"))
                 #expect(stdout.contains("Hello, world!"))
             } when: {
-                buildSystemProvider == .swiftbuild && ProcessInfo.hostOperatingSystem == .linux
+                buildSystemProvider == .swiftbuild && (ProcessInfo.hostOperatingSystem == .linux || ProcessInfo.hostOperatingSystem == .windows)
             }
             }
     }
 
     @Test(
         .requireThreadSafeWorkingDirectory,
-        .bug(id: 0, "SWBINTTODO: Linux: /lib/x86_64-linux-gnu/Scrt1.o:function _start: error:"),
+        .bug(id: 0, "SwiftBuildTodo: Linux: /lib/x86_64-linux-gnu/Scrt1.o:function _start: error:"),
         .bug("https://github.com/swiftlang/swift-package-manager/issues/8380", "lld-link: error: subsystem must be defined"),
-        .bug(id: 0, "SWBINTTODO: MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found"),
+        .bug(id: 0, "SwiftBuildTodo: MacOS: Could not find or use auto-linked library 'Testing': library 'Testing' not found"),
         arguments: BuildSystemProvider.allCases
     )
     func packageInitLibrary(_ buildSystemProvider: BuildSystemProvider) throws {

Sources/_InternalTestSupport/Commands.swift

@@ -1,8 +1,20 @@
-import SPMBuildCore
+import struct SPMBuildCore.BuildSystemProvider
+import enum PackageModel.BuildConfiguration
 import XCTest
 
 open class BuildSystemProviderTestCase: XCTestCase {
     open var buildSystemProvider: BuildSystemProvider.Kind {
         fatalError("\(self) does not implement \(#function)")
     }
 }
+
+open class BuildConfigurationTestCase: BuildSystemProviderTestCase {
+    open var binPathSuffixes: [String] {
+        fatalError("\(self) does not implement \(#function)")
+    }
+
+    open var buildConfig: BuildConfiguration {
+        fatalError("\(self) does not implement \(#function)")
+    }
+
+}

Sources/_InternalTestSupport/XCTAssertHelpers.swift

@@ -15,6 +15,7 @@ import Basics
 import class Foundation.Bundle
 #endif
 import SPMBuildCore
+import enum PackageModel.BuildConfiguration
 import TSCTestSupport
 import XCTest
 
@@ -50,6 +51,10 @@ public func XCTSkipIfCI(file: StaticString = #filePath, line: UInt = #line) thro
     }
 }
 
+public func XCTSkipSwiftBuildTodo(because reason: String) throws {
+    throw XCTSkip("SwiftBuildTodo: \(reason)")
+}
+
 /// An `async`-friendly replacement for `XCTAssertThrowsError`.
 public func XCTAssertAsyncThrowsError<T>(
     _ expression: @autoclosure () async throws -> T,
@@ -81,7 +86,7 @@ package func XCTAssertAsyncNoThrow<T>(
 
 public func XCTAssertBuilds(
     _ path: AbsolutePath,
-    configurations: Set<Configuration> = [.Debug, .Release],
+    configurations: Set<BuildConfiguration> = [.debug, .release],
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
@@ -111,7 +116,7 @@ public func XCTAssertBuilds(
 
 public func XCTAssertSwiftTest(
     _ path: AbsolutePath,
-    configuration: Configuration = .Debug,
+    configuration: BuildConfiguration = .debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],

Sources/_InternalTestSupport/misc.swift

@@ -267,7 +267,7 @@ public func getBuildSystemArgs(for buildSystem: BuildSystemProvider.Kind?) -> [S
 @discardableResult
 public func executeSwiftBuild(
     _ packagePath: AbsolutePath?,
-    configuration: Configuration = .Debug,
+    configuration: BuildConfiguration = .debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
@@ -302,7 +302,7 @@ public func skipOnWindowsAsTestCurrentlyFails(because reason: String? = nil) thr
 public func executeSwiftRun(
     _ packagePath: AbsolutePath?,
     _ executable: String?,
-    configuration: Configuration = .Debug,
+    configuration: BuildConfiguration = .debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
@@ -327,7 +327,7 @@ public func executeSwiftRun(
 @discardableResult
 public func executeSwiftPackage(
     _ packagePath: AbsolutePath?,
-    configuration: Configuration = .Debug,
+    configuration: BuildConfiguration = .debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
@@ -349,7 +349,7 @@ public func executeSwiftPackage(
 @discardableResult
 public func executeSwiftPackageRegistry(
     _ packagePath: AbsolutePath?,
-    configuration: Configuration = .Debug,
+    configuration: BuildConfiguration = .debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
@@ -371,7 +371,7 @@ public func executeSwiftPackageRegistry(
 @discardableResult
 public func executeSwiftTest(
     _ packagePath: AbsolutePath?,
-    configuration: Configuration = .Debug,
+    configuration: BuildConfiguration = .debug,
     extraArgs: [String] = [],
     Xcc: [String] = [],
     Xld: [String] = [],
@@ -392,7 +392,7 @@ public func executeSwiftTest(
 }
 
 private func swiftArgs(
-    configuration: Configuration,
+    configuration: BuildConfiguration,
     extraArgs: [String],
     Xcc: [String],
     Xld: [String],
@@ -401,9 +401,9 @@ private func swiftArgs(
 ) -> [String] {
     var args = ["--configuration"]
     switch configuration {
-    case .Debug:
+    case .debug:
         args.append("debug")
-    case .Release:
+    case .release:
         args.append("release")
     }
 

Tests/CommandsTests/BuildCommandTests.swift

@@ -855,7 +855,7 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
     }
 
     override func testNonReachableProductsAndTargetsFunctional() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "Test failed. This needs to be investigated")
     }
 
     override func testParseableInterfaces() async throws {
@@ -877,7 +877,7 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
     }
 
     override func testAutomaticParseableInterfacesWithLibraryEvolution() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed because of missing 'A.swiftmodule/*.swiftinterface' files")
+        try XCTSkipSwiftBuildTodo(because: "Test failed because of missing 'A.swiftmodule/*.swiftinterface' files")
         // TODO: We still need to override this test just like we did for `testParseableInterfaces` above.
     }
 
@@ -896,50 +896,50 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
     }
 
     override func testGetTaskAllowEntitlement() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed because swiftbuild doesn't output precis codesign commands. Once swift run works with swiftbuild the test can be investigated.")
+        try XCTSkipSwiftBuildTodo(because: "Test failed because swiftbuild doesn't output precis codesign commands. Once swift run works with swiftbuild the test can be investigated.")
     }
 
     override func testCodeCoverage() async throws {
-        throw XCTSkip("SWBINTTODO: Test failed because of missing plugin support in the PIF builder. This can be reinvestigated after the support is there.")
+        try XCTSkipSwiftBuildTodo(because: "Test failed because of missing plugin support in the PIF builder. This can be reinvestigated after the support is there.")
     }
 
     override func testAtMainSupport() async throws {
         #if !os(macOS)
-        throw XCTSkip("SWBINTTODO: File not found or missing libclang errors on non-macOS platforms. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "File not found or missing libclang errors on non-macOS platforms. This needs to be investigated")
         #else
         try await super.testAtMainSupport()
         #endif
     }
 
     override func testImportOfMissedDepWarning() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because the warning message regarding missing imports is expected to be more verbose and actionable at the SwiftPM level with mention of the involved targets. This needs to be investigated. See case targetDiagnostic(TargetDiagnosticInfo) as a message type that may help.")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because the warning message regarding missing imports is expected to be more verbose and actionable at the SwiftPM level with mention of the involved targets. This needs to be investigated. See case targetDiagnostic(TargetDiagnosticInfo) as a message type that may help.")
     }
 
     override func testProductAndTarget() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because there isn't a clear warning message about the lib1 being an automatic product and that the default product is being built instead. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because there isn't a clear warning message about the lib1 being an automatic product and that the default product is being built instead. This needs to be investigated")
     }
 
     override func testSwiftGetVersion() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because the dummy-swiftc used in the test isn't accepted by swift-build. This needs to be investigated")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because the dummy-swiftc used in the test isn't accepted by swift-build. This needs to be investigated")
     }
 
     override func testSymlink() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because of a difference in the build layout. This needs to be updated to the expected path")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because of a difference in the build layout. This needs to be updated to the expected path")
     }
 
 #if !canImport(Darwin)
     override func testIgnoresLinuxMain() async throws {
-        throw XCTSkip("SWBINTTODO: Swift build doesn't currently ignore Linux main when linking on Linux. This needs further investigation.")
+        try XCTSkipSwiftBuildTodo(because: "Swift build doesn't currently ignore Linux main when linking on Linux. This needs further investigation.")
     }
 #endif
 
 #if !os(macOS)
     override func testBuildStartMessage() async throws {
-        throw XCTSkip("SWBINTTODO: Swift build produces an error building the fixture for this test.")
+        try XCTSkipSwiftBuildTodo(because: "Swift build produces an error building the fixture for this test.")
     }
 
     override func testSwiftDriverRawOutputGetsNewlines() async throws {
-        throw XCTSkip("SWBINTTODO: Swift build produces an error building the fixture for this test.")
+        try XCTSkipSwiftBuildTodo(because: "Swift build produces an error building the fixture for this test.")
     }
 #endif
 
@@ -959,7 +959,7 @@ class BuildCommandSwiftBuildTests: BuildCommandTestCases {
 
     override func testBuildCompleteMessage() async throws {
         #if os(Linux)
-        throw XCTSkip("SWBINTTODO: Need to properly set LD_LIBRARY_PATH on linux")
+        try XCTSkipSwiftBuildTodo(because: "Need to properly set LD_LIBRARY_PATH on linux")
         #else
         try await super.testBuildCompleteMessage()
         #endif

Tests/CommandsTests/MultiRootSupportTests.swift

@@ -16,7 +16,7 @@ import _InternalTestSupport
 import Workspace
 import XCTest
 
-final class MultiRootSupportTests: CommandsTestCase {
+final class MultiRootSupportTests: XCTestCase {
     func testWorkspaceLoader() throws {
         let fs = InMemoryFileSystem(emptyFiles: [
             "/tmp/test/dep/Package.swift",

Tests/CommandsTests/PackageCommandTests.swift

@@ -1385,7 +1385,7 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase {
 
     func testPackageClean() async throws {
         try await fixture(name: "DependencyResolution/External/Simple") { fixturePath in
-            let packageRoot = fixturePath.appending("Bar")
+            let packageRoot: AbsolutePath = fixturePath.appending("Bar")
 
             // Build it.
             await XCTAssertBuilds(packageRoot)
@@ -1397,6 +1397,8 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase {
             // Clean, and check for removal of the build directory but not Packages.
             _ = try await execute(["clean"], packagePath: packageRoot)
             XCTAssertNoSuchPath(binFile)
+            XCTAssertFalse(try localFileSystem.getDirectoryContents(buildPath.appending("repositories")).isEmpty)
+
             // Clean again to ensure we get no error.
             _ = try await execute(["clean"], packagePath: packageRoot)
         }
@@ -1412,15 +1414,10 @@ class PackageCommandTestCase: CommandsBuildProviderTestCase {
             let binFile = buildPath.appending(components: try UserToolchain.default.targetTriple.platformBuildPathComponent, "debug", "Bar")
             XCTAssertFileExists(binFile)
             XCTAssert(localFileSystem.isDirectory(buildPath))
-            // Clean, and check for removal of the build directory but not Packages.
-
-            _ = try await execute(["clean"], packagePath: packageRoot)
-            XCTAssertNoSuchPath(binFile)
-            XCTAssertFalse(try localFileSystem.getDirectoryContents(buildPath.appending("repositories")).isEmpty)
 
             // Fully clean.
             _ = try await execute(["reset"], packagePath: packageRoot)
-            XCTAssertFalse(localFileSystem.isDirectory(buildPath))
+            XCTAssertFalse(localFileSystem.exists(buildPath))
 
             // Test that we can successfully run reset again.
             _ = try await execute(["reset"], packagePath: packageRoot)
@@ -3898,15 +3895,15 @@ class PackageCommandSwiftBuildTests: PackageCommandTestCase {
     }
 
     override func testCommandPluginBuildingCallbacks() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails as plugins are not currenty supported")
+        try XCTSkipSwiftBuildTodo(because: "Test fails as plugins are not currenty supported")
     }
     override func testCommandPluginBuildTestability() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails as plugins are not currenty supported")
+        try XCTSkipSwiftBuildTodo(because: "Test fails as plugins are not currenty supported")
     }
 
 #if !os(macOS)
     override func testCommandPluginTestingCallbacks() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails on inability to find libclang on Linux. Also, plugins are not currently supported")
+        try XCTSkipSwiftBuildTodo(because: "Test fails on inability to find libclang on Linux. Also, plugins are not currently supported")
     }
 #endif
 }

Tests/CommandsTests/RunCommandTests.swift

@@ -255,18 +255,18 @@ class RunCommandSwiftBuildTests: RunCommandTestCase {
     }
 
     override func testMultipleExecutableAndExplicitExecutable() async throws {
-        throw XCTSkip("SWBINTTODO: https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
+        try XCTSkipSwiftBuildTodo(because: "https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
     }
 
     override func testUnknownProductAndArgumentPassing() async throws {
-        throw XCTSkip("SWBINTTODO: https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
+        try XCTSkipSwiftBuildTodo(because: "https://github.com/swiftlang/swift-package-manager/issues/8279: Swift run using Swift Build does not output executable content to the terminal")
     }
 
     override func testToolsetDebugger() async throws {
-        throw XCTSkip("SWBINTTODO: Test fixture fails to build")
+        try XCTSkipSwiftBuildTodo(because: "Test fixture fails to build")
     }
 
     override func testUnreachableExecutable() async throws {
-        throw XCTSkip("SWBINTTODO: Test fails because of build layout differences.")
+        try XCTSkipSwiftBuildTodo(because: "Test fails because of build layout differences.")
     }
 }