Merge pull request #1592 from ahoppen/6.0/no-lazy-prepartion

[6.0] Add option to allow SourceKit-LSP to prepare a target without lazy type checking

Author: ahoppen

Documentation/Configuration File.md

@@ -38,5 +38,7 @@ The structure of the file is currently not guaranteed to be stable. Options may
     - `updateIndexStoreTimeout: int`: Number of seconds to wait for an update index store task to finish before killing it.
 - `defaultWorkspaceType: "buildserver"|"compdb"|"swiftpm"`: Overrides workspace type selection logic.
 - `generatedFilesPath: string`: Directory in which generated interfaces and macro expansions should be stored.
+- `backgroundIndexing: bool`: Explicitly enable or disable background indexing.
+- `backgroundPreparationMode: "build"|"noLazy"|"enabled"`: Determines how background indexing should prepare a target. Possible values are: `build`: Build a target to prepare it, `noLazy`: Prepare a target without generating object files but do not do lazy type checking and function body skipping, `enabled`: Prepare a target without generating object files and the like
 - `experimentalFeatures: string[]`: Experimental features to enable
 - `swiftPublishDiagnosticsDebounce`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.

Documentation/Enable Experimental Background Indexing.md

@@ -4,9 +4,11 @@ Background indexing in SourceKit-LSP is available as an experimental feature. Th
 
 ## Set Up
 
-1. Install a `main` or `release/6.0` Swift Development Snapshot from https://www.swift.org/install.
+1. Install a `main` or `release/6.0` Swift Development Snapshot from https://www.swift.org/install or install the [Xcode 16 beta](https://developer.apple.com/xcode/).
 2. Point your editor to the newly installed toolchain.
-   - In VS Code on macOS this can be done by adding the following to your `settings.json`: `"swift.path": "/Library/Developer/Toolchains/swift-latest.xctoolchain/usr/bin"`
+   - In VS Code on macOS this can be done by adding the following to your `settings.json`:
+     - For open source toolchains `"swift.path": "/Library/Developer/Toolchains/swift-latest.xctoolchain/usr/bin"`
+     - When installing the Xcode 16 beta `"swift.path": "/Applications/Xcode-beta.app/Library/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin"`
    - In VS Code on other platforms, you need to set the `swift.path` to the `usr/bin` directory of your toolchain’s install location.
    - Other editors likely also have a way to pick the Swift toolchain, the exact steps vary by your setup.
 3. Enable the experimental `background-indexing` feature.
@@ -16,14 +18,10 @@ Background indexing in SourceKit-LSP is available as an experimental feature. Th
 ## Known issues
 
 - Background Indexing is only supported for SwiftPM projects [#1269](https://github.com/swiftlang/sourcekit-lsp/issues/1269), [#1271](https://github.com/swiftlang/sourcekit-lsp/issues/1271)
-- If a module or one of its dependencies has a compilation error, it cannot be properly prepared for indexing because we are running a regular `swift build` to generate its modules [#1254](https://github.com/swiftlang/sourcekit-lsp/issues/1254) rdar://128683404
-  - Workaround 1: Ensure that your files dependencies are in a buildable state to get an up-to-date index and proper cross-module functionality
-  - Workaround 2: Enable the `swiftpm-prepare-for-indexing` experimental feature, which continues to build Swift module even in the presence of errors.
 - If you change a function in a way that changes its USR but keeps it API compatible (such as adding a defaulted parameter), references to it will be lost and not re-indexed automatically [#1264](https://github.com/swiftlang/sourcekit-lsp/issues/1264)
   - Workaround: Make some edit to the files that had references to re-index them
 - The index build is currently completely separate from the command line build generated using `swift build`. Building *does not* update the index (break your habits of always building!) [#1270](https://github.com/swiftlang/sourcekit-lsp/issues/1270)
-- The initial indexing might take 2-3x more time than a regular build [#1254](https://github.com/swiftlang/sourcekit-lsp/issues/1254), [#1262](https://github.com/swiftlang/sourcekit-lsp/issues/1262), [#1268](https://github.com/swiftlang/sourcekit-lsp/issues/1268)
-- Spurious re-indexing of ~10-20 source files when `swift build` writes a header to the build directory [rdar://128573306](rdar://128573306)
+- The initial indexing might take 2-3x more time than a regular build [#1262](https://github.com/swiftlang/sourcekit-lsp/issues/1262), [#1268](https://github.com/swiftlang/sourcekit-lsp/issues/1268)
 
 ## Filing issues
 

Sources/SKCore/ExperimentalFeatures.swift

@@ -13,6 +13,5 @@
 /// An experimental feature that can be enabled by passing `--experimental-feature` to `sourcekit-lsp` on the command
 /// line. The raw value of this feature is how it is named on the command line.
 public enum ExperimentalFeature: String, Codable, Sendable, CaseIterable {
-  /// Add `--experimental-prepare-for-indexing` to the `swift build` command run to prepare a target for indexing.
-  case swiftpmPrepareForIndexing = "swiftpm-prepare-for-indexing"
+  case dummy
 }

Sources/SKCore/SourceKitLSPOptions.swift

@@ -178,6 +178,19 @@ public struct SourceKitLSPOptions: Sendable, Codable {
     }
   }
 
+  public enum BackgroundPreparationMode: String {
+    /// Build a target to prepare it
+    case build
+
+    /// Prepare a target without generating object files but do not do lazy type checking.
+    ///
+    /// This uses SwiftPM's `--experimental-prepare-for-indexing-no-lazy` flag.
+    case noLazy
+
+    /// Prepare a target without generating object files.
+    case enabled
+  }
+
   public var swiftPM: SwiftPMOptions
   public var compilationDatabase: CompilationDatabaseOptions
   public var fallbackBuildSystem: FallbackBuildSystemOptions
@@ -195,6 +208,15 @@ public struct SourceKitLSPOptions: Sendable, Codable {
     return backgroundIndexing ?? false
   }
 
+  public var backgroundPreparationMode: String?
+
+  public var backgroundPreparationModeOrDefault: BackgroundPreparationMode {
+    if let backgroundPreparationMode, let parsed = BackgroundPreparationMode(rawValue: backgroundPreparationMode) {
+      return parsed
+    }
+    return .build
+  }
+
   /// Experimental features that are enabled.
   public var experimentalFeatures: Set<ExperimentalFeature>? = nil
 
@@ -250,6 +272,7 @@ public struct SourceKitLSPOptions: Sendable, Codable {
     defaultWorkspaceType: WorkspaceType? = nil,
     generatedFilesPath: String? = nil,
     backgroundIndexing: Bool? = nil,
+    backgroundPreparationMode: String? = nil,
     experimentalFeatures: Set<ExperimentalFeature>? = nil,
     swiftPublishDiagnosticsDebounceDuration: Double? = nil,
     workDoneProgressDebounceDuration: Double? = nil,
@@ -263,6 +286,7 @@ public struct SourceKitLSPOptions: Sendable, Codable {
     self.generatedFilesPath = generatedFilesPath
     self.defaultWorkspaceType = defaultWorkspaceType
     self.backgroundIndexing = backgroundIndexing
+    self.backgroundPreparationMode = backgroundPreparationMode
     self.experimentalFeatures = experimentalFeatures
     self.swiftPublishDiagnosticsDebounceDuration = swiftPublishDiagnosticsDebounceDuration
     self.workDoneProgressDebounceDuration = workDoneProgressDebounceDuration
@@ -315,6 +339,7 @@ public struct SourceKitLSPOptions: Sendable, Codable {
       defaultWorkspaceType: override?.defaultWorkspaceType ?? base.defaultWorkspaceType,
       generatedFilesPath: override?.generatedFilesPath ?? base.generatedFilesPath,
       backgroundIndexing: override?.backgroundIndexing ?? base.backgroundIndexing,
+      backgroundPreparationMode: override?.backgroundPreparationMode ?? base.backgroundPreparationMode,
       experimentalFeatures: override?.experimentalFeatures ?? base.experimentalFeatures,
       swiftPublishDiagnosticsDebounceDuration: override?.swiftPublishDiagnosticsDebounceDuration
         ?? base.swiftPublishDiagnosticsDebounceDuration,

Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift

@@ -637,8 +637,10 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem {
     arguments += options.swiftPM.cxxCompilerFlags?.flatMap { ["-Xcxx", $0] } ?? []
     arguments += options.swiftPM.swiftCompilerFlags?.flatMap { ["-Xswiftc", $0] } ?? []
     arguments += options.swiftPM.linkerFlags?.flatMap { ["-Xlinker", $0] } ?? []
-    if options.hasExperimentalFeature(.swiftpmPrepareForIndexing) {
-      arguments.append("--experimental-prepare-for-indexing")
+    switch options.backgroundPreparationModeOrDefault {
+    case .build: break
+    case .noLazy: arguments += ["--experimental-prepare-for-indexing", "--experimental-prepare-for-indexing-no-lazy"]
+    case .enabled: arguments.append("--experimental-prepare-for-indexing")
     }
     if Task.isCancelled {
       return

Tests/SourceKitLSPTests/BackgroundIndexingTests.swift

@@ -1066,7 +1066,55 @@ final class BackgroundIndexingTests: XCTestCase {
 
   func testCrossModuleFunctionalityEvenIfLowLevelModuleHasErrors() async throws {
     try await SkipUnless.swiftPMSupportsExperimentalPrepareForIndexing()
-    let options = SourceKitLSPOptions.testDefault(experimentalFeatures: [.swiftpmPrepareForIndexing])
+    var options = SourceKitLSPOptions.testDefault()
+    options.backgroundPreparationMode = SourceKitLSPOptions.BackgroundPreparationMode.enabled.rawValue
+    let project = try await SwiftPMTestProject(
+      files: [
+        "LibA/LibA.swift": """
+        public func test() -> Invalid {
+          return ""
+        }
+        """,
+        "LibB/LibB.swift": """
+        import LibA
+
+        public func 1️⃣libBTest() -> Int {
+          return libATest()
+        }
+        """,
+        "MyExec/MyExec.swift": """
+        import LibB
+
+        func test() -> Int {
+          return 2️⃣libBTest()
+        }
+        """,
+      ],
+      manifest: """
+        let package = Package(
+          name: "MyLibrary",
+          targets: [
+           .target(name: "LibA"),
+           .target(name: "LibB", dependencies: ["LibA"]),
+           .executableTarget(name: "MyExec", dependencies: ["LibB"]),
+          ]
+        )
+        """,
+      options: options,
+      enableBackgroundIndexing: true
+    )
+
+    let (uri, positions) = try project.openDocument("MyExec.swift")
+    let response = try await project.testClient.send(
+      DefinitionRequest(textDocument: TextDocumentIdentifier(uri), position: positions["2️⃣"])
+    )
+    XCTAssertEqual(response, .locations([try project.location(from: "1️⃣", to: "1️⃣", in: "LibB.swift")]))
+  }
+
+  func testCrossModuleFunctionalityWithPreparationNoSkipping() async throws {
+    try await SkipUnless.swiftPMSupportsExperimentalPrepareForIndexing()
+    var options = SourceKitLSPOptions.testDefault()
+    options.backgroundPreparationMode = SourceKitLSPOptions.BackgroundPreparationMode.noLazy.rawValue
     let project = try await SwiftPMTestProject(
       files: [
         "LibA/LibA.swift": """
@@ -1250,7 +1298,8 @@ final class BackgroundIndexingTests: XCTestCase {
     try await SkipUnless.swiftPMSupportsExperimentalPrepareForIndexing()
     try SkipUnless.longTestsEnabled()
 
-    var options = SourceKitLSPOptions.testDefault(experimentalFeatures: [.swiftpmPrepareForIndexing])
+    var options = SourceKitLSPOptions.testDefault()
+    options.backgroundPreparationMode = SourceKitLSPOptions.BackgroundPreparationMode.enabled.rawValue
     options.index.updateIndexStoreTimeout = 1 /* second */
 
     let dateStarted = Date()