If sourcekitd or clangd don’t respond to a request for 5 minutes, terminate them and use crash recovery to restore behavior

This should be a last stop-gap measure in case sourcekitd or clangd get stuck, don’t respond to any requests anymore and don’t honor cancellation either. In that case we can restore SourceKit-LSP behavior by killing them and using the crash recovery logic to restore functionality.

rdar://149492159

Author: ahoppen

Documentation/Configuration File.md

@@ -58,3 +58,4 @@ The structure of the file is currently not guaranteed to be stable. Options may
 - `swiftPublishDiagnosticsDebounceDuration: number`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.
 - `workDoneProgressDebounceDuration: number`: When a task is started that should be displayed to the client as a work done progress, how many milliseconds to wait before actually starting the work done progress. This prevents flickering of the work done progress in the client for short-lived index tasks which end within this duration.
 - `sourcekitdRequestTimeout: number`: The maximum duration that a sourcekitd request should be allowed to execute before being declared as timed out. In general, editors should cancel requests that they are no longer interested in, but in case editors don't cancel requests, this ensures that a long-running non-cancelled request is not blocking sourcekitd and thus most semantic functionality. In particular, VS Code does not cancel the semantic tokens request, which can cause a long-running AST build that blocks sourcekitd.
+- `semanticServiceRestartTimeout: number`: If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging for some reason and won't recover. To restore semantic functionality, we terminate and restart it.

Sources/BuildSystemIntegration/ExternalBuildSystemAdapter.swift

@@ -185,11 +185,11 @@ actor ExternalBuildSystemAdapter {
       protocol: bspRegistry,
       stderrLoggingCategory: "bsp-server-stderr",
       client: messagesToSourceKitLSPHandler,
-      terminationHandler: { [weak self] terminationStatus in
+      terminationHandler: { [weak self] terminationReason in
         guard let self else {
           return
         }
-        if terminationStatus != 0 {
+        if terminationReason != .exited(exitCode: 0) {
           Task {
             await orLog("Restarting BSP server") {
               try await self.handleBspServerCrash()

Sources/LanguageServerProtocolJSONRPC/JSONRPCConnection.swift

@@ -31,6 +31,14 @@ import struct CDispatch.dispatch_fd_t
 /// For example, inside a language server, the `JSONRPCConnection` takes the language service implementation as its
 // `receiveHandler` and itself provides the client connection for sending notifications and callbacks.
 public final class JSONRPCConnection: Connection {
+  public enum TerminationReason: Sendable, Equatable {
+    /// The process on the other end of the `JSONRPCConnection` terminated with the given exit code.
+    case exited(exitCode: Int32)
+
+    /// The process on the other end of the `JSONRPCConnection` terminated with a signal. The signal that it terminated
+    /// with is not known.
+    case uncaughtSignal
+  }
 
   /// A name of the endpoint for this connection, used for logging, e.g. `clangd`.
   private let name: String
@@ -198,7 +206,7 @@ public final class JSONRPCConnection: Connection {
     protocol messageRegistry: MessageRegistry,
     stderrLoggingCategory: String,
     client: MessageHandler,
-    terminationHandler: @Sendable @escaping (_ terminationStatus: Int32) -> Void
+    terminationHandler: @Sendable @escaping (_ terminationReason: TerminationReason) -> Void
   ) throws -> (connection: JSONRPCConnection, process: Process) {
     let clientToServer = Pipe()
     let serverToClient = Pipe()
@@ -238,10 +246,22 @@ public final class JSONRPCConnection: Connection {
     process.terminationHandler = { process in
       logger.log(
         level: process.terminationReason == .exit ? .default : .error,
-        "\(name) exited: \(String(reflecting: process.terminationReason)) \(process.terminationStatus)"
+        "\(name) exited: \(process.terminationReason.rawValue) \(process.terminationStatus)"
       )
       connection.close()
-      terminationHandler(process.terminationStatus)
+      let terminationReason: TerminationReason
+      switch process.terminationReason {
+      case .exit:
+        terminationReason = .exited(exitCode: process.terminationStatus)
+      case .uncaughtSignal:
+        terminationReason = .uncaughtSignal
+      @unknown default:
+        logger.fault(
+          "Process terminated with unknown termination reason: \(process.terminationReason.rawValue, privacy: .public)"
+        )
+        terminationReason = .exited(exitCode: 0)
+      }
+      terminationHandler(terminationReason)
     }
     try process.run()
 

Sources/SKOptions/SourceKitLSPOptions.swift

@@ -422,6 +422,17 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     return .seconds(120)
   }
 
+  /// If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging
+  /// for some reason and won't recover. To restore semantic functionality, we terminate and restart it.
+  public var semanticServiceRestartTimeout: Double? = nil
+
+  public var semanticServiceRestartTimeoutOrDefault: Duration {
+    if let semanticServiceRestartTimeout {
+      return .seconds(semanticServiceRestartTimeout)
+    }
+    return .seconds(300)
+  }
+
   public init(
     swiftPM: SwiftPMOptions? = .init(),
     fallbackBuildSystem: FallbackBuildSystemOptions? = .init(),
@@ -439,7 +450,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     experimentalFeatures: Set<ExperimentalFeature>? = nil,
     swiftPublishDiagnosticsDebounceDuration: Double? = nil,
     workDoneProgressDebounceDuration: Double? = nil,
-    sourcekitdRequestTimeout: Double? = nil
+    sourcekitdRequestTimeout: Double? = nil,
+    semanticServiceRestartTimeout: Double? = nil
   ) {
     self.swiftPM = swiftPM
     self.fallbackBuildSystem = fallbackBuildSystem
@@ -458,6 +470,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     self.swiftPublishDiagnosticsDebounceDuration = swiftPublishDiagnosticsDebounceDuration
     self.workDoneProgressDebounceDuration = workDoneProgressDebounceDuration
     self.sourcekitdRequestTimeout = sourcekitdRequestTimeout
+    self.semanticServiceRestartTimeout = semanticServiceRestartTimeout
   }
 
   public init?(fromLSPAny lspAny: LSPAny?) throws {
@@ -517,7 +530,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
         ?? base.swiftPublishDiagnosticsDebounceDuration,
       workDoneProgressDebounceDuration: override?.workDoneProgressDebounceDuration
         ?? base.workDoneProgressDebounceDuration,
-      sourcekitdRequestTimeout: override?.sourcekitdRequestTimeout ?? base.sourcekitdRequestTimeout
+      sourcekitdRequestTimeout: override?.sourcekitdRequestTimeout ?? base.sourcekitdRequestTimeout,
+      semanticServiceRestartTimeout: override?.semanticServiceRestartTimeout ?? base.semanticServiceRestartTimeout
     )
   }
 

Sources/SKTestSupport/SkipUnless.swift

@@ -265,8 +265,7 @@ package actor SkipUnless {
               sourcekitd.keys.useNewAPI: 1
             ],
           ]),
-          timeout: defaultTimeoutDuration,
-          fileContents: nil
+          timeout: defaultTimeoutDuration
         )
         return response[sourcekitd.keys.useNewAPI] == 1
       } catch {

Sources/SKTestSupport/SourceKitD+send.swift

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+package import SourceKitD
+
+extension SourceKitD {
+  /// Convenience overload of the `send` function for testing that doesn't restart sourcekitd if it does not respond
+  /// and doesn't pass any file contents.
+  package func send(
+    _ request: SKDRequestDictionary,
+    timeout: Duration
+  ) async throws -> SKDResponseDictionary {
+    return try await self.send(
+      request,
+      timeout: timeout,
+      restartTimeout: .seconds(60 * 60 * 24),
+      fileContents: nil
+    )
+  }
+}

Sources/SourceKitD/SourceKitD.swift

@@ -162,7 +162,10 @@ package actor SourceKitD {
   /// List of notification handlers that will be called for each notification.
   private var notificationHandlers: [WeakSKDNotificationHandler] = []
 
-  /// List of hooks that should be executed for every request sent to sourcekitd.
+  /// List of hooks that should be executed and that need to finish executing before a request is sent to sourcekitd.
+  private var preRequestHandlingHooks: [UUID: @Sendable (SKDRequestDictionary) async -> Void] = [:]
+
+  /// List of hooks that should be executed after a request sent to sourcekitd.
   private var requestHandlingHooks: [UUID: (SKDRequestDictionary) -> Void] = [:]
 
   package static func getOrCreate(
@@ -261,6 +264,30 @@ package actor SourceKitD {
     notificationHandlers.removeAll(where: { $0.value == nil || $0.value === handler })
   }
 
+  /// Execute `body` and invoke `hook` for every sourcekitd request that is sent during the execution time of `body`.
+  ///
+  /// Note that `hook` will not only be executed for requests sent *by* body but this may also include sourcekitd
+  /// requests that were sent by other clients of the same `DynamicallyLoadedSourceKitD` instance that just happen to
+  /// send a request during that time.
+  ///
+  /// This is intended for testing only.
+  package func withPreRequestHandlingHook(
+    body: () async throws -> Void,
+    hook: @escaping @Sendable (SKDRequestDictionary) async -> Void
+  ) async rethrows {
+    let id = UUID()
+    preRequestHandlingHooks[id] = hook
+    defer { preRequestHandlingHooks[id] = nil }
+    try await body()
+  }
+
+  func willSend(request: SKDRequestDictionary) async {
+    let request = request
+    for hook in preRequestHandlingHooks.values {
+      await hook(request)
+    }
+  }
+
   /// Execute `body` and invoke `hook` for every sourcekitd request that is sent during the execution time of `body`.
   ///
   /// Note that `hook` will not only be executed for requests sent *by* body but this may also include sourcekitd
@@ -293,37 +320,56 @@ package actor SourceKitD {
   package func send(
     _ request: SKDRequestDictionary,
     timeout: Duration,
+    restartTimeout: Duration,
     fileContents: String?
   ) async throws -> SKDResponseDictionary {
     let sourcekitdResponse = try await withTimeout(timeout) {
-      return try await withCancellableCheckedThrowingContinuation { (continuation) -> SourceKitDRequestHandle? in
-        logger.info(
-          """
-          Sending sourcekitd request:
-          \(request.forLogging)
-          """
-        )
-        var handle: sourcekitd_api_request_handle_t? = nil
-        self.api.send_request(request.dict, &handle) { response in
-          continuation.resume(returning: SKDResponse(response!, sourcekitd: self))
-        }
-        Task {
-          await self.didSend(request: request)
-        }
-        if let handle {
-          return SourceKitDRequestHandle(handle: handle)
+      let restartTimeoutHandle = TimeoutHandle()
+      do {
+        return try await withTimeout(restartTimeout, handle: restartTimeoutHandle) {
+          await self.willSend(request: request)
+          return try await withCancellableCheckedThrowingContinuation { (continuation) -> SourceKitDRequestHandle? in
+            logger.info(
+              """
+              Sending sourcekitd request:
+              \(request.forLogging)
+              """
+            )
+            var handle: sourcekitd_api_request_handle_t? = nil
+            self.api.send_request(request.dict, &handle) { response in
+              continuation.resume(returning: SKDResponse(response!, sourcekitd: self))
+            }
+            Task {
+              await self.didSend(request: request)
+            }
+            if let handle {
+              return SourceKitDRequestHandle(handle: handle)
+            }
+            return nil
+          } cancel: { (handle: SourceKitDRequestHandle?) in
+            if let handle {
+              logger.info(
+                """
+                Cancelling sourcekitd request:
+                \(request.forLogging)
+                """
+              )
+              self.api.cancel_request(handle.handle)
+            }
+          }
         }
-        return nil
-      } cancel: { (handle: SourceKitDRequestHandle?) in
-        if let handle {
-          logger.info(
-            """
-            Cancelling sourcekitd request:
-            \(request.forLogging)
-            """
+      } catch let error as TimeoutError where error.handle == restartTimeoutHandle {
+        if !self.path.lastPathComponent.contains("InProc") {
+          logger.fault(
+            "Did not receive reply from sourcekitd after \(restartTimeout, privacy: .public). Terminating and restarting sourcekitd."
+          )
+          await self.crash()
+        } else {
+          logger.fault(
+            "Did not receive reply from sourcekitd after \(restartTimeout, privacy: .public). Not terminating sourcekitd because it is run in-process."
           )
-          self.api.cancel_request(handle.handle)
         }
+        throw error
       }
     }
 
@@ -362,6 +408,13 @@ package actor SourceKitD {
 
     return dict
   }
+
+  package func crash() async {
+    let req = dictionary([
+      keys.request: requests.crashWithExit
+    ])
+    _ = try? await send(req, timeout: .seconds(60), restartTimeout: .seconds(24 * 60 * 60), fileContents: nil)
+  }
 }
 
 /// A sourcekitd notification handler in a class to allow it to be uniquely referenced.

Sources/SourceKitD/SourceKitDRegistry.swift

@@ -30,7 +30,7 @@ package actor SourceKitDRegistry<SourceKitDType: AnyObject> {
   private var active: [URL: (pluginPaths: PluginPaths?, sourcekitd: SourceKitDType)] = [:]
 
   /// Instances that have been unregistered, but may be resurrected if accessed before destruction.
-  private var cemetary: [URL: (pluginPaths: PluginPaths?, sourcekitd: WeakSourceKitD<SourceKitDType>)] = [:]
+  private var cemetery: [URL: (pluginPaths: PluginPaths?, sourcekitd: WeakSourceKitD<SourceKitDType>)] = [:]
 
   /// Initialize an empty registry.
   package init() {}
@@ -49,8 +49,8 @@ package actor SourceKitDRegistry<SourceKitDType: AnyObject> {
       }
       return existing.sourcekitd
     }
-    if let resurrected = cemetary[key], let resurrectedSourcekitD = resurrected.sourcekitd.value {
-      cemetary[key] = nil
+    if let resurrected = cemetery[key], let resurrectedSourcekitD = resurrected.sourcekitd.value {
+      cemetery[key] = nil
       if resurrected.pluginPaths != pluginPaths {
         logger.fault(
           "Already created SourceKitD with plugin paths \(resurrected.pluginPaths?.forLogging), now requesting incompatible plugin paths \(pluginPaths.forLogging)"
@@ -74,8 +74,8 @@ package actor SourceKitDRegistry<SourceKitDType: AnyObject> {
   package func remove(_ key: URL) -> SourceKitDType? {
     let existing = active.removeValue(forKey: key)
     if let existing = existing {
-      assert(self.cemetary[key]?.sourcekitd.value == nil)
-      cemetary[key] = (existing.pluginPaths, WeakSourceKitD(value: existing.sourcekitd))
+      assert(self.cemetery[key]?.sourcekitd.value == nil)
+      cemetery[key] = (existing.pluginPaths, WeakSourceKitD(value: existing.sourcekitd))
     }
     return existing?.sourcekitd
   }

Sources/SourceKitLSP/Clang/ClangLanguageService.swift

@@ -62,7 +62,7 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   /// Path to the `clangd` binary.
   let clangdPath: URL
 
-  let clangdOptions: [String]
+  let options: SourceKitLSPOptions
 
   /// The current state of the `clangd` language server.
   /// Changing the property automatically notified the state change handlers.
@@ -117,7 +117,7 @@ actor ClangLanguageService: LanguageService, MessageHandler {
     }
     self.clangPath = toolchain.clang
     self.clangdPath = clangdPath
-    self.clangdOptions = options.clangdOptions ?? []
+    self.options = options
     self.workspace = WeakWorkspace(workspace)
     self.state = .connected
     self.sourceKitLSPServer = sourceKitLSPServer
@@ -154,10 +154,11 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   /// Restarts `clangd` if it has crashed.
   ///
   /// - Parameter terminationStatus: The exit code of `clangd`.
-  private func handleClangdTermination(terminationStatus: Int32) {
+  private func handleClangdTermination(terminationReason: JSONRPCConnection.TerminationReason) {
     self.clangdProcess = nil
-    if terminationStatus != 0 {
+    if terminationReason != .exited(exitCode: 0) {
       self.state = .connectionInterrupted
+      logger.info("clangd crashed. Restarting it.")
       self.restartClangd()
     }
   }
@@ -173,15 +174,15 @@ actor ClangLanguageService: LanguageService, MessageHandler {
         "-compile_args_from=lsp",  // Provide compiler args programmatically.
         "-background-index=false",  // Disable clangd indexing, we use the build
         "-index=false",  // system index store instead.
-      ] + clangdOptions,
+      ] + (options.clangdOptions ?? []),
       name: "clangd",
       protocol: MessageRegistry.lspProtocol,
       stderrLoggingCategory: "clangd-stderr",
       client: self,
-      terminationHandler: { [weak self] terminationStatus in
+      terminationHandler: { [weak self] terminationReason in
         guard let self = self else { return }
         Task {
-          await self.handleClangdTermination(terminationStatus: terminationStatus)
+          await self.handleClangdTermination(terminationReason: terminationReason)
         }
 
       }
@@ -291,14 +292,20 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   }
 
   /// Forward the given request to `clangd`.
-  ///
-  /// This method calls `readyToHandleNextRequest` once the request has been
-  /// transmitted to `clangd` and another request can be safely transmitted to
-  /// `clangd` while guaranteeing ordering.
-  ///
-  /// The response of the request is  returned asynchronously as the return value.
   func forwardRequestToClangd<R: RequestType>(_ request: R) async throws -> R.Response {
-    return try await clangd.send(request)
+    let timeoutHandle = TimeoutHandle()
+    do {
+      return try await withTimeout(options.semanticServiceRestartTimeoutOrDefault, handle: timeoutHandle) {
+        await self.sourceKitLSPServer?.hooks.preForwardRequestToClangd?(request)
+        return try await self.clangd.send(request)
+      }
+    } catch let error as TimeoutError where error.handle == timeoutHandle {
+      logger.fault(
+        "Did not receive reply from clangd after \(options.semanticServiceRestartTimeoutOrDefault, privacy: .public). Terminating and restarting clangd."
+      )
+      self.crash()
+      throw error
+    }
   }
 
   package func canonicalDeclarationPosition(of position: Position, in uri: DocumentURI) async -> Position? {

Sources/SourceKitLSP/Hooks.swift

@@ -30,17 +30,24 @@ public struct Hooks: Sendable {
   /// This allows requests to be artificially delayed.
   package var preHandleRequest: (@Sendable (any RequestType) async -> Void)?
 
+  /// Closure that is executed before a request is forwarded to clangd.
+  ///
+  /// This allows tests to simulate a `clangd` process that's unresponsive.
+  package var preForwardRequestToClangd: (@Sendable (any RequestType) async -> Void)?
+
   public init() {
     self.init(indexHooks: IndexHooks(), buildSystemHooks: BuildSystemHooks())
   }
 
   package init(
     indexHooks: IndexHooks = IndexHooks(),
     buildSystemHooks: BuildSystemHooks = BuildSystemHooks(),
-    preHandleRequest: (@Sendable (any RequestType) async -> Void)? = nil
+    preHandleRequest: (@Sendable (any RequestType) async -> Void)? = nil,
+    preForwardRequestToClangd: (@Sendable (any RequestType) async -> Void)? = nil
   ) {
     self.indexHooks = indexHooks
     self.buildSystemHooks = buildSystemHooks
     self.preHandleRequest = preHandleRequest
+    self.preForwardRequestToClangd = preForwardRequestToClangd
   }
 }

Sources/SourceKitLSP/Swift/CodeCompletionSession.swift

@@ -201,6 +201,7 @@ class CodeCompletionSession {
       return await Self.resolveDocumentation(
         in: item,
         timeout: session.options.sourcekitdRequestTimeoutOrDefault,
+        restartTimeout: session.options.semanticServiceRestartTimeoutOrDefault,
         sourcekitd: sourcekitd
       )
     }
@@ -287,11 +288,7 @@ class CodeCompletionSession {
       keys.codeCompleteOptions: optionsDictionary(filterText: filterText),
     ])
 
-    let dict = try await sourcekitd.send(
-      req,
-      timeout: options.sourcekitdRequestTimeoutOrDefault,
-      fileContents: snapshot.text
-    )
+    let dict = try await sendSourceKitdRequest(req, snapshot: snapshot)
     self.state = .open
 
     guard let completions: SKDResponseArray = dict[keys.results] else {
@@ -325,11 +322,7 @@ class CodeCompletionSession {
       keys.codeCompleteOptions: optionsDictionary(filterText: filterText),
     ])
 
-    let dict = try await sourcekitd.send(
-      req,
-      timeout: options.sourcekitdRequestTimeoutOrDefault,
-      fileContents: snapshot.text
-    )
+    let dict = try await sendSourceKitdRequest(req, snapshot: snapshot)
     guard let completions: SKDResponseArray = dict[keys.results] else {
       return CompletionList(isIncomplete: false, items: [])
     }
@@ -378,13 +371,25 @@ class CodeCompletionSession {
         keys.codeCompleteOptions: [keys.useNewAPI: 1],
       ])
       logger.info("Closing code completion session: \(self.description)")
-      _ = try? await sourcekitd.send(req, timeout: options.sourcekitdRequestTimeoutOrDefault, fileContents: nil)
+      _ = try? await sendSourceKitdRequest(req, snapshot: nil)
       self.state = .closed
     }
   }
 
   // MARK: - Helpers
 
+  private func sendSourceKitdRequest(
+    _ request: SKDRequestDictionary,
+    snapshot: DocumentSnapshot?
+  ) async throws -> SKDResponseDictionary {
+    try await sourcekitd.send(
+      request,
+      timeout: options.sourcekitdRequestTimeoutOrDefault,
+      restartTimeout: options.semanticServiceRestartTimeoutOrDefault,
+      fileContents: snapshot?.text
+    )
+  }
+
   private func expandClosurePlaceholders(insertText: String) -> String? {
     guard insertText.contains("<#") && insertText.contains("->") else {
       // Fast path: There is no closure placeholder to expand
@@ -532,8 +537,14 @@ class CodeCompletionSession {
     }
 
     if !clientSupportsDocumentationResolve {
+      let semanticServiceRestartTimeoutOrDefault = self.options.semanticServiceRestartTimeoutOrDefault
       completionItems = await completionItems.asyncMap { item in
-        return await Self.resolveDocumentation(in: item, timeout: .seconds(1), sourcekitd: sourcekitd)
+        return await Self.resolveDocumentation(
+          in: item,
+          timeout: .seconds(1),
+          restartTimeout: semanticServiceRestartTimeoutOrDefault,
+          sourcekitd: sourcekitd
+        )
       }
     }
 
@@ -543,6 +554,7 @@ class CodeCompletionSession {
   private static func resolveDocumentation(
     in item: CompletionItem,
     timeout: Duration,
+    restartTimeout: Duration,
     sourcekitd: SourceKitD
   ) async -> CompletionItem {
     var item = item
@@ -552,7 +564,7 @@ class CodeCompletionSession {
         sourcekitd.keys.identifier: itemId,
       ])
       let documentationResponse = await orLog("Retrieving documentation for completion item") {
-        try await sourcekitd.send(req, timeout: timeout, fileContents: nil)
+        try await sourcekitd.send(req, timeout: timeout, restartTimeout: restartTimeout, fileContents: nil)
       }
       if let docString: String = documentationResponse?[sourcekitd.keys.docBrief] {
         item.documentation = .markupContent(MarkupContent(kind: .markdown, value: docString))

Sources/SourceKitLSP/Swift/DiagnosticReportManager.swift

@@ -124,6 +124,7 @@ actor DiagnosticReportManager {
       dict = try await self.sourcekitd.send(
         skreq,
         timeout: options.sourcekitdRequestTimeoutOrDefault,
+        restartTimeout: options.semanticServiceRestartTimeoutOrDefault,
         fileContents: snapshot.text
       )
     } catch SKDError.requestFailed(let sourcekitdError) {

Sources/SourceKitLSP/Swift/SwiftLanguageService.swift

@@ -101,7 +101,7 @@ package actor SwiftLanguageService: LanguageService, Sendable {
 
   private let sourcekitdPath: URL
 
-  let sourcekitd: SourceKitD
+  package let sourcekitd: SourceKitD
 
   /// Path to the swift-format executable if it exists in the toolchain.
   let swiftFormat: URL?
@@ -307,6 +307,7 @@ package actor SwiftLanguageService: LanguageService, Sendable {
     try await sourcekitd.send(
       request,
       timeout: options.sourcekitdRequestTimeoutOrDefault,
+      restartTimeout: options.semanticServiceRestartTimeoutOrDefault,
       fileContents: fileContents
     )
   }

Sources/SwiftExtensions/AsyncUtils.swift

@@ -181,12 +181,30 @@ extension Collection where Element: Sendable {
 package struct TimeoutError: Error, CustomStringConvertible {
   package var description: String { "Timed out" }
 
+  package let handle: TimeoutHandle?
+
+  package init(handle: TimeoutHandle?) {
+    self.handle = handle
+  }
+}
+
+package final class TimeoutHandle: Equatable, Sendable {
   package init() {}
+
+  static package func == (_ lhs: TimeoutHandle, _ rhs: TimeoutHandle) -> Bool {
+    return lhs === rhs
+  }
 }
 
-/// Executes `body`. If it doesn't finish after `duration`, throws a `TimeoutError`.
+/// Executes `body`. If it doesn't finish after `duration`, throws a `TimeoutError` and cancels `body`.
+///
+/// `TimeoutError` is thrown immediately an the function does not wait for `body` to honor the cancellation.
+///
+/// If a `handle` is passed in and this `withTimeout` call times out, the thrown `TimeoutError` contains this handle.
+/// This way a caller can identify whether this call to `withTimeout` timed out or if a nested call timed out.
 package func withTimeout<T: Sendable>(
   _ duration: Duration,
+  handle: TimeoutHandle? = nil,
   _ body: @escaping @Sendable () async throws -> T
 ) async throws -> T {
   // Get the priority with which to launch the body task here so that we can pass the same priority as the initial
@@ -207,7 +225,7 @@ package func withTimeout<T: Sendable>(
 
     let timeoutTask = Task(priority: priority) {
       try await Task.sleep(for: duration)
-      continuation.yield(with: .failure(TimeoutError()))
+      continuation.yield(with: .failure(TimeoutError(handle: handle)))
       bodyTask.cancel()
     }
     mutableTasks = [bodyTask, timeoutTask]

Tests/SourceKitDTests/SourceKitDTests.swift

@@ -84,15 +84,15 @@ final class SourceKitDTests: XCTestCase {
       keys.compilerArgs: args,
     ])
 
-    _ = try await sourcekitd.send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await sourcekitd.send(req, timeout: defaultTimeoutDuration)
 
     try await fulfillmentOfOrThrow(expectation1, expectation2)
 
     let close = sourcekitd.dictionary([
       keys.request: sourcekitd.requests.editorClose,
       keys.name: path,
     ])
-    _ = try await sourcekitd.send(close, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await sourcekitd.send(close, timeout: defaultTimeoutDuration)
   }
 }
 

Tests/SourceKitLSPTests/ClangdTests.swift

@@ -11,7 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 import LanguageServerProtocol
+import SKLogging
+import SKOptions
 import SKTestSupport
+import SourceKitLSP
+import SwiftExtensions
+import TSCBasic
 import XCTest
 
 final class ClangdTests: XCTestCase {
@@ -127,4 +132,58 @@ final class ClangdTests: XCTestCase {
       }
     }
   }
+
+  func testRestartClangdIfItDoesntReply() async throws {
+    // We simulate clangd not replying until it is restarted using a hook.
+    let clangdRestarted = AtomicBool(initialValue: false)
+    let clangdRestartedExpectation = self.expectation(description: "clangd restarted")
+    let hooks = Hooks(preForwardRequestToClangd: { request in
+      if !clangdRestarted.value {
+        try? await Task.sleep(for: .seconds(60 * 60))
+      }
+    })
+
+    let testClient = try await TestSourceKitLSPClient(
+      options: SourceKitLSPOptions(semanticServiceRestartTimeout: 1),
+      hooks: hooks
+    )
+    let uri = DocumentURI(for: .c)
+    let positions = testClient.openDocument(
+      """
+      void test() {
+        int x1️⃣;
+      }
+      """,
+      uri: uri
+    )
+
+    // Monitor clangd to notice when it gets restarted
+    let clangdServer = try await unwrap(
+      testClient.server.languageService(for: uri, .c, in: unwrap(testClient.server.workspaceForDocument(uri: uri)))
+    )
+    await clangdServer.addStateChangeHandler { oldState, newState in
+      if oldState == .connectionInterrupted, newState == .connected {
+        clangdRestarted.value = true
+        clangdRestartedExpectation.fulfill()
+      }
+    }
+
+    // The first hover request should get cancelled by `semanticServiceRestartTimeout`
+    await assertThrowsError(
+      try await testClient.send(HoverRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"]))
+    ) { error in
+      XCTAssert(
+        (error as? ResponseError)?.message.contains("Timed out") ?? false,
+        "Received unexpected error: \(error)"
+      )
+    }
+
+    try await fulfillmentOfOrThrow(clangdRestartedExpectation)
+
+    // After clangd gets restarted
+    let hover = try await testClient.send(
+      HoverRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"])
+    )
+    assertContains(hover?.contents.markupContent?.value ?? "", "Type: int")
+  }
 }

Tests/SourceKitDTests/CrashRecoveryTests.swift renamed to Tests/SourceKitLSPTests/CrashRecoveryTests.swift

@@ -13,6 +13,7 @@
 import LanguageServerProtocol
 import LanguageServerProtocolExtensions
 import SKLogging
+import SKOptions
 import SKTestSupport
 import SourceKitD
 @_spi(Testing) import SourceKitLSP
@@ -349,4 +350,70 @@ final class CrashRecoveryTests: XCTestCase {
       return true
     }
   }
+
+  func testRestartSourceKitDIfItDoesntReply() async throws {
+    try SkipUnless.longTestsEnabled()
+    try SkipUnless.platformIsDarwin("Linux and Windows use in-process sourcekitd")
+
+    let sourcekitdTerminatedExpectation = self.expectation(description: "sourcekitd terminated")
+
+    let testClient = try await TestSourceKitLSPClient(options: SourceKitLSPOptions(semanticServiceRestartTimeout: 2))
+    let uri = DocumentURI(for: .swift)
+    let positions = testClient.openDocument(
+      """
+      func test() {
+        let 1️⃣x = 1
+      }
+      """,
+      uri: uri
+    )
+
+    // Monitor sourcekitd to notice when it gets terminated
+    let swiftService = try await unwrap(
+      testClient.server.languageService(for: uri, .swift, in: unwrap(testClient.server.workspaceForDocument(uri: uri)))
+        as? SwiftLanguageService
+    )
+    await swiftService.addStateChangeHandler { oldState, newState in
+      logger.debug("sourcekitd changed state: \(String(describing: oldState)) -> \(String(describing: newState))")
+      if newState == .connectionInterrupted {
+        sourcekitdTerminatedExpectation.fulfill()
+      }
+    }
+
+    try await swiftService.sourcekitd.withPreRequestHandlingHook {
+      // The first hover request should get cancelled by `semanticServiceRestartTimeout`
+      await assertThrowsError(
+        try await testClient.send(HoverRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"]))
+      ) { error in
+        XCTAssert(
+          (error as? ResponseError)?.message.contains("Timed out") ?? false,
+          "Received unexpected error: \(error)"
+        )
+      }
+    } hook: { request in
+      // Simulate a stuck sourcekitd that only gets unstuck when sourcekitd is terminated.
+      if request.description.contains("cursorinfo") {
+        // Use a detached task here so that a cancellation of the Task that runs this doesn't cancel the await of
+        // sourcekitdTerminatedExpectation. We want to simulate a sourecekitd that is stuck and doesn't listen to
+        // cancellation.
+        await Task.detached {
+          await orLog("awaiting sourcekitdTerminatedExpectation") {
+            try await fulfillmentOfOrThrow(sourcekitdTerminatedExpectation)
+          }
+        }.value
+      }
+    }
+
+    // After sourcekitd is restarted, we should get a hover result once the semantic editor is enabled again.
+    try await repeatUntilExpectedResult {
+      do {
+        let hover = try await testClient.send(
+          HoverRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"])
+        )
+        return hover?.contents.markupContent?.value != nil
+      } catch {
+        return false
+      }
+    }
+  }
 }

Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift

@@ -1830,7 +1830,7 @@ fileprivate extension SourceKitD {
       keys.syntacticOnly: 1,
       keys.compilerArgs: compilerArguments as [SKDRequestValue],
     ])
-    _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await send(req, timeout: defaultTimeoutDuration)
     return DocumentPositions(markers: markers, textWithoutMarkers: textWithoutMarkers)
   }
 
@@ -1844,7 +1844,7 @@ fileprivate extension SourceKitD {
       keys.syntacticOnly: 1,
     ])
 
-    _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await send(req, timeout: defaultTimeoutDuration)
   }
 
   nonisolated func closeDocument(_ name: String) async throws {
@@ -1853,7 +1853,7 @@ fileprivate extension SourceKitD {
       keys.name: name,
     ])
 
-    _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await send(req, timeout: defaultTimeoutDuration)
   }
 
   nonisolated func completeImpl(
@@ -1888,7 +1888,7 @@ fileprivate extension SourceKitD {
       keys.compilerArgs: compilerArguments as [SKDRequestValue]?,
     ])
 
-    let res = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    let res = try await send(req, timeout: defaultTimeoutDuration)
     return try CompletionResultSet(res)
   }
 
@@ -1946,7 +1946,7 @@ fileprivate extension SourceKitD {
       keys.codeCompleteOptions: dictionary([keys.useNewAPI: 1]),
     ])
 
-    _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await send(req, timeout: defaultTimeoutDuration)
   }
 
   nonisolated func completeDocumentation(id: Int) async throws -> CompletionDocumentation {
@@ -1955,7 +1955,7 @@ fileprivate extension SourceKitD {
       keys.identifier: id,
     ])
 
-    let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    let resp = try await send(req, timeout: defaultTimeoutDuration)
     return CompletionDocumentation(resp)
   }
 
@@ -1964,7 +1964,7 @@ fileprivate extension SourceKitD {
       keys.request: requests.codeCompleteDiagnostic,
       keys.identifier: id,
     ])
-    let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    let resp = try await send(req, timeout: defaultTimeoutDuration)
 
     return CompletionDiagnostic(resp)
   }
@@ -1973,7 +1973,7 @@ fileprivate extension SourceKitD {
     let req = dictionary([
       keys.request: requests.dependencyUpdated
     ])
-    _ = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    _ = try await send(req, timeout: defaultTimeoutDuration)
   }
 
   nonisolated func setPopularAPI(popular: [String], unpopular: [String]) async throws {
@@ -1984,7 +1984,7 @@ fileprivate extension SourceKitD {
       keys.unpopular: unpopular as [SKDRequestValue],
     ])
 
-    let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    let resp = try await send(req, timeout: defaultTimeoutDuration)
     XCTAssertEqual(resp[keys.useNewAPI], 1)
   }
 
@@ -2001,7 +2001,7 @@ fileprivate extension SourceKitD {
       keys.notoriousModules: notoriousModules as [SKDRequestValue],
     ])
 
-    let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    let resp = try await send(req, timeout: defaultTimeoutDuration)
     XCTAssertEqual(resp[keys.useNewAPI], 1)
   }
 
@@ -2025,7 +2025,7 @@ fileprivate extension SourceKitD {
       keys.modulePopularity: modulePopularity as [SKDRequestValue],
     ])
 
-    let resp = try await send(req, timeout: defaultTimeoutDuration, fileContents: nil)
+    let resp = try await send(req, timeout: defaultTimeoutDuration)
     XCTAssertEqual(resp[keys.useNewAPI], 1)
   }
 

config.schema.json

@@ -181,6 +181,11 @@
       },
       "type" : "object"
     },
+    "semanticServiceRestartTimeout" : {
+      "description" : "If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging for some reason and won't recover. To restore semantic functionality, we terminate and restart it.",
+      "markdownDescription" : "If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging for some reason and won't recover. To restore semantic functionality, we terminate and restart it.",
+      "type" : "number"
+    },
     "sourcekitd" : {
       "description" : "Options modifying the behavior of sourcekitd.",
       "markdownDescription" : "Options modifying the behavior of sourcekitd.",