If sourcekitd or clangd don’t respond to a request for 5 minutes, terminate them and use crash recovery to restore behavior

This should be a last stop-gap measure in case sourcekitd or clangd get stuck, don’t respond to any requests anymore and don’t honor cancellation either. In that case we can restore SourceKit-LSP behavior by killing them and using the crash recovery logic to restore functionality.

rdar://149492159

Author: ahoppen

Sources/BuildSystemIntegration/ExternalBuildSystemAdapter.swift

@@ -185,11 +185,11 @@ actor ExternalBuildSystemAdapter {
       protocol: bspRegistry,
       stderrLoggingCategory: "bsp-server-stderr",
       client: messagesToSourceKitLSPHandler,
-      terminationHandler: { [weak self] terminationStatus in
+      terminationHandler: { [weak self] terminationReason in
         guard let self else {
           return
         }
-        if terminationStatus != 0 {
+        if terminationReason != .exited(exitCode: 0) {
           Task {
             await orLog("Restarting BSP server") {
               try await self.handleBspServerCrash()

Sources/LanguageServerProtocolJSONRPC/JSONRPCConnection.swift

@@ -31,6 +31,14 @@ import struct CDispatch.dispatch_fd_t
 /// For example, inside a language server, the `JSONRPCConnection` takes the language service implementation as its
 // `receiveHandler` and itself provides the client connection for sending notifications and callbacks.
 public final class JSONRPCConnection: Connection {
+  public enum TerminationReason: Sendable, Equatable {
+    /// The process on the other end of the `JSONRPCConnection` terminated with the given exit code.
+    case exited(exitCode: Int32)
+
+    /// The process on the other end of the `JSONRPCConnection` terminated with a signal. The signal that it terminated
+    /// with is not known.
+    case uncaughtSignal
+  }
 
   /// A name of the endpoint for this connection, used for logging, e.g. `clangd`.
   private let name: String
@@ -198,7 +206,7 @@ public final class JSONRPCConnection: Connection {
     protocol messageRegistry: MessageRegistry,
     stderrLoggingCategory: String,
     client: MessageHandler,
-    terminationHandler: @Sendable @escaping (_ terminationStatus: Int32) -> Void
+    terminationHandler: @Sendable @escaping (_ terminationReason: TerminationReason) -> Void
   ) throws -> (connection: JSONRPCConnection, process: Process) {
     let clientToServer = Pipe()
     let serverToClient = Pipe()
@@ -238,10 +246,22 @@ public final class JSONRPCConnection: Connection {
     process.terminationHandler = { process in
       logger.log(
         level: process.terminationReason == .exit ? .default : .error,
-        "\(name) exited: \(String(reflecting: process.terminationReason)) \(process.terminationStatus)"
+        "\(name) exited: \(process.terminationReason.rawValue) \(process.terminationStatus)"
       )
       connection.close()
-      terminationHandler(process.terminationStatus)
+      let terminationReason: TerminationReason
+      switch process.terminationReason {
+      case .exit:
+        terminationReason = .exited(exitCode: process.terminationStatus)
+      case .uncaughtSignal:
+        terminationReason = .uncaughtSignal
+      @unknown default:
+        logger.fault(
+          "Process terminated with unknown termination reason: \(process.terminationReason.rawValue, privacy: .public)"
+        )
+        terminationReason = .exited(exitCode: 0)
+      }
+      terminationHandler(terminationReason)
     }
     try process.run()
 

Sources/SKOptions/SourceKitLSPOptions.swift

@@ -422,6 +422,17 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     return .seconds(120)
   }
 
+  /// If a request to sourcekitd or clangd exceeds this timeout, we assume that the semantic service provider is hanging
+  /// for some reason and won't recover. To restore semantic functionality, we terminate and restart it.
+  public var semanticServiceRestartTimeout: Double? = nil
+
+  public var semanticServiceRestartTimeoutOrDefault: Duration {
+    if let semanticServiceRestartTimeout {
+      return .seconds(semanticServiceRestartTimeout)
+    }
+    return .seconds(300)
+  }
+
   public init(
     swiftPM: SwiftPMOptions? = .init(),
     fallbackBuildSystem: FallbackBuildSystemOptions? = .init(),
@@ -439,7 +450,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     experimentalFeatures: Set<ExperimentalFeature>? = nil,
     swiftPublishDiagnosticsDebounceDuration: Double? = nil,
     workDoneProgressDebounceDuration: Double? = nil,
-    sourcekitdRequestTimeout: Double? = nil
+    sourcekitdRequestTimeout: Double? = nil,
+    semanticServiceRestartTimeout: Double? = nil
   ) {
     self.swiftPM = swiftPM
     self.fallbackBuildSystem = fallbackBuildSystem
@@ -458,6 +470,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     self.swiftPublishDiagnosticsDebounceDuration = swiftPublishDiagnosticsDebounceDuration
     self.workDoneProgressDebounceDuration = workDoneProgressDebounceDuration
     self.sourcekitdRequestTimeout = sourcekitdRequestTimeout
+    self.semanticServiceRestartTimeout = semanticServiceRestartTimeout
   }
 
   public init?(fromLSPAny lspAny: LSPAny?) throws {
@@ -517,7 +530,8 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
         ?? base.swiftPublishDiagnosticsDebounceDuration,
       workDoneProgressDebounceDuration: override?.workDoneProgressDebounceDuration
         ?? base.workDoneProgressDebounceDuration,
-      sourcekitdRequestTimeout: override?.sourcekitdRequestTimeout ?? base.sourcekitdRequestTimeout
+      sourcekitdRequestTimeout: override?.sourcekitdRequestTimeout ?? base.sourcekitdRequestTimeout,
+      semanticServiceRestartTimeout: override?.semanticServiceRestartTimeout ?? base.semanticServiceRestartTimeout
     )
   }
 

Sources/SKTestSupport/SkipUnless.swift

@@ -265,8 +265,7 @@ package actor SkipUnless {
               sourcekitd.keys.useNewAPI: 1
             ],
           ]),
-          timeout: defaultTimeoutDuration,
-          fileContents: nil
+          timeout: defaultTimeoutDuration
         )
         return response[sourcekitd.keys.useNewAPI] == 1
       } catch {

Sources/SKTestSupport/SourceKitD+send.swift

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+package import SourceKitD
+
+extension SourceKitD {
+  /// Convenience overload of the `send` function for testing that doesn't restart sourcekitd if it does not respond
+  /// and doesn't pass any file contents.
+  package func send(
+    _ request: SKDRequestDictionary,
+    timeout: Duration
+  ) async throws -> SKDResponseDictionary {
+    return try await self.send(
+      request,
+      timeout: timeout,
+      restartTimeout: .seconds(60 * 60 * 24),
+      fileContents: nil
+    )
+  }
+}

Sources/SourceKitD/SourceKitD.swift

@@ -162,7 +162,10 @@ package actor SourceKitD {
   /// List of notification handlers that will be called for each notification.
   private var notificationHandlers: [WeakSKDNotificationHandler] = []
 
-  /// List of hooks that should be executed for every request sent to sourcekitd.
+  /// List of hooks that should be executed and that need to finish executing before a request is sent to sourcekitd.
+  private var preRequestHandlingHooks: [UUID: @Sendable (SKDRequestDictionary) async -> Void] = [:]
+
+  /// List of hooks that should be executed after a request sent to sourcekitd.
   private var requestHandlingHooks: [UUID: (SKDRequestDictionary) -> Void] = [:]
 
   package static func getOrCreate(
@@ -261,6 +264,30 @@ package actor SourceKitD {
     notificationHandlers.removeAll(where: { $0.value == nil || $0.value === handler })
   }
 
+  /// Execute `body` and invoke `hook` for every sourcekitd request that is sent during the execution time of `body`.
+  ///
+  /// Note that `hook` will not only be executed for requests sent *by* body but this may also include sourcekitd
+  /// requests that were sent by other clients of the same `DynamicallyLoadedSourceKitD` instance that just happen to
+  /// send a request during that time.
+  ///
+  /// This is intended for testing only.
+  package func withPreRequestHandlingHook(
+    body: () async throws -> Void,
+    hook: @escaping @Sendable (SKDRequestDictionary) async -> Void
+  ) async rethrows {
+    let id = UUID()
+    preRequestHandlingHooks[id] = hook
+    defer { preRequestHandlingHooks[id] = nil }
+    try await body()
+  }
+
+  func willSend(request: SKDRequestDictionary) async {
+    let request = request
+    for hook in preRequestHandlingHooks.values {
+      await hook(request)
+    }
+  }
+
   /// Execute `body` and invoke `hook` for every sourcekitd request that is sent during the execution time of `body`.
   ///
   /// Note that `hook` will not only be executed for requests sent *by* body but this may also include sourcekitd
@@ -293,37 +320,56 @@ package actor SourceKitD {
   package func send(
     _ request: SKDRequestDictionary,
     timeout: Duration,
+    restartTimeout: Duration,
     fileContents: String?
   ) async throws -> SKDResponseDictionary {
     let sourcekitdResponse = try await withTimeout(timeout) {
-      return try await withCancellableCheckedThrowingContinuation { (continuation) -> SourceKitDRequestHandle? in
-        logger.info(
-          """
-          Sending sourcekitd request:
-          \(request.forLogging)
-          """
-        )
-        var handle: sourcekitd_api_request_handle_t? = nil
-        self.api.send_request(request.dict, &handle) { response in
-          continuation.resume(returning: SKDResponse(response!, sourcekitd: self))
-        }
-        Task {
-          await self.didSend(request: request)
-        }
-        if let handle {
-          return SourceKitDRequestHandle(handle: handle)
+      let restartTimeoutHandle = TimeoutHandle()
+      do {
+        return try await withTimeout(restartTimeout, handle: restartTimeoutHandle) {
+          await self.willSend(request: request)
+          return try await withCancellableCheckedThrowingContinuation { (continuation) -> SourceKitDRequestHandle? in
+            logger.info(
+              """
+              Sending sourcekitd request:
+              \(request.forLogging)
+              """
+            )
+            var handle: sourcekitd_api_request_handle_t? = nil
+            self.api.send_request(request.dict, &handle) { response in
+              continuation.resume(returning: SKDResponse(response!, sourcekitd: self))
+            }
+            Task {
+              await self.didSend(request: request)
+            }
+            if let handle {
+              return SourceKitDRequestHandle(handle: handle)
+            }
+            return nil
+          } cancel: { (handle: SourceKitDRequestHandle?) in
+            if let handle {
+              logger.info(
+                """
+                Cancelling sourcekitd request:
+                \(request.forLogging)
+                """
+              )
+              self.api.cancel_request(handle.handle)
+            }
+          }
         }
-        return nil
-      } cancel: { (handle: SourceKitDRequestHandle?) in
-        if let handle {
-          logger.info(
-            """
-            Cancelling sourcekitd request:
-            \(request.forLogging)
-            """
+      } catch let error as TimeoutError where error.handle == restartTimeoutHandle {
+        if !self.path.lastPathComponent.contains("InProc") {
+          logger.fault(
+            "Did not receive reply from sourcekitd after \(restartTimeout, privacy: .public). Terminating and restarting sourcekitd."
+          )
+          await self.crash()
+        } else {
+          logger.fault(
+            "Did not receive reply from sourcekitd after \(restartTimeout, privacy: .public). Not terminating sourcekitd because it is run in-process."
           )
-          self.api.cancel_request(handle.handle)
         }
+        throw error
       }
     }
 
@@ -362,6 +408,13 @@ package actor SourceKitD {
 
     return dict
   }
+
+  package func crash() async {
+    let req = dictionary([
+      keys.request: requests.crashWithExit
+    ])
+    _ = try? await send(req, timeout: .seconds(60), restartTimeout: .seconds(24 * 60 * 60), fileContents: nil)
+  }
 }
 
 /// A sourcekitd notification handler in a class to allow it to be uniquely referenced.

Sources/SourceKitD/SourceKitDRegistry.swift

@@ -30,7 +30,7 @@ package actor SourceKitDRegistry<SourceKitDType: AnyObject> {
   private var active: [URL: (pluginPaths: PluginPaths?, sourcekitd: SourceKitDType)] = [:]
 
   /// Instances that have been unregistered, but may be resurrected if accessed before destruction.
-  private var cemetary: [URL: (pluginPaths: PluginPaths?, sourcekitd: WeakSourceKitD<SourceKitDType>)] = [:]
+  private var cemetery: [URL: (pluginPaths: PluginPaths?, sourcekitd: WeakSourceKitD<SourceKitDType>)] = [:]
 
   /// Initialize an empty registry.
   package init() {}
@@ -49,8 +49,8 @@ package actor SourceKitDRegistry<SourceKitDType: AnyObject> {
       }
       return existing.sourcekitd
     }
-    if let resurrected = cemetary[key], let resurrectedSourcekitD = resurrected.sourcekitd.value {
-      cemetary[key] = nil
+    if let resurrected = cemetery[key], let resurrectedSourcekitD = resurrected.sourcekitd.value {
+      cemetery[key] = nil
       if resurrected.pluginPaths != pluginPaths {
         logger.fault(
           "Already created SourceKitD with plugin paths \(resurrected.pluginPaths?.forLogging), now requesting incompatible plugin paths \(pluginPaths.forLogging)"
@@ -74,8 +74,8 @@ package actor SourceKitDRegistry<SourceKitDType: AnyObject> {
   package func remove(_ key: URL) -> SourceKitDType? {
     let existing = active.removeValue(forKey: key)
     if let existing = existing {
-      assert(self.cemetary[key]?.sourcekitd.value == nil)
-      cemetary[key] = (existing.pluginPaths, WeakSourceKitD(value: existing.sourcekitd))
+      assert(self.cemetery[key]?.sourcekitd.value == nil)
+      cemetery[key] = (existing.pluginPaths, WeakSourceKitD(value: existing.sourcekitd))
     }
     return existing?.sourcekitd
   }

Sources/SourceKitLSP/Clang/ClangLanguageService.swift

@@ -62,7 +62,7 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   /// Path to the `clangd` binary.
   let clangdPath: URL
 
-  let clangdOptions: [String]
+  let options: SourceKitLSPOptions
 
   /// The current state of the `clangd` language server.
   /// Changing the property automatically notified the state change handlers.
@@ -117,7 +117,7 @@ actor ClangLanguageService: LanguageService, MessageHandler {
     }
     self.clangPath = toolchain.clang
     self.clangdPath = clangdPath
-    self.clangdOptions = options.clangdOptions ?? []
+    self.options = options
     self.workspace = WeakWorkspace(workspace)
     self.state = .connected
     self.sourceKitLSPServer = sourceKitLSPServer
@@ -154,10 +154,11 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   /// Restarts `clangd` if it has crashed.
   ///
   /// - Parameter terminationStatus: The exit code of `clangd`.
-  private func handleClangdTermination(terminationStatus: Int32) {
+  private func handleClangdTermination(terminationReason: JSONRPCConnection.TerminationReason) {
     self.clangdProcess = nil
-    if terminationStatus != 0 {
+    if terminationReason != .exited(exitCode: 0) {
       self.state = .connectionInterrupted
+      logger.info("clangd crashed. Restarting it.")
       self.restartClangd()
     }
   }
@@ -173,15 +174,15 @@ actor ClangLanguageService: LanguageService, MessageHandler {
         "-compile_args_from=lsp",  // Provide compiler args programmatically.
         "-background-index=false",  // Disable clangd indexing, we use the build
         "-index=false",  // system index store instead.
-      ] + clangdOptions,
+      ] + (options.clangdOptions ?? []),
       name: "clangd",
       protocol: MessageRegistry.lspProtocol,
       stderrLoggingCategory: "clangd-stderr",
       client: self,
-      terminationHandler: { [weak self] terminationStatus in
+      terminationHandler: { [weak self] terminationReason in
         guard let self = self else { return }
         Task {
-          await self.handleClangdTermination(terminationStatus: terminationStatus)
+          await self.handleClangdTermination(terminationReason: terminationReason)
         }
 
       }
@@ -291,14 +292,20 @@ actor ClangLanguageService: LanguageService, MessageHandler {
   }
 
   /// Forward the given request to `clangd`.
-  ///
-  /// This method calls `readyToHandleNextRequest` once the request has been
-  /// transmitted to `clangd` and another request can be safely transmitted to
-  /// `clangd` while guaranteeing ordering.
-  ///
-  /// The response of the request is  returned asynchronously as the return value.
   func forwardRequestToClangd<R: RequestType>(_ request: R) async throws -> R.Response {
-    return try await clangd.send(request)
+    let timeoutHandle = TimeoutHandle()
+    do {
+      return try await withTimeout(options.semanticServiceRestartTimeoutOrDefault, handle: timeoutHandle) {
+        await self.sourceKitLSPServer?.hooks.preForwardRequestToClangd?(request)
+        return try await self.clangd.send(request)
+      }
+    } catch let error as TimeoutError where error.handle == timeoutHandle {
+      logger.fault(
+        "Did not receive reply from clangd after \(options.semanticServiceRestartTimeoutOrDefault, privacy: .public). Terminating and restarting clangd."
+      )
+      self.crash()
+      throw error
+    }
   }
 
   package func canonicalDeclarationPosition(of position: Position, in uri: DocumentURI) async -> Position? {