fix(load): only fall back to text-only for VLM checkpoint mismatches

Review on #173: `catch where !self.vision` caught every error, so a download
failure or a CancellationError triggered a second, LLM load attempt and then
surfaced a confusing error. The fallback now requires
isVLMCheckpointMismatch(error): a DecodingError, an MLXNN UpdateError
(unhandledKeys, keyNotFound, mismatchedSize, ...), or a ModelFactoryError
for an unsupported model/processor type or an undecodable/invalid config.
Every other error propagates unchanged.

VLMFallbackTests covers the fallback cases (missing image_mean, unhandled
weight keys, unsupported model type) and the pass-through cases
(cancellation, URLError, missing config file).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: 2 people

Sources/SwiftLM/Server.swift

@@ -205,6 +205,28 @@ private struct TransformersTokenizerBridge: MLXLMCommon.Tokenizer, Sendable {
 
 /// Returns `nil` when the value must be dropped (JSON `null` / NSNull), otherwise a
 /// structure with every nested null removed. See `TransformersTokenizerBridge.applyChatTemplate`.
+/// True when a VLM load failed because the checkpoint doesn't match the VLM code:
+/// its config doesn't decode, its weights don't line up with the module tree, or the
+/// factory doesn't know the model/processor type. Only these justify retrying an
+/// auto-detected VLM as a text-only LLM. Anything else (cancellation, download, I/O)
+/// would fail the same way again and just hide the real error.
+func isVLMCheckpointMismatch(_ error: any Error) -> Bool {
+    switch error {
+    case is DecodingError, is UpdateError:
+        return true
+    case let factoryError as ModelFactoryError:
+        switch factoryError {
+        case .unsupportedModelType, .unsupportedProcessorType, .configurationDecodingError,
+            .invalidConfiguration:
+            return true
+        default:
+            return false
+        }
+    default:
+        return false
+    }
+}
+
 func sanitizeForJinja(_ value: any Sendable) -> (any Sendable)? {
     if value is NSNull { return nil }
     let mirror = Mirror(reflecting: value)
@@ -904,11 +926,12 @@ struct MLXServer: AsyncParsableCommand {
                 ) { progress in
                     tracker.printProgress(progress)
                 }
-            } catch where !self.vision {
-                // Vision was only auto-detected. A checkpoint whose vision side
-                // doesn't load (e.g. a preprocessor_config.json without
-                // image_mean) can still serve text, so fall back rather than exit.
-                // With an explicit --vision, the error still propagates.
+            } catch where !self.vision && isVLMCheckpointMismatch(error) {
+                // Vision was only auto-detected, and the vision side of the checkpoint
+                // doesn't match the VLM code (e.g. a preprocessor_config.json without
+                // image_mean). The text model can still serve, so fall back rather than
+                // exit. Cancellation, network and I/O errors still propagate, as does
+                // any error under an explicit --vision.
                 print("[SwiftLM] ⚠️  Auto-detected VLM failed to load (\(error)); loading as a text-only LLM. Pass --vision to make this fatal.")
                 isVision = false
                 container = try await LLMModelFactory.shared.loadContainer(

tests/SwiftLMTests/VLMFallbackTests.swift

@@ -0,0 +1,44 @@
+import XCTest
+import Foundation
+import MLXLMCommon
+import MLXNN
+@testable import SwiftLM
+
+/// An auto-detected VLM that fails to load falls back to text-only only for
+/// checkpoint mismatches. Other failures must surface as themselves.
+final class VLMFallbackTests: XCTestCase {
+
+    private struct Probe: Decodable { let image_mean: [Double] }
+
+    func testConfigDecodingErrorFallsBack() {
+        // The real case: unsloth/Qwen3.6-35B-A3B's preprocessor_config.json has no image_mean.
+        do {
+            _ = try JSONDecoder().decode(Probe.self, from: Data("{}".utf8))
+            XCTFail("expected a DecodingError")
+        } catch {
+            XCTAssertTrue(isVLMCheckpointMismatch(error))
+        }
+    }
+
+    func testWeightMismatchFallsBack() {
+        let error = UpdateError.unhandledKeys(path: [], modules: ["Vision"], keys: ["pre_projection"])
+        XCTAssertTrue(isVLMCheckpointMismatch(error))
+    }
+
+    func testUnsupportedModelTypeFallsBack() {
+        XCTAssertTrue(isVLMCheckpointMismatch(ModelFactoryError.unsupportedModelType("qwen4_exp")))
+    }
+
+    func testCancellationDoesNotFallBack() {
+        XCTAssertFalse(isVLMCheckpointMismatch(CancellationError()))
+    }
+
+    func testNetworkErrorDoesNotFallBack() {
+        XCTAssertFalse(isVLMCheckpointMismatch(URLError(.notConnectedToInternet)))
+    }
+
+    func testMissingConfigFileDoesNotFallBack() {
+        let io = CocoaError(.fileReadNoSuchFile)
+        XCTAssertFalse(isVLMCheckpointMismatch(ModelFactoryError.configurationFileError("config.json", "m", io)))
+    }
+}