diff --git a/README.md b/README.md index b4bdeac8..e76cff32 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ There is an example of using: - **Local Execution:** Run Gemma and other LLMs (Qwen, DeepSeek, Phi, FastVLM, SmolLM, …) directly on user devices for enhanced privacy and offline functionality. - **Platform Support:** Compatible with iOS, Android, Web, macOS, Windows, and Linux platforms. - **🖥️ Desktop Support:** Native desktop apps (macOS, Windows, Linux) with GPU acceleration via LiteRT-LM, called directly from Dart through `dart:ffi` — no JVM/JRE bundling. See [DESKTOP_SUPPORT.md](DESKTOP_SUPPORT.md) for details. +- **🧩 Desktop Runtime Extensions:** Register custom desktop runtimes and fall back to the built-in LiteRT path when an extension declines a model. +- **🍎 Built-in macOS MLX bridge:** When the host app links `flm_dispatch_json` / `flm_bridge_free_string`, flutter_gemma now auto-routes local model directories to MLX without changing the chat/session API. - **🖼️ Multimodal Support:** Text + Image input with Gemma 4, Gemma3n, and FastVLM vision models - **🎙️ Audio Input:** Record and send audio messages with Gemma3n E2B/E4B models (Android, iOS device, Desktop) - **🛠️ Function Calling:** Enable your models to call external functions and integrate with other services (supported by select models) @@ -153,6 +155,38 @@ await FlutterGemma.installModel(modelType: ModelType.general) 2. Run `flutter pub get` to install. +## Custom desktop runtimes + +On macOS, flutter_gemma now includes a built-in MLX desktop extension. If the +active model resolves to a local directory and the host process already links an +MLX bridge exposing `flm_dispatch_json` / `flm_bridge_free_string`, the desktop +plugin auto-selects MLX before falling back to LiteRT-LM. + +If you already have a different native desktop runtime in another package, you +can still replace or extend that behavior without forking the main chat/session +APIs: + +```dart +FlutterGemmaDesktop.registerRuntimeExtension( + DesktopRuntimeExtension( + name: 'mlx', + createInferenceModel: (request) async { + if (!request.modelPath.endsWith('.mlx') && + !request.modelPath.contains('mlx')) { + return null; + } + + // Return your own InferenceModel implementation here. + return MyMlxInferenceModel.fromRequest(request); + }, + ), +); +``` + +The first extension that returns a non-null model handles the request. If every +extension returns `null`, flutter_gemma continues with its built-in LiteRT +desktop runtime exactly as before. + ## Platform & Architecture Support The plugin ships native prebuilts only for the architectures below. Other ABIs fail at native load with a typed error. diff --git a/lib/core/infrastructure/platform_file_system_service.dart b/lib/core/infrastructure/platform_file_system_service.dart index 3b6a9103..4194bc92 100644 --- a/lib/core/infrastructure/platform_file_system_service.dart +++ b/lib/core/infrastructure/platform_file_system_service.dart @@ -51,7 +51,10 @@ class PlatformFileSystemService implements FileSystemService { @override Future fileExists(String filePath) async { final file = File(filePath); - return await file.exists(); + if (await file.exists()) { + return true; + } + return Directory(filePath).exists(); } @override @@ -59,7 +62,11 @@ class PlatformFileSystemService implements FileSystemService { final file = File(filePath); if (!await file.exists()) { - return 0; + final directory = Directory(filePath); + if (!await directory.exists()) { + return 0; + } + return _directorySize(directory); } final stat = await file.stat(); @@ -175,6 +182,16 @@ class PlatformFileSystemService implements FileSystemService { // The actual tracking is done in ProtectedFilesRegistry.registerExternalPath() } + Future _directorySize(Directory directory) async { + int total = 0; + await for (final entity in directory.list(recursive: true)) { + if (entity is File) { + total += await entity.length(); + } + } + return total; + } + /// Gets the model storage directory with caching. /// /// Mobile (Android, iOS): app's Documents — sandboxed, never cloud-synced. diff --git a/lib/core/model_management/utils/file_system_manager.dart b/lib/core/model_management/utils/file_system_manager.dart index a6d7d8a6..a37c8844 100644 --- a/lib/core/model_management/utils/file_system_manager.dart +++ b/lib/core/model_management/utils/file_system_manager.dart @@ -62,6 +62,15 @@ class ModelFileSystemManager { int minSizeBytes = _defaultMinSizeBytes, }) async { try { + final directory = Directory(filePath); + if (await directory.exists()) { + final isValidDirectory = await _isModelDirectoryValid(directory); + if (!isValidDirectory) { + debugPrint('Model directory validation failed: $filePath'); + } + return isValidDirectory; + } + final file = File(filePath); if (!await file.exists()) { @@ -83,6 +92,28 @@ class ModelFileSystemManager { } } + static Future _isModelDirectoryValid(Directory directory) async { + final entries = await directory.list(followLinks: false).toList(); + if (entries.isEmpty) { + return false; + } + + for (final entry in entries) { + final name = path.basename(entry.path).toLowerCase(); + if (entry is File && + (name == 'config.json' || + name == 'tokenizer.json' || + name == 'tokenizer_config.json' || + name.endsWith('.safetensors') || + name.endsWith('.gguf') || + name.endsWith('.bin'))) { + return true; + } + } + + return entries.any((entry) => entry is File); + } + /// Gets the full file path for a model file. /// /// Delegates to [FileSystemService.getTargetPath] so that the correct diff --git a/lib/desktop/desktop_runtime_extension.dart b/lib/desktop/desktop_runtime_extension.dart new file mode 100644 index 00000000..8e39b325 --- /dev/null +++ b/lib/desktop/desktop_runtime_extension.dart @@ -0,0 +1,307 @@ +import 'dart:async'; + +import '../core/chat.dart'; +import '../core/model.dart'; +import '../core/tool.dart'; +import '../core/domain/model_source.dart'; +import '../flutter_gemma_interface.dart'; +import '../mobile/flutter_gemma_mobile.dart' + show EmbeddingModelSpec, InferenceModelSpec; +import '../pigeon.g.dart'; + +typedef DesktopInferenceModelFactory = FutureOr Function( + DesktopInferenceRequest request); + +typedef DesktopEmbeddingModelFactory = FutureOr Function( + DesktopEmbeddingRequest request); + +/// Request payload for custom desktop inference runtimes. +/// +/// Integrators can inspect the resolved file paths, active [InferenceModelSpec], +/// and requested runtime knobs to decide whether they want to handle the model. +/// Returning `null` means "not supported, continue with the built-in LiteRT +/// desktop runtime". +class DesktopInferenceRequest { + const DesktopInferenceRequest({ + required this.spec, + required this.modelPath, + required this.modelType, + required this.fileType, + required this.maxTokens, + required this.cacheDir, + this.loraPath, + this.preferredBackend, + this.loraRanks, + this.maxNumImages, + this.supportImage = false, + this.supportAudio = false, + this.enableSpeculativeDecoding, + }); + + final InferenceModelSpec spec; + final String modelPath; + final String? loraPath; + final ModelType modelType; + final ModelFileType fileType; + final int maxTokens; + final String cacheDir; + final PreferredBackend? preferredBackend; + final List? loraRanks; + final int? maxNumImages; + final bool supportImage; + final bool supportAudio; + final bool? enableSpeculativeDecoding; +} + +/// Request payload for custom desktop embedding runtimes. +/// +/// Returning `null` from the factory means the extension declines the request +/// and flutter_gemma should keep using its bundled embedding path. +class DesktopEmbeddingRequest { + const DesktopEmbeddingRequest({ + required this.modelPath, + required this.tokenizerPath, + this.spec, + this.preferredBackend, + }); + + final EmbeddingModelSpec? spec; + final String modelPath; + final String tokenizerPath; + final PreferredBackend? preferredBackend; +} + +/// Describes a pluggable desktop runtime. +/// +/// Example use cases: +/// - MLX-backed inference on macOS +/// - Project-specific native bridges +/// - Experimental runtimes that should coexist with LiteRT fallback +class DesktopRuntimeExtension { + const DesktopRuntimeExtension({ + required this.name, + this.createInferenceModel, + this.createEmbeddingModel, + }); + + final String name; + final DesktopInferenceModelFactory? createInferenceModel; + final DesktopEmbeddingModelFactory? createEmbeddingModel; +} + +/// Registry for pluggable desktop runtimes. +/// +/// flutter_gemma ships with a built-in LiteRT desktop path. This registry makes +/// it possible to add alternative runtimes without forking core model/session +/// APIs. The first extension that returns a non-null model wins; otherwise the +/// built-in LiteRT implementation is used. +class DesktopRuntimeRegistry { + final List _extensions = []; + + List get extensions => + List.unmodifiable(_extensions); + + void register(DesktopRuntimeExtension extension) { + unregister(extension.name); + _extensions.add(extension); + } + + bool unregister(String name) { + final index = _extensions.indexWhere((extension) => extension.name == name); + if (index == -1) return false; + _extensions.removeAt(index); + return true; + } + + void clear() => _extensions.clear(); + + Future createInferenceModel( + DesktopInferenceRequest request, + ) async { + for (final extension in _extensions) { + final factory = extension.createInferenceModel; + if (factory == null) continue; + final model = await factory(request); + if (model != null) { + return model; + } + } + return null; + } + + Future createEmbeddingModel( + DesktopEmbeddingRequest request, + ) async { + for (final extension in _extensions) { + final factory = extension.createEmbeddingModel; + if (factory == null) continue; + final model = await factory(request); + if (model != null) { + return model; + } + } + return null; + } + + Future createManagedInferenceModel( + DesktopInferenceRequest request, { + required FutureOr Function() onClose, + }) async { + final model = await createInferenceModel(request); + if (model == null) return null; + return _ManagedInferenceModel(delegate: model, onClose: onClose); + } + + Future createManagedEmbeddingModel( + DesktopEmbeddingRequest request, { + required FutureOr Function() onClose, + }) async { + final model = await createEmbeddingModel(request); + if (model == null) return null; + return _ManagedEmbeddingModel(delegate: model, onClose: onClose); + } +} + +class _ManagedInferenceModel extends InferenceModel { + _ManagedInferenceModel({ + required InferenceModel delegate, + required FutureOr Function() onClose, + }) : _delegate = delegate, + _onClose = onClose; + + final InferenceModel _delegate; + final FutureOr Function() _onClose; + bool _closed = false; + + @override + InferenceModelSession? get session => _delegate.session; + + @override + InferenceChat? get chat => _delegate.chat; + + @override + set chat(InferenceChat? value) => _delegate.chat = value; + + @override + int get maxTokens => _delegate.maxTokens; + + @override + ModelFileType get fileType => _delegate.fileType; + + @override + Future createSession({ + double temperature = .8, + int randomSeed = 1, + int topK = 1, + double? topP, + String? loraPath, + bool? enableVisionModality, + bool? enableAudioModality, + String? systemInstruction, + bool enableThinking = false, + List tools = const [], + }) { + return _delegate.createSession( + temperature: temperature, + randomSeed: randomSeed, + topK: topK, + topP: topP, + loraPath: loraPath, + enableVisionModality: enableVisionModality, + enableAudioModality: enableAudioModality, + systemInstruction: systemInstruction, + enableThinking: enableThinking, + tools: tools, + ); + } + + @override + Future createChat({ + double temperature = .8, + int randomSeed = 1, + int topK = 1, + double? topP, + int tokenBuffer = 256, + String? loraPath, + bool? supportImage, + bool? supportAudio, + List tools = const [], + bool? supportsFunctionCalls, + bool isThinking = false, + ModelType? modelType, + ToolChoice toolChoice = ToolChoice.auto, + int? maxFunctionBufferLength, + String? systemInstruction, + }) { + return _delegate.createChat( + temperature: temperature, + randomSeed: randomSeed, + topK: topK, + topP: topP, + tokenBuffer: tokenBuffer, + loraPath: loraPath, + supportImage: supportImage, + supportAudio: supportAudio, + tools: tools, + supportsFunctionCalls: supportsFunctionCalls, + isThinking: isThinking, + modelType: modelType, + toolChoice: toolChoice, + maxFunctionBufferLength: maxFunctionBufferLength, + systemInstruction: systemInstruction, + ); + } + + @override + Future close() async { + if (_closed) return; + _closed = true; + try { + await _delegate.close(); + } finally { + await _onClose(); + } + } +} + +class _ManagedEmbeddingModel extends EmbeddingModel { + _ManagedEmbeddingModel({ + required EmbeddingModel delegate, + required FutureOr Function() onClose, + }) : _delegate = delegate, + _onClose = onClose; + + final EmbeddingModel _delegate; + final FutureOr Function() _onClose; + bool _closed = false; + + @override + Future> generateEmbedding( + String text, { + TaskType taskType = TaskType.retrievalQuery, + }) { + return _delegate.generateEmbedding(text, taskType: taskType); + } + + @override + Future>> generateEmbeddings( + List texts, { + TaskType taskType = TaskType.retrievalQuery, + }) { + return _delegate.generateEmbeddings(texts, taskType: taskType); + } + + @override + Future getDimension() => _delegate.getDimension(); + + @override + Future close() async { + if (_closed) return; + _closed = true; + try { + await _delegate.close(); + } finally { + await _onClose(); + } + } +} diff --git a/lib/desktop/flutter_gemma_desktop.dart b/lib/desktop/flutter_gemma_desktop.dart index 70af857c..2eaa959d 100644 --- a/lib/desktop/flutter_gemma_desktop.dart +++ b/lib/desktop/flutter_gemma_desktop.dart @@ -14,10 +14,13 @@ import '../core/di/service_registry.dart'; import '../core/ffi/litert_lm_client.dart'; import '../core/ffi/ffi_inference_model.dart'; import '../core/litert/litert_embedding_model.dart'; +import 'desktop_runtime_extension.dart'; +import 'mlx_inference_model.dart'; +import 'mlx_runtime_extension.dart'; // Import model management types from mobile (reuse for desktop) import '../mobile/flutter_gemma_mobile.dart' - show InferenceModelSpec, MobileModelManager; + show EmbeddingModelSpec, InferenceModelSpec, MobileModelManager; import '../core/model_management/constants/preferences_keys.dart'; @@ -31,16 +34,44 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { FlutterGemmaDesktop._(); static FlutterGemmaDesktop? _instance; + static final DesktopRuntimeRegistry _runtimeRegistry = + DesktopRuntimeRegistry(); + static bool _builtInRuntimeExtensionsRegistered = false; /// Get the singleton instance - static FlutterGemmaDesktop get instance => - _instance ??= FlutterGemmaDesktop._(); + static FlutterGemmaDesktop get instance => _instance ??= (() { + _ensureBuiltInRuntimeExtensionsRegistered(); + return FlutterGemmaDesktop._(); + })(); + + static DesktopRuntimeRegistry get runtimeRegistry => _runtimeRegistry; + + static void _ensureBuiltInRuntimeExtensionsRegistered() { + if (_builtInRuntimeExtensionsRegistered) { + return; + } + _runtimeRegistry.register(createBuiltInMlxRuntimeExtension()); + _builtInRuntimeExtensionsRegistered = true; + } + + static void registerRuntimeExtension(DesktopRuntimeExtension extension) { + _runtimeRegistry.register(extension); + } + + static bool unregisterRuntimeExtension(String name) { + return _runtimeRegistry.unregister(name); + } + + static void clearRuntimeExtensions() { + _runtimeRegistry.clear(); + } /// Register this implementation as the plugin instance /// /// This is called automatically by Flutter for dartPluginClass. /// No parameters needed for desktop platforms. static void registerWith() { + _ensureBuiltInRuntimeExtensionsRegistered(); FlutterGemmaPlugin.instance = instance; debugPrint('[FlutterGemmaDesktop] Plugin registered for desktop platform'); } @@ -93,17 +124,29 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { _lastActiveInferenceSpec != null) { final currentSpec = _lastActiveInferenceSpec!; final requestedSpec = activeModel as InferenceModelSpec; - final currentModel = _initializedModel as DesktopInferenceModel?; + final currentModel = _initializedModel; + final currentSupportImage = switch (currentModel) { + DesktopInferenceModel(:final supportImage) => supportImage, + MlxInferenceModel(:final supportImage) => supportImage, + _ => null, + }; + final currentSupportAudio = switch (currentModel) { + DesktopInferenceModel(:final supportAudio) => supportAudio, + MlxInferenceModel(:final supportAudio) => supportAudio, + _ => null, + }; + final currentMaxTokens = currentModel?.maxTokens; final modelChanged = currentSpec.name != requestedSpec.name; final paramsChanged = currentModel != null && - (currentModel.supportImage != supportImage || - currentModel.supportAudio != supportAudio || - currentModel.maxTokens != maxTokens); + (currentSupportImage != supportImage || + currentSupportAudio != supportAudio || + currentMaxTokens != maxTokens); if (modelChanged || paramsChanged) { debugPrint( - 'Model recreation: modelChanged=$modelChanged, paramsChanged=$paramsChanged'); + 'Model recreation: modelChanged=$modelChanged, paramsChanged=$paramsChanged', + ); await _initializedModel?.close(); _initCompleter = null; _initializedModel = null; @@ -134,12 +177,43 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { throw Exception('Model file paths not found'); } + final activeSpec = activeModel as InferenceModelSpec; final modelPath = modelFilePaths.values.first; + final loraPath = modelFilePaths[PreferencesKeys.installedLoraFileName]; debugPrint('[FlutterGemmaDesktop] Using model: $modelPath'); // Get cache dir for faster reloads final cacheDir = (await getApplicationSupportDirectory()).path; + final extensionModel = await _runtimeRegistry.createManagedInferenceModel( + DesktopInferenceRequest( + spec: activeSpec, + modelPath: modelPath, + loraPath: loraPath, + modelType: modelType, + fileType: fileType, + maxTokens: maxTokens, + preferredBackend: preferredBackend, + loraRanks: loraRanks, + maxNumImages: maxNumImages, + supportImage: supportImage, + supportAudio: supportAudio, + enableSpeculativeDecoding: enableSpeculativeDecoding, + cacheDir: cacheDir, + ), + onClose: () { + _initializedModel = null; + _initCompleter = null; + _lastActiveInferenceSpec = null; + }, + ); + if (extensionModel != null) { + _initializedModel = extensionModel; + _lastActiveInferenceSpec = activeSpec; + completer.complete(extensionModel); + return extensionModel; + } + // Initialize via dart:ffi → C API (no JRE, no gRPC) final ffiClient = LiteRtLmFfiClient(); // NPU is supported via LiteRT-LM's `Backend::NPU` enum on macOS / Linux / @@ -178,7 +252,7 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { }, ); - _lastActiveInferenceSpec = activeModel as InferenceModelSpec; + _lastActiveInferenceSpec = activeSpec; completer.complete(model); return model; @@ -255,6 +329,28 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { ); } + final extensionModel = await _runtimeRegistry.createManagedEmbeddingModel( + DesktopEmbeddingRequest( + spec: currentActiveModel is EmbeddingModelSpec + ? currentActiveModel + : null, + modelPath: modelPath, + tokenizerPath: tokenizerPath, + preferredBackend: preferredBackend, + ), + onClose: () { + _initializedEmbeddingModel = null; + _initEmbeddingCompleter = null; + _lastActiveEmbeddingModelName = null; + }, + ); + if (extensionModel != null) { + _initializedEmbeddingModel = extensionModel; + _lastActiveEmbeddingModelName = currentActiveModel?.name; + completer.complete(extensionModel); + return extensionModel; + } + // 0.15.2: Desktop embedding now uses the same LiteRT FFI path as // mobile (Android + iOS). No more separate TFLiteC + Dart tokenizer // wiring per call site — everything lives in LitertEmbeddingModel. @@ -283,8 +379,9 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { @override Future initializeVectorStore(String databasePath) async { - await ServiceRegistry.instance.vectorStoreRepository - .initialize(databasePath); + await ServiceRegistry.instance.vectorStoreRepository.initialize( + databasePath, + ); } @override @@ -310,7 +407,8 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { }) async { if (initializedEmbeddingModel == null) { throw StateError( - 'EmbeddingModel not initialized. Call createEmbeddingModel first.'); + 'EmbeddingModel not initialized. Call createEmbeddingModel first.', + ); } final embedding = await initializedEmbeddingModel!.generateEmbedding( content, @@ -333,10 +431,12 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { }) async { if (initializedEmbeddingModel == null) { throw StateError( - 'EmbeddingModel not initialized. Call createEmbeddingModel first.'); + 'EmbeddingModel not initialized. Call createEmbeddingModel first.', + ); } - final queryEmbedding = - await initializedEmbeddingModel!.generateEmbedding(query); + final queryEmbedding = await initializedEmbeddingModel!.generateEmbedding( + query, + ); return await ServiceRegistry.instance.vectorStoreRepository.searchSimilar( queryEmbedding: queryEmbedding, topK: topK, diff --git a/lib/desktop/mlx_inference_model.dart b/lib/desktop/mlx_inference_model.dart new file mode 100644 index 00000000..4451d96b --- /dev/null +++ b/lib/desktop/mlx_inference_model.dart @@ -0,0 +1,335 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; + +import '../core/chat.dart'; +import '../core/message.dart'; +import '../core/model.dart'; +import '../core/tool.dart'; +import '../flutter_gemma_interface.dart'; +import 'mlx_native_dispatch.dart'; + +class MlxInferenceModel extends InferenceModel { + MlxInferenceModel({ + required this.dispatcher, + required this.modelPath, + required this.maxTokens, + required this.modelType, + required this.fileType, + this.supportImage = false, + this.supportAudio = false, + required this.onClose, + }); + + final MlxDispatching dispatcher; + final String modelPath; + @override + final int maxTokens; + final ModelType modelType; + @override + final ModelFileType fileType; + final bool supportImage; + final bool supportAudio; + final VoidCallback onClose; + + MlxInferenceModelSession? _session; + bool _isClosed = false; + + @override + InferenceModelSession? get session => _session; + + @override + Future createSession({ + double temperature = .8, + int randomSeed = 1, + int topK = 1, + double? topP, + String? loraPath, + bool? enableVisionModality, + bool? enableAudioModality, + String? systemInstruction, + bool enableThinking = false, + List tools = const [], + }) async { + if (_isClosed) { + throw StateError( + 'Model is closed. Create a new instance to use it again'); + } + if (loraPath != null) { + throw UnsupportedError( + 'LoRA weights are not yet supported on the MLX desktop runtime ' + '(loraPath=$loraPath). Use a merged MLX model directory instead.', + ); + } + + await _session?.close(); + final session = _session = MlxInferenceModelSession( + dispatcher: dispatcher, + modelPath: modelPath, + maxTokens: maxTokens, + modelType: modelType, + fileType: fileType, + supportImage: enableVisionModality ?? supportImage, + supportAudio: enableAudioModality ?? supportAudio, + temperature: temperature, + topP: topP, + systemInstruction: systemInstruction, + enableThinking: enableThinking, + tools: tools, + onClose: () => _session = null, + ); + return session; + } + + @override + Future createChat({ + double temperature = .8, + int randomSeed = 1, + int topK = 1, + double? topP, + int tokenBuffer = 256, + String? loraPath, + bool? supportImage, + bool? supportAudio, + List tools = const [], + bool? supportsFunctionCalls, + bool isThinking = false, + ModelType? modelType, + ToolChoice toolChoice = ToolChoice.auto, + int? maxFunctionBufferLength, + String? systemInstruction, + }) async { + if (_isClosed) { + throw StateError( + 'Model is closed. Create a new instance to use it again'); + } + chat = InferenceChat( + sessionCreator: () => createSession( + temperature: temperature, + randomSeed: randomSeed, + topK: topK, + topP: topP, + loraPath: loraPath, + enableVisionModality: supportImage ?? this.supportImage, + enableAudioModality: supportAudio ?? this.supportAudio, + systemInstruction: systemInstruction, + enableThinking: isThinking, + tools: tools, + ), + maxTokens: maxTokens, + tokenBuffer: tokenBuffer, + supportImage: supportImage ?? this.supportImage, + supportAudio: supportAudio ?? this.supportAudio, + supportsFunctionCalls: supportsFunctionCalls ?? false, + maxFunctionBufferLength: + maxFunctionBufferLength ?? defaultMaxFunctionBufferLength, + tools: tools, + modelType: modelType ?? this.modelType, + isThinking: isThinking, + fileType: fileType, + toolChoice: toolChoice, + systemInstruction: systemInstruction, + ); + await chat!.initSession(); + return chat!; + } + + @override + Future close() async { + _isClosed = true; + try { + await _session?.close(); + } finally { + _session = null; + onClose(); + } + } +} + +class MlxInferenceModelSession extends InferenceModelSession + with RawSdkResponseSession { + MlxInferenceModelSession({ + required this.dispatcher, + required this.modelPath, + required this.maxTokens, + required this.modelType, + required this.fileType, + required this.supportImage, + required this.supportAudio, + required this.temperature, + required this.topP, + required this.systemInstruction, + required this.enableThinking, + required this.tools, + required this.onClose, + }); + + final MlxDispatching dispatcher; + final String modelPath; + final int maxTokens; + final ModelType modelType; + final ModelFileType fileType; + final bool supportImage; + final bool supportAudio; + final double temperature; + final double? topP; + final String? systemInstruction; + final bool enableThinking; + final List tools; + final VoidCallback onClose; + + final List _history = []; + bool _isClosed = false; + SessionMetrics _metrics = SessionMetrics(); + String? _lastRawResponse; + + @override + String? get lastRawResponse => _lastRawResponse; + + void _assertNotClosed() { + if (_isClosed) { + throw StateError('Session is closed'); + } + } + + @override + Future addQueryChunk(Message message) async { + _assertNotClosed(); + if (message.hasImage) { + throw UnsupportedError( + 'The built-in MLX desktop runtime currently supports text-only prompts. ' + 'Pass image paths through a custom desktop runtime extension instead.', + ); + } + if (message.hasAudio) { + throw UnsupportedError( + 'The built-in MLX desktop runtime currently supports text-only prompts. ' + 'Transcribe audio before sending it to the session.', + ); + } + _history.add(message); + } + + @override + Future getResponse() async { + _assertNotClosed(); + if (_history.isEmpty) { + throw StateError('No query chunks added before getResponse()'); + } + + final payload = { + 'modelPath': modelPath, + 'runtimeAdapter': supportImage ? 'mlx_vlm' : 'mlx_lm', + 'messages': _toWireMessages(), + 'maxTokens': maxTokens, + 'temperature': temperature, + if (topP != null) 'topP': topP, + 'enableThinking': enableThinking, + }; + + final response = dispatcher.invoke('lm.generate', payload); + _lastRawResponse = jsonEncode(response); + + if (response['ok'] != true) { + throw StateError('${response['error'] ?? 'MLX generation failed'}'); + } + + final text = (response['text'] as String?) ?? ''; + _metrics = _metricsFromResponse(response, text: text); + _history.add(Message.text(text: text, isUser: false)); + return text; + } + + @override + Stream getResponseAsync() async* { + yield await getResponse(); + } + + @override + Future sizeInTokens(String text) async => _estimateTokens(text); + + @override + Future stopGeneration() async {} + + @override + SessionMetrics getSessionMetrics() => _metrics; + + @override + Future close() async { + if (_isClosed) { + return; + } + _isClosed = true; + onClose(); + } + + List> _toWireMessages() { + final messages = >[]; + final instruction = systemInstruction?.trim(); + if (instruction != null && instruction.isNotEmpty) { + messages.add({ + 'role': 'system', + 'content': instruction, + }); + } + for (final message in _history) { + messages.add({ + 'role': _roleForMessage(message), + 'content': message.text, + }); + } + return messages; + } + + String _roleForMessage(Message message) { + if (message.type == MessageType.toolResponse) { + return 'tool'; + } + if (message.isUser) { + return 'user'; + } + return 'assistant'; + } + + SessionMetrics _metricsFromResponse( + Map response, { + required String text, + }) { + final inputText = _history.map((message) => message.text).join('\n'); + final inputTokens = _estimateTokens(inputText); + final outputTokens = _estimateTokens(text); + final generateMs = _asDouble(response['swiftGenerateMs']); + final ttftMs = _asDouble(response['swiftFirstTokenMs']); + final loadMs = _asDouble(response['swiftLoadMs']); + final tokensPerSecond = (generateMs != null && generateMs > 0) + ? outputTokens * 1000 / generateMs + : null; + return SessionMetrics( + inputTokens: inputTokens, + outputTokens: outputTokens, + totalTokens: inputTokens + outputTokens, + timeToFirstTokenMs: ttftMs, + tokensPerSecond: tokensPerSecond, + initTimeMs: loadMs, + ); + } + + double? _asDouble(Object? value) { + if (value is int) { + return value.toDouble(); + } + if (value is double) { + return value; + } + return null; + } + + int _estimateTokens(String text) { + final trimmed = text.trim(); + if (trimmed.isEmpty) { + return 0; + } + return ((trimmed.runes.length / 4).ceil().clamp(1, 1 << 30)) as int; + } +} diff --git a/lib/desktop/mlx_native_dispatch.dart b/lib/desktop/mlx_native_dispatch.dart new file mode 100644 index 00000000..b4ecc642 --- /dev/null +++ b/lib/desktop/mlx_native_dispatch.dart @@ -0,0 +1,119 @@ +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:ffi/ffi.dart'; + +abstract interface class MlxDispatching { + Map invoke(String operation, Map payload); + + bool get isBlockingInvoke => true; +} + +typedef _DispatchNative = Pointer Function( + Pointer op, + Pointer json, +); +typedef _DispatchDart = Pointer Function( + Pointer op, + Pointer json, +); +typedef _FreeNative = Void Function(Pointer); +typedef _FreeDart = void Function(Pointer); + +final class MlxNativeDispatcher implements MlxDispatching { + MlxNativeDispatcher(); + + DynamicLibrary? _lib; + _DispatchDart? _dispatch; + _FreeDart? _free; + + @override + bool get isBlockingInvoke => true; + + static bool isAvailable() { + if (!Platform.isMacOS) { + return false; + } + try { + final lib = DynamicLibrary.process(); + lib.lookup>('flm_dispatch_json'); + lib.lookup)>>( + 'flm_bridge_free_string', + ); + return true; + } catch (_) { + return false; + } + } + + void _ensureLoaded() { + if (_lib != null) { + return; + } + if (!Platform.isMacOS) { + throw UnsupportedError('Native MLX dispatch requires macOS.'); + } + _lib = DynamicLibrary.process(); + _dispatch = _lib!.lookupFunction<_DispatchNative, _DispatchDart>( + 'flm_dispatch_json', + ); + _free = _lib!.lookupFunction<_FreeNative, _FreeDart>( + 'flm_bridge_free_string', + ); + } + + @override + Map invoke(String operation, Map payload) { + _ensureLoaded(); + final opPtr = operation.toNativeUtf8(); + final jsonPtr = jsonEncode(payload).toNativeUtf8(); + try { + final outPtr = _dispatch!(opPtr, jsonPtr); + if (outPtr.address == 0) { + throw StateError('flm_dispatch_json returned null'); + } + try { + final jsonStr = outPtr.toDartString(); + final decoded = jsonDecode(jsonStr); + if (decoded is! Map) { + throw FormatException('Expected JSON object, got: $jsonStr'); + } + return Map.from( + decoded.map((key, value) => MapEntry('$key', value)), + ); + } finally { + _free!(outPtr); + } + } finally { + malloc.free(opPtr); + malloc.free(jsonPtr); + } + } +} + +final class RecordingMlxDispatcher implements MlxDispatching { + RecordingMlxDispatcher(); + + @override + bool get isBlockingInvoke => false; + + final List<({String operation, Map payload})> calls = + <({String operation, Map payload})>[]; + + Map Function(String op, Map payload)? + onInvoke; + + @override + Map invoke(String operation, Map payload) { + calls.add(( + operation: operation, + payload: Map.from(payload), + )); + return onInvoke?.call(operation, payload) ?? + { + 'ok': false, + 'error': 'recording dispatcher: no handler configured', + }; + } +} diff --git a/lib/desktop/mlx_runtime_extension.dart b/lib/desktop/mlx_runtime_extension.dart new file mode 100644 index 00000000..ec045847 --- /dev/null +++ b/lib/desktop/mlx_runtime_extension.dart @@ -0,0 +1,44 @@ +import 'dart:io'; + +import 'package:flutter/foundation.dart'; + +import 'desktop_runtime_extension.dart'; +import 'mlx_inference_model.dart'; +import 'mlx_native_dispatch.dart'; + +DesktopRuntimeExtension createBuiltInMlxRuntimeExtension({ + MlxDispatching? dispatcher, +}) { + return DesktopRuntimeExtension( + name: 'mlx', + createInferenceModel: (request) async { + if (!Platform.isMacOS) { + return null; + } + final modelDirectory = Directory(request.modelPath); + if (!await modelDirectory.exists()) { + return null; + } + + final effectiveDispatcher = dispatcher; + if (effectiveDispatcher == null && !MlxNativeDispatcher.isAvailable()) { + debugPrint( + '[FlutterGemmaDesktop/MLX] Model directory detected but ' + 'flm_dispatch_json is not linked into the host process.', + ); + return null; + } + + return MlxInferenceModel( + dispatcher: effectiveDispatcher ?? MlxNativeDispatcher(), + modelPath: request.modelPath, + maxTokens: request.maxTokens, + modelType: request.modelType, + fileType: request.fileType, + supportImage: request.supportImage, + supportAudio: request.supportAudio, + onClose: () {}, + ); + }, + ); +} diff --git a/lib/flutter_gemma.dart b/lib/flutter_gemma.dart index a77ad076..e57b5013 100644 --- a/lib/flutter_gemma.dart +++ b/lib/flutter_gemma.dart @@ -55,5 +55,11 @@ export 'mobile/flutter_gemma_mobile.dart' export 'desktop/flutter_gemma_desktop.dart' if (dart.library.js_interop) 'desktop/flutter_gemma_desktop_stub.dart' show FlutterGemmaDesktop, isDesktop; +export 'desktop/desktop_runtime_extension.dart' + show + DesktopEmbeddingRequest, + DesktopInferenceRequest, + DesktopRuntimeExtension, + DesktopRuntimeRegistry; // ModelReplacePolicy is already exported from model_file_manager_interface.dart diff --git a/test/desktop_runtime_extension_test.dart b/test/desktop_runtime_extension_test.dart new file mode 100644 index 00000000..dca8e118 --- /dev/null +++ b/test/desktop_runtime_extension_test.dart @@ -0,0 +1,208 @@ +import 'package:flutter_gemma/flutter_gemma.dart'; +import 'package:flutter_gemma/core/domain/model_source.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class _FakeInferenceModel extends InferenceModel { + _FakeInferenceModel({this.maxTokensValue = 42}); + + final int maxTokensValue; + bool closed = false; + + @override + InferenceModelSession? get session => null; + + @override + InferenceChat? chat; + + @override + int get maxTokens => maxTokensValue; + + @override + ModelFileType get fileType => ModelFileType.task; + + @override + Future createSession({ + double temperature = .8, + int randomSeed = 1, + int topK = 1, + double? topP, + String? loraPath, + bool? enableVisionModality, + bool? enableAudioModality, + String? systemInstruction, + bool enableThinking = false, + List tools = const [], + }) { + throw UnimplementedError(); + } + + @override + Future close() async { + closed = true; + } +} + +class _FakeEmbeddingModel extends EmbeddingModel { + bool closed = false; + + @override + Future> generateEmbedding( + String text, { + TaskType taskType = TaskType.retrievalQuery, + }) async { + return [text.length.toDouble()]; + } + + @override + Future>> generateEmbeddings( + List texts, { + TaskType taskType = TaskType.retrievalQuery, + }) async { + return texts.map((text) => [text.length.toDouble()]).toList(); + } + + @override + Future getDimension() async => 1; + + @override + Future close() async { + closed = true; + } +} + +InferenceModelSpec _inferenceSpec() => InferenceModelSpec( + name: 'Qwen3 MLX', + modelSource: ModelSource.file('/tmp/qwen3-router-mlx'), + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + ); + +void main() { + group('DesktopRuntimeRegistry', () { + late DesktopRuntimeRegistry registry; + + setUp(() { + registry = DesktopRuntimeRegistry(); + }); + + test('uses the first extension that returns a model', () async { + registry.register( + DesktopRuntimeExtension( + name: 'skip', + createInferenceModel: (_) => null, + ), + ); + registry.register( + DesktopRuntimeExtension( + name: 'mlx', + createInferenceModel: (_) async => _FakeInferenceModel(), + ), + ); + + final model = await registry.createInferenceModel( + DesktopInferenceRequest( + spec: _inferenceSpec(), + modelPath: '/tmp/qwen3-router-mlx', + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + maxTokens: 256, + cacheDir: '/tmp/cache', + ), + ); + + expect(model, isA<_FakeInferenceModel>()); + }); + + test( + 'managed inference model forwards close to lifecycle callback once', + () async { + final fakeModel = _FakeInferenceModel(); + var closeCalls = 0; + registry.register( + DesktopRuntimeExtension( + name: 'mlx', + createInferenceModel: (_) async => fakeModel, + ), + ); + + final model = await registry.createManagedInferenceModel( + DesktopInferenceRequest( + spec: _inferenceSpec(), + modelPath: '/tmp/qwen3-router-mlx', + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + maxTokens: 256, + cacheDir: '/tmp/cache', + ), + onClose: () => closeCalls++, + ); + + expect(model, isNotNull); + await model!.close(); + await model.close(); + + expect(fakeModel.closed, isTrue); + expect(closeCalls, 1); + }, + ); + + test( + 'managed embedding model forwards close to lifecycle callback once', + () async { + final fakeModel = _FakeEmbeddingModel(); + var closeCalls = 0; + registry.register( + DesktopRuntimeExtension( + name: 'mlx-embedding', + createEmbeddingModel: (_) async => fakeModel, + ), + ); + + final model = await registry.createManagedEmbeddingModel( + const DesktopEmbeddingRequest( + modelPath: '/tmp/gecko-mlx', + tokenizerPath: '/tmp/gecko-tokenizer.json', + ), + onClose: () => closeCalls++, + ); + + expect(model, isNotNull); + await model!.close(); + await model.close(); + + expect(fakeModel.closed, isTrue); + expect(closeCalls, 1); + }, + ); + + test('register replaces extension with same name', () async { + registry.register( + DesktopRuntimeExtension( + name: 'mlx', + createInferenceModel: (_) async => null, + ), + ); + registry.register( + DesktopRuntimeExtension( + name: 'mlx', + createInferenceModel: (_) async => + _FakeInferenceModel(maxTokensValue: 7), + ), + ); + + final model = await registry.createInferenceModel( + DesktopInferenceRequest( + spec: _inferenceSpec(), + modelPath: '/tmp/qwen3-router-mlx', + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + maxTokens: 256, + cacheDir: '/tmp/cache', + ), + ); + + expect((model as _FakeInferenceModel).maxTokens, 7); + expect(registry.extensions, hasLength(1)); + }); + }); +} diff --git a/test/file_source_directory_validation_test.dart b/test/file_source_directory_validation_test.dart new file mode 100644 index 00000000..f7b5fdeb --- /dev/null +++ b/test/file_source_directory_validation_test.dart @@ -0,0 +1,39 @@ +import 'dart:io'; + +import 'package:flutter_gemma/core/domain/model_source.dart'; +import 'package:flutter_gemma/core/model.dart'; +import 'package:flutter_gemma/mobile/flutter_gemma_mobile.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('FileSource directory validation', () { + test('accepts non-empty local model directories', () async { + final modelDir = await Directory.systemTemp.createTemp('mlx-model-'); + addTearDown(() => modelDir.delete(recursive: true)); + await File('${modelDir.path}/config.json').writeAsString('{}'); + await File('${modelDir.path}/model.safetensors').writeAsString('weights'); + + final spec = InferenceModelSpec( + name: 'MLX model', + modelSource: ModelSource.file(modelDir.path), + modelType: ModelType.qwen3, + ); + + expect(await ModelFileSystemManager.validateModelFiles(spec), isTrue); + }); + + test('rejects empty local model directories', () async { + final modelDir = + await Directory.systemTemp.createTemp('mlx-model-empty-'); + addTearDown(() => modelDir.delete(recursive: true)); + + final spec = InferenceModelSpec( + name: 'Empty MLX model', + modelSource: ModelSource.file(modelDir.path), + modelType: ModelType.qwen3, + ); + + expect(await ModelFileSystemManager.validateModelFiles(spec), isFalse); + }); + }); +} diff --git a/test/mlx_runtime_extension_test.dart b/test/mlx_runtime_extension_test.dart new file mode 100644 index 00000000..95904913 --- /dev/null +++ b/test/mlx_runtime_extension_test.dart @@ -0,0 +1,119 @@ +import 'dart:io'; + +import 'package:flutter_gemma/core/domain/model_source.dart'; +import 'package:flutter_gemma/flutter_gemma.dart'; +import 'package:flutter_gemma/desktop/mlx_inference_model.dart'; +import 'package:flutter_gemma/desktop/mlx_native_dispatch.dart'; +import 'package:flutter_gemma/desktop/mlx_runtime_extension.dart'; +import 'package:flutter_test/flutter_test.dart'; + +InferenceModelSpec _inferenceSpec(String modelPath) => InferenceModelSpec( + name: 'Qwen3 MLX', + modelSource: ModelSource.file(modelPath), + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + ); + +void main() { + group('built-in MLX runtime extension', () { + test('selects MLX for directory-backed local models', () async { + final modelDir = await Directory.systemTemp.createTemp('mlx-model-'); + addTearDown(() => modelDir.delete(recursive: true)); + + final extension = createBuiltInMlxRuntimeExtension( + dispatcher: RecordingMlxDispatcher(), + ); + + final model = await extension.createInferenceModel!( + DesktopInferenceRequest( + spec: _inferenceSpec(modelDir.path), + modelPath: modelDir.path, + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + maxTokens: 256, + cacheDir: '/tmp/cache', + ), + ); + + expect(model, isA()); + }); + }); + + group('MLX inference model', () { + test('sends structured chat history through the dispatcher', () async { + final dispatcher = RecordingMlxDispatcher() + ..onInvoke = (operation, payload) { + expect(operation, 'lm.generate'); + return { + 'ok': true, + 'text': '{"command":"create_note"}', + 'swiftLoadMs': 5, + 'swiftFirstTokenMs': 12, + 'swiftGenerateMs': 40, + }; + }; + + final model = MlxInferenceModel( + dispatcher: dispatcher, + modelPath: '/tmp/qwen3-router-mlx', + maxTokens: 256, + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + onClose: () {}, + ); + + final session = await model.createSession( + systemInstruction: 'Route user requests to board commands.', + ); + await session.addQueryChunk(Message.text(text: 'hello', isUser: true)); + await session.addQueryChunk(Message.text(text: 'hi', isUser: false)); + await session.addQueryChunk( + Message.text(text: 'create a note called inbox', isUser: true), + ); + + final response = await session.getResponse(); + + expect(response, '{"command":"create_note"}'); + expect(dispatcher.calls, hasLength(1)); + expect( + dispatcher.calls.single.payload['messages'], + >[ + { + 'role': 'system', + 'content': 'Route user requests to board commands.', + }, + {'role': 'user', 'content': 'hello'}, + {'role': 'assistant', 'content': 'hi'}, + {'role': 'user', 'content': 'create a note called inbox'}, + ], + ); + + final metrics = session.getSessionMetrics(); + expect(metrics.timeToFirstTokenMs, 12); + expect(metrics.initTimeMs, 5); + expect(metrics.outputTokens, greaterThan(0)); + }); + + test('async response yields the generated text once', () async { + final dispatcher = RecordingMlxDispatcher() + ..onInvoke = (_, __) => { + 'ok': true, + 'text': 'ok', + }; + + final model = MlxInferenceModel( + dispatcher: dispatcher, + modelPath: '/tmp/qwen3-router-mlx', + maxTokens: 64, + modelType: ModelType.qwen3, + fileType: ModelFileType.task, + onClose: () {}, + ); + + final session = await model.createSession(); + await session.addQueryChunk(Message.text(text: 'ping', isUser: true)); + + expect(await session.getResponseAsync().toList(), ['ok']); + }); + }); +}