From bad457639429b673136b294c53afe861caa066cc Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:16:26 +0200 Subject: [PATCH 01/20] feat(genai): add genai_primitives + http deps and side barrel (#181) --- packages/flutter_gemma/lib/genai.dart | 8 ++++++++ packages/flutter_gemma/pubspec.yaml | 2 ++ .../flutter_gemma/test/genai/barrel_test.dart | 10 ++++++++++ pubspec.lock | 16 ++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 packages/flutter_gemma/lib/genai.dart create mode 100644 packages/flutter_gemma/test/genai/barrel_test.dart diff --git a/packages/flutter_gemma/lib/genai.dart b/packages/flutter_gemma/lib/genai.dart new file mode 100644 index 00000000..d2ee9c4e --- /dev/null +++ b/packages/flutter_gemma/lib/genai.dart @@ -0,0 +1,8 @@ +/// genai_primitives adoption surface for flutter_gemma (#181). +/// +/// A side barrel (NOT re-exported from `flutter_gemma.dart`) so genai_primitives +/// 0.x churn stays contained. Import `package:flutter_gemma/genai.dart` to use +/// [ChatMessage] with [InferenceChat.sendMessage] / [generateContent]. +library; + +export 'package:genai_primitives/genai_primitives.dart'; diff --git a/packages/flutter_gemma/pubspec.yaml b/packages/flutter_gemma/pubspec.yaml index f1f89572..abd0d747 100644 --- a/packages/flutter_gemma/pubspec.yaml +++ b/packages/flutter_gemma/pubspec.yaml @@ -33,6 +33,8 @@ dependencies: hooks: ^2.0.0 code_assets: ^1.2.0 crypto: ^3.0.6 + genai_primitives: ^0.2.3 + http: ^1.6.0 # Annotations (@immutable, @visibleForTesting) used across core, e.g. chat.dart. meta: ^1.15.0 # Serializes native LiteRT-LM conversation calls across concurrent diff --git a/packages/flutter_gemma/test/genai/barrel_test.dart b/packages/flutter_gemma/test/genai/barrel_test.dart new file mode 100644 index 00000000..8c3fea4a --- /dev/null +++ b/packages/flutter_gemma/test/genai/barrel_test.dart @@ -0,0 +1,10 @@ +import 'package:flutter_gemma/genai.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('genai barrel re-exports genai_primitives core types', () { + final msg = ChatMessage.user('hello'); + expect(msg.role, ChatMessageRole.user); + expect(msg.parts.whereType().first.text, 'hello'); + }); +} diff --git a/pubspec.lock b/pubspec.lock index 6986cf51..770bc669 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -225,6 +225,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.15.1" + cross_file: + dependency: transitive + description: + name: cross_file + sha256: "92c9c43c383bfa1c32079d3bc492d55d6d4318044b7b47edaff8971cbb555c51" + url: "https://pub.dev" + source: hosted + version: "0.3.5+4" crypto: dependency: transitive description: @@ -456,6 +464,14 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.0" + genai_primitives: + dependency: transitive + description: + name: genai_primitives + sha256: "2468ecd2984f32232ab199315e384821c61a198e356828e34b9a5bcf0fda863e" + url: "https://pub.dev" + source: hosted + version: "0.2.3" genkit: dependency: transitive description: From 0e53cbb6e7752960d498e7f56a0b7ffda570539e Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:19:48 +0200 Subject: [PATCH 02/20] feat(genai): conditional-import LinkPart reader (data/file/http) (#181) --- .../lib/core/genai/link_reader.dart | 39 +++++++++++++++++++ .../lib/core/genai/link_reader_io.dart | 7 ++++ .../lib/core/genai/link_reader_stub.dart | 7 ++++ .../test/genai/link_reader_test.dart | 21 ++++++++++ 4 files changed, 74 insertions(+) create mode 100644 packages/flutter_gemma/lib/core/genai/link_reader.dart create mode 100644 packages/flutter_gemma/lib/core/genai/link_reader_io.dart create mode 100644 packages/flutter_gemma/lib/core/genai/link_reader_stub.dart create mode 100644 packages/flutter_gemma/test/genai/link_reader_test.dart diff --git a/packages/flutter_gemma/lib/core/genai/link_reader.dart b/packages/flutter_gemma/lib/core/genai/link_reader.dart new file mode 100644 index 00000000..56547abd --- /dev/null +++ b/packages/flutter_gemma/lib/core/genai/link_reader.dart @@ -0,0 +1,39 @@ +import 'dart:convert'; +import 'dart:typed_data'; +import 'package:http/http.dart' as http; + +import 'link_reader_stub.dart' if (dart.library.io) 'link_reader_io.dart'; + +/// Resolves a genai_primitives LinkPart URI to bytes. +/// - data: decoded in-memory (no I/O) +/// - file:// dart:io on native; throws on web +/// - http(s):: injected client, else default http.Client (wasm-safe BrowserClient on web) +Future readLinkBytes(Uri uri, {http.Client? httpClient}) async { + switch (uri.scheme) { + case 'data': + final data = uri.data; + if (data == null) { + throw ArgumentError('Malformed data: URI: $uri'); + } + return data.contentAsBytes(); + case 'http': + case 'https': + final client = httpClient ?? http.Client(); + try { + final resp = await client.get(uri); + if (resp.statusCode != 200) { + throw http.ClientException('HTTP ${resp.statusCode} for $uri', uri); + } + return resp.bodyBytes; + } finally { + if (httpClient == null) client.close(); + } + case 'file': + return readFileUri(uri); + default: + if (uri.scheme.isEmpty) return readFileUri(uri); // bare path + throw UnsupportedError( + 'Unsupported LinkPart scheme "${uri.scheme}": $uri', + ); + } +} diff --git a/packages/flutter_gemma/lib/core/genai/link_reader_io.dart b/packages/flutter_gemma/lib/core/genai/link_reader_io.dart new file mode 100644 index 00000000..b422e9e4 --- /dev/null +++ b/packages/flutter_gemma/lib/core/genai/link_reader_io.dart @@ -0,0 +1,7 @@ +import 'dart:io'; +import 'dart:typed_data'; + +Future readFileUri(Uri uri) async { + final path = uri.scheme == 'file' ? uri.toFilePath() : uri.path; + return File(path).readAsBytes(); +} diff --git a/packages/flutter_gemma/lib/core/genai/link_reader_stub.dart b/packages/flutter_gemma/lib/core/genai/link_reader_stub.dart new file mode 100644 index 00000000..69e9da1d --- /dev/null +++ b/packages/flutter_gemma/lib/core/genai/link_reader_stub.dart @@ -0,0 +1,7 @@ +import 'dart:typed_data'; + +/// Web: no filesystem. Callers must pass DataPart bytes, not file:// links. +Future readFileUri(Uri uri) => throw UnimplementedError( + 'file:// links are not readable on web ($uri). ' + 'Resolve to DataPart bytes before sending.', +); diff --git a/packages/flutter_gemma/test/genai/link_reader_test.dart b/packages/flutter_gemma/test/genai/link_reader_test.dart new file mode 100644 index 00000000..053b950e --- /dev/null +++ b/packages/flutter_gemma/test/genai/link_reader_test.dart @@ -0,0 +1,21 @@ +import 'dart:convert'; +import 'dart:typed_data'; +import 'package:flutter_gemma/core/genai/link_reader.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('resolves a data: URI to bytes', () async { + final bytes = Uint8List.fromList([1, 2, 3, 4]); + final uri = Uri.parse( + 'data:application/octet-stream;base64,${base64Encode(bytes)}', + ); + expect(await readLinkBytes(uri), bytes); + }); + + test('throws on an unknown scheme', () async { + expect( + () => readLinkBytes(Uri.parse('ftp://x/y')), + throwsA(isA()), + ); + }); +} From 0436481c789e23b7095534213dccd60d2bdf5b0f Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:25:56 +0200 Subject: [PATCH 03/20] =?UTF-8?q?feat(genai):=20ChatMessage=20=E2=86=92=20?= =?UTF-8?q?List=20input=20converter=20(#181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/core/genai/genai_input_converter.dart | 111 ++++++++++++++++++ .../genai/genai_input_converter_test.dart | 87 ++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 packages/flutter_gemma/lib/core/genai/genai_input_converter.dart create mode 100644 packages/flutter_gemma/test/genai/genai_input_converter_test.dart diff --git a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart new file mode 100644 index 00000000..079793be --- /dev/null +++ b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart @@ -0,0 +1,111 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:genai_primitives/genai_primitives.dart'; +import 'package:http/http.dart' as http; + +import '../message.dart'; +import 'link_reader.dart'; + +/// Converts a genai_primitives [ChatMessage] into flutter_gemma [Message]s. +/// +/// One ChatMessage may yield >1 Message (tool results become sibling messages). +/// Async because LinkPart resolution may need I/O. Throws — never silently drops. +Future> messagesFromChatMessage( + ChatMessage message, { + http.Client? httpClient, +}) async { + if (message.role == ChatMessageRole.system) { + throw ArgumentError( + 'System messages are not input — set them via createChat(systemInstruction:).', + ); + } + final parts = message.parts; + if (parts.isEmpty) { + throw ArgumentError('ChatMessage has no parts.'); + } + final isUser = message.role == ChatMessageRole.user; + + final textBuffer = StringBuffer(); + final images = []; + Uint8List? audio; + final messages = []; + + for (final part in parts) { + switch (part) { + case TextPart(:final text): + textBuffer.write(text); + case DataPart(:final bytes, :final mimeType): + _routeMedia(bytes, mimeType, images, () => audio, (v) => audio = v); + case LinkPart(:final url, :final mimeType): + final bytes = await readLinkBytes(url, httpClient: httpClient); + _routeMedia(bytes, mimeType, images, () => audio, (v) => audio = v); + case ToolPart(kind: ToolPartKind.result, :final toolName, :final result): + final resp = result is Map + ? result + : {'result': result}; + messages.add(Message.toolResponse(toolName: toolName, response: resp)); + case ToolPart(kind: ToolPartKind.call, :final toolName, :final arguments): + if (isUser) { + throw UnsupportedError( + 'A tool call is model output, not user input.', + ); + } + messages.add( + Message.toolCall( + text: jsonEncode({'name': toolName, 'parameters': arguments ?? {}}), + ), + ); + case ThinkingPart(): + throw UnsupportedError( + 'ThinkingPart is model output (streamed), not input.', + ); + } + } + + final hasText = textBuffer.isNotEmpty; + if (hasText || images.isNotEmpty || audio != null) { + messages.insert( + 0, + Message( + text: textBuffer.toString(), + isUser: isUser, + images: images, + imageBytes: images.isNotEmpty ? images.first : null, + audioBytes: audio, + ), + ); + } + return messages; +} + +Future> messagesFromChatMessages( + List messages, { + http.Client? httpClient, +}) async { + final out = []; + for (final m in messages) { + out.addAll(await messagesFromChatMessage(m, httpClient: httpClient)); + } + return out; +} + +void _routeMedia( + Uint8List bytes, + String? mimeType, + List images, + Uint8List? Function() getAudio, + void Function(Uint8List) setAudio, +) { + final mime = mimeType ?? ''; + if (mime.startsWith('image/')) { + images.add(bytes); + } else if (mime.startsWith('audio/')) { + if (getAudio() != null) { + throw UnsupportedError('Only one audio part per message is supported.'); + } + setAudio(bytes); + } else { + throw UnsupportedError('Unsupported DataPart mime "$mime".'); + } +} diff --git a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart new file mode 100644 index 00000000..e126601d --- /dev/null +++ b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart @@ -0,0 +1,87 @@ +import 'dart:typed_data'; +import 'package:flutter_gemma/genai.dart'; +import 'package:flutter_gemma/core/genai/genai_input_converter.dart'; +import 'package:flutter_gemma/core/message.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('user text → one user Message', () async { + final out = await messagesFromChatMessage(ChatMessage.user('hi')); + expect(out, hasLength(1)); + expect(out.first.text, 'hi'); + expect(out.first.isUser, isTrue); + }); + + test( + 'user text + two image DataParts → one Message with both images', + () async { + final a = Uint8List.fromList([1]); + final b = Uint8List.fromList([2]); + final msg = ChatMessage.user( + 'look', + parts: [ + TextPart('look'), + DataPart(a, mimeType: 'image/png'), + DataPart(b, mimeType: 'image/png'), + ], + ); + final out = await messagesFromChatMessage(msg); + expect(out, hasLength(1)); + expect(out.first.images, [a, b]); + }, + ); + + test('model role → non-user Message', () async { + final out = await messagesFromChatMessage(ChatMessage.model('ok')); + expect(out.single.isUser, isFalse); + }); + + test('system role throws', () async { + expect( + () => messagesFromChatMessage(ChatMessage.system('sys')), + throwsA(isA()), + ); + }); + + test('ThinkingPart throws', () async { + final msg = ChatMessage( + role: ChatMessageRole.model, + parts: [const ThinkingPart('t')], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); + + test('tool result → Message.toolResponse', () async { + final msg = ChatMessage( + role: ChatMessageRole.user, + parts: [ + ToolPart.result(callId: '1', toolName: 'calc', result: {'v': 7}), + ], + ); + final out = await messagesFromChatMessage(msg); + expect(out.single.type, MessageType.toolResponse); + expect(out.single.toolName, 'calc'); + }); + + test('second audio DataPart throws', () async { + final msg = ChatMessage( + role: ChatMessageRole.user, + parts: [ + DataPart(Uint8List.fromList([1]), mimeType: 'audio/wav'), + DataPart(Uint8List.fromList([2]), mimeType: 'audio/wav'), + ], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); + + test('empty parts throws', () async { + final msg = ChatMessage(role: ChatMessageRole.user, parts: const []); + expect(() => messagesFromChatMessage(msg), throwsA(isA())); + }); +} From a2b5f0726ad2300f19cf06489242ea78d32e94dc Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:29:57 +0200 Subject: [PATCH 04/20] test(genai): cover tool-call-in-user + non-media-mime throw paths (#181) --- .../genai/genai_input_converter_test.dart | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart index e126601d..8aa9f220 100644 --- a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart +++ b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart @@ -84,4 +84,30 @@ void main() { final msg = ChatMessage(role: ChatMessageRole.user, parts: const []); expect(() => messagesFromChatMessage(msg), throwsA(isA())); }); + + test('tool call in a user-role message throws', () async { + final msg = ChatMessage( + role: ChatMessageRole.user, + parts: [ + ToolPart.call(callId: '1', toolName: 'calc', arguments: {'a': 1}), + ], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); + + test('non-media DataPart mime throws', () async { + final msg = ChatMessage( + role: ChatMessageRole.user, + parts: [ + DataPart(Uint8List.fromList([1, 2]), mimeType: 'application/pdf'), + ], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); } From 7efd84fe6043afecd21331b20abe262fe7b140c2 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:36:31 +0200 Subject: [PATCH 05/20] =?UTF-8?q?feat(genai):=20ModelResponse=20=E2=86=92?= =?UTF-8?q?=20ChatMessage=20output=20converter=20(#181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/genai/genai_output_converter.dart | 57 +++++++++++++++++++ .../genai/genai_output_converter_test.dart | 41 +++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 packages/flutter_gemma/lib/core/genai/genai_output_converter.dart create mode 100644 packages/flutter_gemma/test/genai/genai_output_converter_test.dart diff --git a/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart new file mode 100644 index 00000000..8448316a --- /dev/null +++ b/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart @@ -0,0 +1,57 @@ +import 'package:genai_primitives/genai_primitives.dart'; +// FinishStatus is a public field type on ChatMessage but genai_primitives +// 0.2.3 doesn't re-export it from the top-level barrel (package gap). +// ignore: implementation_imports +import 'package:genai_primitives/src/finish_status.dart' show FinishStatus; + +import '../model_response.dart'; + +/// One streamed [ModelResponse] variant → one single-part model [ChatMessage]. +ChatMessage chatMessageFromChunk(ModelResponse chunk) { + final parts = []; + switch (chunk) { + case TextResponse(:final token): + parts.add(TextPart(token)); + case ThinkingResponse(:final content): + parts.add(ThinkingPart(content)); + case FunctionCallResponse(:final name, :final args): + parts.add(ToolPart.call(callId: name, toolName: name, arguments: args)); + case ParallelFunctionCallResponse(:final calls): + for (final c in calls) { + parts.add( + ToolPart.call(callId: c.name, toolName: c.name, arguments: c.args), + ); + } + } + return ChatMessage(role: ChatMessageRole.model, parts: parts); +} + +/// Coalesce a whole model turn into one [ChatMessage]. +/// Order: ThinkingPart → (ToolPart.call… OR TextPart). Tool calls suppress text. +ChatMessage chatMessageFromParts({ + String? text, + List calls = const [], + String? thinking, + FinishStatus? finishStatus, + Map metadata = const {}, +}) { + final parts = []; + if (thinking != null && thinking.isNotEmpty) { + parts.add(ThinkingPart(thinking)); + } + if (calls.isNotEmpty) { + for (final c in calls) { + parts.add( + ToolPart.call(callId: c.name, toolName: c.name, arguments: c.args), + ); + } + } else if (text != null && text.isNotEmpty) { + parts.add(TextPart(text)); + } + return ChatMessage( + role: ChatMessageRole.model, + parts: parts, + metadata: metadata, + finishStatus: finishStatus, + ); +} diff --git a/packages/flutter_gemma/test/genai/genai_output_converter_test.dart b/packages/flutter_gemma/test/genai/genai_output_converter_test.dart new file mode 100644 index 00000000..822b4d1d --- /dev/null +++ b/packages/flutter_gemma/test/genai/genai_output_converter_test.dart @@ -0,0 +1,41 @@ +import 'package:flutter_gemma/genai.dart'; +import 'package:flutter_gemma/core/genai/genai_output_converter.dart'; +import 'package:flutter_gemma/core/model_response.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('TextResponse chunk → model ChatMessage with one TextPart', () { + final m = chatMessageFromChunk(const TextResponse('hi')); + expect(m.role, ChatMessageRole.model); + expect(m.parts.whereType().single.text, 'hi'); + }); + + test('ThinkingResponse chunk → ThinkingPart', () { + final m = chatMessageFromChunk(const ThinkingResponse('reason')); + expect(m.parts.whereType().single.text, 'reason'); + }); + + test('FunctionCallResponse chunk → ToolPart.call', () { + final m = chatMessageFromChunk( + const FunctionCallResponse(name: 'calc', args: {'a': 1}), + ); + final tp = m.parts.whereType().single; + expect(tp.kind, ToolPartKind.call); + expect(tp.toolName, 'calc'); + }); + + test('coalesced: thinking + text order', () { + final m = chatMessageFromParts(text: 'answer', thinking: 'think'); + expect(m.parts.first, isA()); + expect(m.parts.whereType().single.text, 'answer'); + }); + + test('coalesced: tool calls suppress text', () { + final m = chatMessageFromParts( + text: 'ignored', + calls: const [FunctionCallResponse(name: 'f', args: {})], + ); + expect(m.parts.whereType(), hasLength(1)); + expect(m.parts.whereType(), isEmpty); + }); +} From 556a61c9f51e350daefba8bae1154155aa02f79f Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:39:32 +0200 Subject: [PATCH 06/20] refactor(genai): drop unused finishStatus + private genai_primitives import (#181) --- .../lib/core/genai/genai_output_converter.dart | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart index 8448316a..b7b156fe 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart @@ -1,8 +1,4 @@ import 'package:genai_primitives/genai_primitives.dart'; -// FinishStatus is a public field type on ChatMessage but genai_primitives -// 0.2.3 doesn't re-export it from the top-level barrel (package gap). -// ignore: implementation_imports -import 'package:genai_primitives/src/finish_status.dart' show FinishStatus; import '../model_response.dart'; @@ -32,7 +28,6 @@ ChatMessage chatMessageFromParts({ String? text, List calls = const [], String? thinking, - FinishStatus? finishStatus, Map metadata = const {}, }) { final parts = []; @@ -52,6 +47,5 @@ ChatMessage chatMessageFromParts({ role: ChatMessageRole.model, parts: parts, metadata: metadata, - finishStatus: finishStatus, ); } From cc48310c014de34c89052781758532d59c9b0d10 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:41:42 +0200 Subject: [PATCH 07/20] feat(genai): capability guard prevents silent image/audio/tool drop (#181) --- .../lib/core/genai/genai_input_converter.dart | 29 ++++++++++ .../test/genai/capability_guard_test.dart | 54 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 packages/flutter_gemma/test/genai/capability_guard_test.dart diff --git a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart index 079793be..02fcce87 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart @@ -109,3 +109,32 @@ void _routeMedia( throw UnsupportedError('Unsupported DataPart mime "$mime".'); } } + +/// Throws if any message needs a capability the chat wasn't built with, +/// so the engine never silently drops image/audio/tool content. +void assertMessagesFitChat( + List messages, { + required bool supportImage, + required bool supportAudio, + required bool supportsFunctionCalls, +}) { + for (final m in messages) { + if (m.hasImage && !supportImage) { + throw UnsupportedError( + 'This chat was created without image support; recreate it with a vision model.', + ); + } + if (m.hasAudio && !supportAudio) { + throw UnsupportedError( + 'This chat was created without audio support; recreate it with an audio model.', + ); + } + if ((m.type == MessageType.toolResponse || + m.type == MessageType.toolCall) && + !supportsFunctionCalls) { + throw UnsupportedError( + 'This chat was created without function-call support (tools were not passed to createChat).', + ); + } + } +} diff --git a/packages/flutter_gemma/test/genai/capability_guard_test.dart b/packages/flutter_gemma/test/genai/capability_guard_test.dart new file mode 100644 index 00000000..1d6b8e70 --- /dev/null +++ b/packages/flutter_gemma/test/genai/capability_guard_test.dart @@ -0,0 +1,54 @@ +import 'dart:typed_data'; +import 'package:flutter_gemma/core/genai/genai_input_converter.dart'; +import 'package:flutter_gemma/core/message.dart'; +import 'package:flutter_test/flutter_test.dart'; + +void main() { + test('image on a non-vision chat throws', () { + final msgs = [ + Message.withImage( + text: 'x', + imageBytes: Uint8List.fromList([1]), + isUser: true, + ), + ]; + expect( + () => assertMessagesFitChat( + msgs, + supportImage: false, + supportAudio: true, + supportsFunctionCalls: true, + ), + throwsA(isA()), + ); + }); + + test('audio on a non-audio chat throws', () { + final msgs = [ + Message.withAudio( + text: 'x', + audioBytes: Uint8List.fromList([1]), + isUser: true, + ), + ]; + expect( + () => assertMessagesFitChat( + msgs, + supportImage: true, + supportAudio: false, + supportsFunctionCalls: true, + ), + throwsA(isA()), + ); + }); + + test('capable chat passes', () { + final msgs = [Message(text: 'x', isUser: true)]; + assertMessagesFitChat( + msgs, + supportImage: true, + supportAudio: true, + supportsFunctionCalls: true, + ); + }); +} From eed4923a26e3c299682dd99ec0809d98298e4d36 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:46:06 +0200 Subject: [PATCH 08/20] feat(genai): sendMessage/sendMessageStream/generateContent on InferenceChat (#181) --- packages/flutter_gemma/lib/core/chat.dart | 5 + .../lib/core/genai/genai_chat_extension.dart | 159 ++++++++++++++++++ .../test/genai/genai_chat_extension_test.dart | 16 ++ 3 files changed, 180 insertions(+) create mode 100644 packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart create mode 100644 packages/flutter_gemma/test/genai/genai_chat_extension_test.dart diff --git a/packages/flutter_gemma/lib/core/chat.dart b/packages/flutter_gemma/lib/core/chat.dart index fe9fcb99..e495ef5a 100644 --- a/packages/flutter_gemma/lib/core/chat.dart +++ b/packages/flutter_gemma/lib/core/chat.dart @@ -8,6 +8,7 @@ import 'package:flutter_gemma/core/model_response.dart'; import 'package:flutter_gemma/core/parsing/sdk_response_parser.dart'; import 'package:flutter_gemma/core/tool.dart'; import 'package:flutter_gemma/flutter_gemma_interface.dart'; +import 'package:mutex/mutex.dart'; import 'model.dart'; import 'package:flutter_gemma/core/utils/gemma_log.dart'; @@ -31,6 +32,10 @@ class InferenceChat { late InferenceModelSession session; final List tools; + /// Serializes genai_primitives sendMessage/generateContent calls so + /// concurrent turns can't interleave staging into the shared session buffer. + final Mutex genaiLock = Mutex(); + final List _fullHistory = []; final List _prefixes = []; // Prefix messages (tools prompt + context message) for replay across sessions diff --git a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart new file mode 100644 index 00000000..24c16364 --- /dev/null +++ b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart @@ -0,0 +1,159 @@ +import 'dart:async'; + +import 'package:genai_primitives/genai_primitives.dart'; +import 'package:http/http.dart' as http; + +import '../chat.dart'; +import '../model_response.dart'; +import 'genai_input_converter.dart'; +import 'genai_output_converter.dart'; + +/// Guard: a model-role ChatMessage is output, not a sendMessage input. +void rejectModelRole(ChatMessage message) { + if (message.role == ChatMessageRole.model) { + throw ArgumentError( + 'sendMessage takes user input; a model turn is output. ' + 'Use generateContent(List) to stage prior model turns.', + ); + } +} + +/// genai_primitives entry points on [InferenceChat]. See #181. +extension GenAiChat on InferenceChat { + /// Send one turn; returns the model turn as a role:model [ChatMessage] + /// (text + tool calls + thinking as parts). + Future sendMessage( + ChatMessage message, { + http.Client? httpClient, + }) { + rejectModelRole(message); + return genaiLock.protect(() async { + await _stage([message], httpClient: httpClient); + return _foldToChatMessage(); + }); + } + + /// Streaming variant — partial role:model ChatMessages, one per delta. + Stream sendMessageStream( + ChatMessage message, { + http.Client? httpClient, + }) { + rejectModelRole(message); + return _lockedStream(() async { + await _stage([message], httpClient: httpClient); + }); + } + + /// STATEFUL batch: stage the whole list into THIS chat, then generate once. + Future generateContent( + List prompt, { + http.Client? httpClient, + }) { + return genaiLock.protect(() async { + await _stage(prompt, httpClient: httpClient); + return _foldToChatMessage(); + }); + } + + Stream generateContentStream( + List prompt, { + http.Client? httpClient, + }) { + return _lockedStream(() async { + await _stage(prompt, httpClient: httpClient); + }); + } + + // --- internals --- + + Future _stage( + List prompt, { + http.Client? httpClient, + }) async { + final messages = await messagesFromChatMessages( + prompt, + httpClient: httpClient, + ); + assertMessagesFitChat( + messages, + supportImage: supportImage, + supportAudio: supportAudio, + supportsFunctionCalls: supportsFunctionCalls, + ); + for (final m in messages) { + await addQueryChunk(m); + } + } + + /// Drive the async generate path, folding all events into one ChatMessage. + Future _foldToChatMessage() async { + final text = StringBuffer(); + final thinking = StringBuffer(); + final calls = []; + await for (final r in generateChatResponseAsync()) { + switch (r) { + case TextResponse(:final token): + text.write(token); + case ThinkingResponse(:final content): + thinking.write(content); + case FunctionCallResponse(): + calls.add(r); + case ParallelFunctionCallResponse(calls: final parallelCalls): + calls.addAll(parallelCalls); + } + } + return chatMessageFromParts( + text: text.toString(), + thinking: thinking.toString(), + calls: calls, + ); + } + + /// Wrap the generate stream, mapping each event to a ChatMessage and holding + /// the chat mutex until a terminal event (done/error/cancel). + Stream _lockedStream(Future Function() stage) { + late StreamController controller; + StreamSubscription? sub; + var released = false; + void release() { + if (!released) { + released = true; + genaiLock.release(); + } + } + + controller = StreamController( + onListen: () async { + await genaiLock.acquire(); + try { + await stage(); + sub = generateChatResponseAsync().listen( + (r) => controller.add(chatMessageFromChunk(r)), + onError: (Object e, StackTrace s) { + controller.addError(e, s); + release(); + controller.close(); + }, + onDone: () { + release(); + controller.close(); + }, + ); + } catch (e, s) { + controller.addError(e, s); + release(); + await controller.close(); + } + }, + onCancel: () async { + await sub?.cancel(); + try { + await stopGeneration(); + } finally { + release(); + } + }, + ); + return controller.stream; + } +} diff --git a/packages/flutter_gemma/test/genai/genai_chat_extension_test.dart b/packages/flutter_gemma/test/genai/genai_chat_extension_test.dart new file mode 100644 index 00000000..b108d822 --- /dev/null +++ b/packages/flutter_gemma/test/genai/genai_chat_extension_test.dart @@ -0,0 +1,16 @@ +import 'package:flutter_gemma/genai.dart'; +import 'package:flutter_gemma/core/genai/genai_chat_extension.dart'; +import 'package:flutter_test/flutter_test.dart'; + +// This suite verifies the guard behavior that needs no live engine. +void main() { + test('sendMessage rejects a model-role ChatMessage', () async { + // Uses the top-level guard the extension calls; see rejectModelRole. + expect( + () => rejectModelRole(ChatMessage.model('echo')), + throwsA(isA()), + ); + // A user message passes the guard (no throw). + rejectModelRole(ChatMessage.user('hi')); + }); +} From 4c150c28d25890b7ac7600cc237ef4d01143233b Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 18:49:54 +0200 Subject: [PATCH 09/20] feat(genai): export sendMessage/generateContent API from genai barrel (#181) --- packages/flutter_gemma/lib/genai.dart | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/flutter_gemma/lib/genai.dart b/packages/flutter_gemma/lib/genai.dart index d2ee9c4e..2adbc4cf 100644 --- a/packages/flutter_gemma/lib/genai.dart +++ b/packages/flutter_gemma/lib/genai.dart @@ -6,3 +6,8 @@ library; export 'package:genai_primitives/genai_primitives.dart'; +export 'core/genai/genai_chat_extension.dart' show GenAiChat; +export 'core/genai/genai_input_converter.dart' + show messagesFromChatMessage, messagesFromChatMessages; +export 'core/genai/genai_output_converter.dart' + show chatMessageFromChunk, chatMessageFromParts; From 78222ef9fff519bb534183246eefbd1b83c0e304 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 20:15:05 +0200 Subject: [PATCH 10/20] test(genai): E2E sendMessage/sendMessageStream round-trip (#181) --- .../genai_primitives_test.dart | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 packages/flutter_gemma/example/integration_test/genai_primitives_test.dart diff --git a/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart b/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart new file mode 100644 index 00000000..dde9ab21 --- /dev/null +++ b/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart @@ -0,0 +1,42 @@ +import 'package:flutter_gemma/flutter_gemma.dart'; +import 'package:flutter_gemma/genai.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets('sendMessage returns a model ChatMessage with text', ( + tester, + ) async { + final model = await FlutterGemma.getActiveModel(maxTokens: 1024); + final chat = await model.createChat(); + final reply = await chat.sendMessage( + ChatMessage.user('Say hello in one word.'), + ); + expect(reply.role, ChatMessageRole.model); + expect( + reply.parts.whereType().map((p) => p.text).join(), + isNotEmpty, + ); + await chat.session.close(); + await model.close(); + }); + + testWidgets('sendMessageStream emits partial model ChatMessages', ( + tester, + ) async { + final model = await FlutterGemma.getActiveModel(maxTokens: 1024); + final chat = await model.createChat(); + final chunks = []; + await for (final c in chat.sendMessageStream( + ChatMessage.user('Count to three.'), + )) { + chunks.add(c); + } + expect(chunks, isNotEmpty); + expect(chunks.every((c) => c.role == ChatMessageRole.model), isTrue); + await chat.session.close(); + await model.close(); + }); +} From 46335a64a9e7dab5746eaee8d1a8d631e929500e Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 20:21:16 +0200 Subject: [PATCH 11/20] refactor(genai): drop dead import + cancel/stop on stream error path (#181) --- .../lib/core/genai/genai_chat_extension.dart | 11 ++++++++--- .../flutter_gemma/lib/core/genai/link_reader.dart | 1 - 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart index 24c16364..b62b1417 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart @@ -129,10 +129,15 @@ extension GenAiChat on InferenceChat { await stage(); sub = generateChatResponseAsync().listen( (r) => controller.add(chatMessageFromChunk(r)), - onError: (Object e, StackTrace s) { + onError: (Object e, StackTrace s) async { controller.addError(e, s); - release(); - controller.close(); + await sub?.cancel(); + try { + await stopGeneration(); + } finally { + release(); + await controller.close(); + } }, onDone: () { release(); diff --git a/packages/flutter_gemma/lib/core/genai/link_reader.dart b/packages/flutter_gemma/lib/core/genai/link_reader.dart index 56547abd..5c263396 100644 --- a/packages/flutter_gemma/lib/core/genai/link_reader.dart +++ b/packages/flutter_gemma/lib/core/genai/link_reader.dart @@ -1,4 +1,3 @@ -import 'dart:convert'; import 'dart:typed_data'; import 'package:http/http.dart' as http; From 0520d91272c04b750a51eb574ec75a5c237ac40c Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Tue, 7 Jul 2026 21:42:19 +0200 Subject: [PATCH 12/20] test(genai): E2E coverage for all methods + vision/audio/thinking/tools/multi-turn (#181) --- .../genai_primitives_test.dart | 351 +++++++++++++++++- 1 file changed, 341 insertions(+), 10 deletions(-) diff --git a/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart b/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart index dde9ab21..fb4d95f9 100644 --- a/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart +++ b/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart @@ -1,15 +1,160 @@ +/// End-to-end integration tests for the genai_primitives adoption surface +/// (#181): `sendMessage` / `sendMessageStream` / `generateContent` / +/// `generateContentStream` on `InferenceChat`, exercised across ALL +/// generation types — text, vision, audio, thinking, tool calls, and +/// multi-turn history — not just plain text. +/// +/// Model: `gemma-4-E2B-it.litertlm` (multimodal + thinking + tools in one +/// model — mirrors the setup machinery in litertlm_ffi_test.dart). +/// +/// Run: +/// flutter test integration_test/genai_primitives_test.dart -d +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter/services.dart' show rootBundle; import 'package:flutter_gemma/flutter_gemma.dart'; import 'package:flutter_gemma/genai.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:integration_test/integration_test.dart'; +import 'inference_test_helpers.dart' show registerTestEngines; + +const _gemma4Url = + 'https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm/resolve/main/gemma-4-E2B-it.litertlm'; + +const _token = String.fromEnvironment('HUGGINGFACE_TOKEN'); + +String get _androidDir => '/data/local/tmp/flutter_gemma_test'; +String get _macosDir => + '${Platform.environment['HOME']}/Library/Containers/dev.flutterberlin.flutterGemmaExample55/Data/Documents'; +String get _linuxDir => '${Platform.environment['HOME']}/models'; +String get _windowsDir => '${Platform.environment['USERPROFILE']}\\models'; + +Uint8List _testImage = Uint8List(0); +Uint8List _testAudio = Uint8List(0); + +const _getWeatherTool = Tool( + name: 'get_weather', + description: 'Get the current weather for a location.', + parameters: { + 'type': 'object', + 'properties': { + 'location': {'type': 'string', 'description': 'City name, e.g. Berlin.'}, + }, + 'required': ['location'], + }, +); + +String? _localPath(String filename) { + if (Platform.isAndroid) return '$_androidDir/$filename'; + if (Platform.isMacOS) return '$_macosDir/$filename'; + if (Platform.isLinux) return '$_linuxDir/$filename'; + if (Platform.isWindows) return '$_windowsDir\\$filename'; + if (Platform.isIOS) { + const iosDocs = String.fromEnvironment('IOS_TEST_DOCS_DIR'); + if (iosDocs.isNotEmpty) { + final p = '$iosDocs/$filename'; + if (File(p).existsSync()) return p; + } + return null; + } + return null; +} + +Future _install() async { + final localPath = _localPath('gemma-4-E2B-it.litertlm'); + if (localPath != null && File(localPath).existsSync()) { + await FlutterGemma.installModel( + modelType: ModelType.gemma4, + fileType: ModelFileType.litertlm, + ).fromFile(localPath).install(); + } else { + await FlutterGemma.installModel( + modelType: ModelType.gemma4, + fileType: ModelFileType.litertlm, + ).fromNetwork(_gemma4Url, token: _token).install(); + } +} + +// Group-scoped model instance — mirrors litertlm_ffi_test.dart's +// _ensureModel/_closeSharedModel: creating one InferenceModel per group and +// reusing it across tests avoids repeated GPU engine_create overhead. +InferenceModel? _sharedModel; + +Future _ensureModel(int maxTokens) async { + if (_sharedModel != null) return _sharedModel!; + _sharedModel = await FlutterGemma.getActiveModel( + maxTokens: maxTokens, + preferredBackend: PreferredBackend.gpu, + supportImage: true, + maxNumImages: 1, + supportAudio: true, + ); + return _sharedModel!; +} + +Future _closeSharedModel() async { + if (_sharedModel != null) { + await _sharedModel!.close(); + _sharedModel = null; + } +} void main() { IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - testWidgets('sendMessage returns a model ChatMessage with text', ( - tester, - ) async { - final model = await FlutterGemma.getActiveModel(maxTokens: 1024); + setUpAll(() async { + await registerTestEngines(); + await _install(); + + for (final path in [ + '$_androidDir/test_image.jpg', + '$_macosDir/test_image.jpg', + if (Platform.isLinux) '$_linuxDir/test_image.jpg', + if (Platform.isWindows) '$_windowsDir\\test_image.jpg', + '${Platform.environment['HOME']}/Downloads/test_image.jpg', + ]) { + if (File(path).existsSync()) { + _testImage = File(path).readAsBytesSync(); + break; + } + } + if (_testImage.isEmpty) { + try { + final data = await rootBundle.load('assets/test/test_image.jpg'); + _testImage = data.buffer.asUint8List(); + } catch (_) { + /* asset not bundled — leave empty */ + } + } + + for (final path in [ + '$_androidDir/test_audio.wav', + '$_macosDir/test_audio.wav', + if (Platform.isLinux) '$_linuxDir/test_audio.wav', + if (Platform.isWindows) '$_windowsDir\\test_audio.wav', + ]) { + if (File(path).existsSync()) { + _testAudio = File(path).readAsBytesSync(); + break; + } + } + if (_testAudio.isEmpty) { + try { + final data = await rootBundle.load('assets/test/test_audio.wav'); + _testAudio = data.buffer.asUint8List(); + } catch (_) { + /* asset not bundled — leave empty */ + } + } + print('Platform: ${Platform.operatingSystem}'); + print('Assets: image=${_testImage.length}B, audio=${_testAudio.length}B'); + }); + + tearDownAll(_closeSharedModel); + + testWidgets('sendMessage — text', (tester) async { + final model = await _ensureModel(1024); final chat = await model.createChat(); final reply = await chat.sendMessage( ChatMessage.user('Say hello in one word.'), @@ -20,13 +165,10 @@ void main() { isNotEmpty, ); await chat.session.close(); - await model.close(); }); - testWidgets('sendMessageStream emits partial model ChatMessages', ( - tester, - ) async { - final model = await FlutterGemma.getActiveModel(maxTokens: 1024); + testWidgets('sendMessageStream — text', (tester) async { + final model = await _ensureModel(1024); final chat = await model.createChat(); final chunks = []; await for (final c in chat.sendMessageStream( @@ -37,6 +179,195 @@ void main() { expect(chunks, isNotEmpty); expect(chunks.every((c) => c.role == ChatMessageRole.model), isTrue); await chat.session.close(); - await model.close(); + }); + + testWidgets('sendMessage — vision', (tester) async { + if (_testImage.isEmpty) { + print('[genai vision] SKIP: no image'); + return; + } + final model = await _ensureModel(4096); + final chat = await model.createChat(supportImage: true); + final reply = await chat.sendMessage( + ChatMessage.user( + '', + parts: [ + TextPart('Describe this image briefly'), + DataPart(_testImage, mimeType: 'image/jpeg'), + ], + ), + ); + expect(reply.role, ChatMessageRole.model); + expect( + reply.parts.whereType().map((p) => p.text).join(), + isNotEmpty, + ); + await chat.session.close(); + }); + + testWidgets('sendMessage — audio', (tester) async { + if (_testAudio.isEmpty) { + print('[genai audio] SKIP: no audio'); + return; + } + final model = await _ensureModel(4096); + final chat = await model.createChat(supportAudio: true); + final reply = await chat.sendMessage( + ChatMessage.user( + '', + parts: [ + TextPart('What did you hear?'), + DataPart(_testAudio, mimeType: 'audio/wav'), + ], + ), + ); + expect(reply.role, ChatMessageRole.model); + expect( + reply.parts.whereType().map((p) => p.text).join(), + isNotEmpty, + ); + await chat.session.close(); + }); + + testWidgets('sendMessage — thinking', (tester) async { + final model = await _ensureModel(4096); + final chat = await model.createChat(isThinking: true); + final reply = await chat.sendMessage( + ChatMessage.user('Why is the sky blue?'), + ); + expect(reply.role, ChatMessageRole.model); + expect( + reply.parts.whereType().map((p) => p.text).join(), + isNotEmpty, + ); + final thinkingParts = reply.parts.whereType().toList(); + if (thinkingParts.isNotEmpty) { + // Model-dependent: only assert non-empty content when present, don't + // hard-require a ThinkingPart to show up. + expect(thinkingParts.map((p) => p.text).join(), isNotEmpty); + } + await chat.session.close(); + }); + + testWidgets('sendMessage — tool call round-trip', (tester) async { + final model = await _ensureModel(4096); + final chat = await model.createChat( + supportsFunctionCalls: true, + tools: const [_getWeatherTool], + modelType: ModelType.gemma4, + ); + + final reply = await chat.sendMessage( + ChatMessage.user('What is the weather like in Berlin?'), + ); + expect(reply.role, ChatMessageRole.model); + + final toolCalls = reply.parts + .whereType() + .where((p) => p.kind == ToolPartKind.call) + .toList(); + final text = reply.parts.whereType().map((p) => p.text).join(); + + // Model may either issue the tool call, or answer directly in text — + // both are valid; assert non-empty either way. + expect( + toolCalls.isNotEmpty || text.isNotEmpty, + isTrue, + reason: 'Reply must contain either a tool call or text', + ); + + if (toolCalls.isNotEmpty) { + final call = toolCalls.first; + expect(call.toolName, isNotEmpty); + + final followUp = await chat.sendMessage( + ChatMessage( + role: ChatMessageRole.user, + parts: [ + ToolPart.result( + callId: call.callId, + toolName: call.toolName, + result: const {'temperature_c': 18, 'condition': 'partly cloudy'}, + ), + ], + ), + ); + expect(followUp.role, ChatMessageRole.model); + final followUpText = followUp.parts + .whereType() + .map((p) => p.text) + .join(); + final followUpCalls = followUp.parts + .whereType() + .where((p) => p.kind == ToolPartKind.call) + .toList(); + expect( + followUpText.isNotEmpty || followUpCalls.isNotEmpty, + isTrue, + reason: 'Follow-up reply must contain either text or another call', + ); + } else { + print('[genai tool call] model answered directly: "$text"'); + } + + await chat.session.close(); + }); + + testWidgets('generateContent — single user message', (tester) async { + final model = await _ensureModel(1024); + final chat = await model.createChat(); + final reply = await chat.generateContent([ChatMessage.user('Say hello.')]); + expect(reply.role, ChatMessageRole.model); + expect( + reply.parts.whereType().map((p) => p.text).join(), + isNotEmpty, + ); + await chat.session.close(); + }); + + testWidgets('generateContentStream — single user message', (tester) async { + final model = await _ensureModel(1024); + final chat = await model.createChat(); + final chunks = []; + await for (final c in chat.generateContentStream([ + ChatMessage.user('Say hello.'), + ])) { + chunks.add(c); + } + expect(chunks, isNotEmpty); + expect(chunks.every((c) => c.role == ChatMessageRole.model), isTrue); + await chat.session.close(); + }); + + testWidgets('generateContent — multi-turn list', (tester) async { + final model = await _ensureModel(1024); + final chat = await model.createChat(); + final reply = await chat.generateContent([ + ChatMessage.user('My name is Sasha.'), + ChatMessage.model('Nice to meet you, Sasha.'), + ChatMessage.user('What is my name?'), + ]); + expect(reply.role, ChatMessageRole.model); + final text = reply.parts.whereType().map((p) => p.text).join(); + expect(text, isNotEmpty); + print('[genai generateContent multi-turn] $text'); + }); + + testWidgets('sendMessage — multi-turn history retained', (tester) async { + final model = await _ensureModel(1024); + final chat = await model.createChat(); + + final r1 = await chat.sendMessage(ChatMessage.user('My name is Sasha.')); + expect( + r1.parts.whereType().map((p) => p.text).join(), + isNotEmpty, + ); + + final r2 = await chat.sendMessage(ChatMessage.user('What is my name?')); + final text2 = r2.parts.whereType().map((p) => p.text).join(); + expect(text2, isNotEmpty); + print('[genai multi-turn history] $text2'); + + await chat.session.close(); }); } From f11f1670822aada1b9fc3ff4b4d9db0d95bf97c0 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Thu, 9 Jul 2026 16:42:33 +0200 Subject: [PATCH 13/20] fix(genai): close cancel-during-stage race + close leaked test session (#181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _lockedStream: a subscriber cancelling during the onListen await (before sub is assigned) triggered onCancel while onListen was suspended, releasing the mutex early; onListen then resumed and started a second concurrent turn with an un-cancellable subscription. Guard with a cancelled flag checked after stage() — bail before attaching the subscription. Verified with an isolated StreamController race repro. - integration test: generateContent multi-turn now closes chat.session (was the one of 10 tests that leaked). --- .../integration_test/genai_primitives_test.dart | 1 + .../lib/core/genai/genai_chat_extension.dart | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart b/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart index fb4d95f9..c348c144 100644 --- a/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart +++ b/packages/flutter_gemma/example/integration_test/genai_primitives_test.dart @@ -351,6 +351,7 @@ void main() { final text = reply.parts.whereType().map((p) => p.text).join(); expect(text, isNotEmpty); print('[genai generateContent multi-turn] $text'); + await chat.session.close(); }); testWidgets('sendMessage — multi-turn history retained', (tester) async { diff --git a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart index b62b1417..5b6478f2 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart @@ -115,6 +115,14 @@ extension GenAiChat on InferenceChat { late StreamController controller; StreamSubscription? sub; var released = false; + // Set by onCancel. onListen's async body awaits `stage()` BEFORE `sub` is + // assigned, so a subscriber that cancels in that window triggers onCancel + // while onListen is still suspended (sub == null → onCancel's sub?.cancel() + // is a no-op). Without this flag, onListen would resume and start generation + // on a chat whose lock onCancel already released — running a second turn + // concurrently and leaking an un-cancellable subscription. onListen checks + // this flag after `stage()` and bails before attaching the subscription. + var cancelled = false; void release() { if (!released) { released = true; @@ -127,6 +135,13 @@ extension GenAiChat on InferenceChat { await genaiLock.acquire(); try { await stage(); + if (cancelled) { + // Cancelled during stage(): onCancel already released the lock and + // called stopGeneration. Do NOT attach a subscription (it would be + // un-cancellable and would drive generation on an unlocked chat). + await controller.close(); + return; + } sub = generateChatResponseAsync().listen( (r) => controller.add(chatMessageFromChunk(r)), onError: (Object e, StackTrace s) async { @@ -151,6 +166,7 @@ extension GenAiChat on InferenceChat { } }, onCancel: () async { + cancelled = true; await sub?.cancel(); try { await stopGeneration(); From c65111162547d3b3209e8984c7a19378dd013ef7 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 12 Jul 2026 16:20:39 +0200 Subject: [PATCH 14/20] =?UTF-8?q?fix(genai):=20address=20#363=20review=20?= =?UTF-8?q?=E2=80=94=20accept=202xx=20links,=20strip=20thoughts=20on=20re-?= =?UTF-8?q?input,=20simplify=20converters=20(#181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lib/core/genai/genai_chat_extension.dart | 2 +- .../lib/core/genai/genai_input_converter.dart | 46 +++++++++++-------- .../lib/core/genai/link_reader.dart | 2 +- .../lib/core/genai/link_reader_io.dart | 5 +- packages/flutter_gemma/lib/genai.dart | 4 -- .../genai/genai_input_converter_test.dart | 21 +++++++-- .../test/genai/link_reader_test.dart | 25 ++++++++++ 7 files changed, 71 insertions(+), 34 deletions(-) diff --git a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart index 5b6478f2..aed57a90 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart @@ -112,7 +112,7 @@ extension GenAiChat on InferenceChat { /// Wrap the generate stream, mapping each event to a ChatMessage and holding /// the chat mutex until a terminal event (done/error/cancel). Stream _lockedStream(Future Function() stage) { - late StreamController controller; + late final StreamController controller; StreamSubscription? sub; var released = false; // Set by onCancel. onListen's async body awaits `stage()` BEFORE `sub` is diff --git a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart index 02fcce87..b49fcb43 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart @@ -10,6 +10,9 @@ import 'link_reader.dart'; /// Converts a genai_primitives [ChatMessage] into flutter_gemma [Message]s. /// /// One ChatMessage may yield >1 Message (tool results become sibling messages). +/// The text/media parts collapse into a single [Message] emitted FIRST, then +/// one sibling [Message] per tool result in part order. A [ThinkingPart] is +/// stripped (thoughts aren't fed back as history). /// Async because LinkPart resolution may need I/O. Throws — never silently drops. Future> messagesFromChatMessage( ChatMessage message, { @@ -26,20 +29,17 @@ Future> messagesFromChatMessage( } final isUser = message.role == ChatMessageRole.user; - final textBuffer = StringBuffer(); final images = []; Uint8List? audio; final messages = []; for (final part in parts) { switch (part) { - case TextPart(:final text): - textBuffer.write(text); case DataPart(:final bytes, :final mimeType): - _routeMedia(bytes, mimeType, images, () => audio, (v) => audio = v); + audio = _routeMedia(bytes, mimeType, images, audio); case LinkPart(:final url, :final mimeType): final bytes = await readLinkBytes(url, httpClient: httpClient); - _routeMedia(bytes, mimeType, images, () => audio, (v) => audio = v); + audio = _routeMedia(bytes, mimeType, images, audio); case ToolPart(kind: ToolPartKind.result, :final toolName, :final result): final resp = result is Map ? result @@ -56,19 +56,22 @@ Future> messagesFromChatMessage( text: jsonEncode({'name': toolName, 'parameters': arguments ?? {}}), ), ); + case TextPart(): case ThinkingPart(): - throw UnsupportedError( - 'ThinkingPart is model output (streamed), not input.', - ); + // TextPart text is read from `message.text` after the loop. A ThinkingPart + // is stripped from history: Gemma re-feeds the answer, not the reasoning + // (see the thinking docs' thought-stripping), and a model turn returned by + // the output converter carries its ThinkingPart, so dropping it here lets + // that turn round-trip back in as history. Both are no-ops in this loop. } } - final hasText = textBuffer.isNotEmpty; - if (hasText || images.isNotEmpty || audio != null) { + final text = message.text; + if (text.isNotEmpty || images.isNotEmpty || audio != null) { messages.insert( 0, Message( - text: textBuffer.toString(), + text: text, isUser: isUser, images: images, imageBytes: images.isNotEmpty ? images.first : null, @@ -79,6 +82,8 @@ Future> messagesFromChatMessage( return messages; } +/// Converts a list of [ChatMessage]s to flutter_gemma [Message]s in order, +/// concatenating each message's expansion (see [messagesFromChatMessage]). Future> messagesFromChatMessages( List messages, { http.Client? httpClient, @@ -90,24 +95,27 @@ Future> messagesFromChatMessages( return out; } -void _routeMedia( +/// Adds an image part to [images], or returns audio bytes as the message audio. +/// Returns the audio value the caller should keep: [current] for an image, +/// the new bytes for an audio part. Throws on a second audio or a non-media mime. +Uint8List? _routeMedia( Uint8List bytes, String? mimeType, List images, - Uint8List? Function() getAudio, - void Function(Uint8List) setAudio, + Uint8List? current, ) { final mime = mimeType ?? ''; if (mime.startsWith('image/')) { images.add(bytes); - } else if (mime.startsWith('audio/')) { - if (getAudio() != null) { + return current; + } + if (mime.startsWith('audio/')) { + if (current != null) { throw UnsupportedError('Only one audio part per message is supported.'); } - setAudio(bytes); - } else { - throw UnsupportedError('Unsupported DataPart mime "$mime".'); + return bytes; } + throw UnsupportedError('Unsupported DataPart mime "$mime".'); } /// Throws if any message needs a capability the chat wasn't built with, diff --git a/packages/flutter_gemma/lib/core/genai/link_reader.dart b/packages/flutter_gemma/lib/core/genai/link_reader.dart index 5c263396..15c563fc 100644 --- a/packages/flutter_gemma/lib/core/genai/link_reader.dart +++ b/packages/flutter_gemma/lib/core/genai/link_reader.dart @@ -20,7 +20,7 @@ Future readLinkBytes(Uri uri, {http.Client? httpClient}) async { final client = httpClient ?? http.Client(); try { final resp = await client.get(uri); - if (resp.statusCode != 200) { + if (resp.statusCode < 200 || resp.statusCode >= 300) { throw http.ClientException('HTTP ${resp.statusCode} for $uri', uri); } return resp.bodyBytes; diff --git a/packages/flutter_gemma/lib/core/genai/link_reader_io.dart b/packages/flutter_gemma/lib/core/genai/link_reader_io.dart index b422e9e4..9337befd 100644 --- a/packages/flutter_gemma/lib/core/genai/link_reader_io.dart +++ b/packages/flutter_gemma/lib/core/genai/link_reader_io.dart @@ -1,7 +1,4 @@ import 'dart:io'; import 'dart:typed_data'; -Future readFileUri(Uri uri) async { - final path = uri.scheme == 'file' ? uri.toFilePath() : uri.path; - return File(path).readAsBytes(); -} +Future readFileUri(Uri uri) => File.fromUri(uri).readAsBytes(); diff --git a/packages/flutter_gemma/lib/genai.dart b/packages/flutter_gemma/lib/genai.dart index 2adbc4cf..a08e1e95 100644 --- a/packages/flutter_gemma/lib/genai.dart +++ b/packages/flutter_gemma/lib/genai.dart @@ -7,7 +7,3 @@ library; export 'package:genai_primitives/genai_primitives.dart'; export 'core/genai/genai_chat_extension.dart' show GenAiChat; -export 'core/genai/genai_input_converter.dart' - show messagesFromChatMessage, messagesFromChatMessages; -export 'core/genai/genai_output_converter.dart' - show chatMessageFromChunk, chatMessageFromParts; diff --git a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart index 8aa9f220..8df7150d 100644 --- a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart +++ b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart @@ -43,15 +43,26 @@ void main() { ); }); - test('ThinkingPart throws', () async { + test('a ThinkingPart is stripped, sibling parts are kept', () async { + // Gemma strips prior-turn thoughts from history; feeding a returned model + // turn (which carries its ThinkingPart) back must not throw — drop the + // thought, keep the answer. See genai_output_converter emitting ThinkingPart. final msg = ChatMessage( role: ChatMessageRole.model, - parts: [const ThinkingPart('t')], + parts: [const ThinkingPart('secret reasoning'), TextPart('final answer')], ); - expect( - () => messagesFromChatMessage(msg), - throwsA(isA()), + final out = await messagesFromChatMessage(msg); + expect(out, hasLength(1)); + expect(out.single.text, 'final answer'); + expect(out.single.isUser, isFalse); + }); + + test('a pure-thought model turn yields no staged Message', () async { + final msg = ChatMessage( + role: ChatMessageRole.model, + parts: [const ThinkingPart('t')], ); + expect(await messagesFromChatMessage(msg), isEmpty); }); test('tool result → Message.toolResponse', () async { diff --git a/packages/flutter_gemma/test/genai/link_reader_test.dart b/packages/flutter_gemma/test/genai/link_reader_test.dart index 053b950e..781266f2 100644 --- a/packages/flutter_gemma/test/genai/link_reader_test.dart +++ b/packages/flutter_gemma/test/genai/link_reader_test.dart @@ -2,6 +2,8 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:flutter_gemma/core/genai/link_reader.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:http/testing.dart'; void main() { test('resolves a data: URI to bytes', () async { @@ -18,4 +20,27 @@ void main() { throwsA(isA()), ); }); + + test('accepts a 2xx HTTP status other than 200', () async { + final bytes = Uint8List.fromList([9, 8, 7]); + final client = MockClient((_) async => http.Response.bytes(bytes, 206)); + expect( + await readLinkBytes( + Uri.parse('https://example.com/media.bin'), + httpClient: client, + ), + bytes, + ); + }); + + test('throws on a non-2xx HTTP status', () async { + final client = MockClient((_) async => http.Response('nope', 404)); + expect( + () => readLinkBytes( + Uri.parse('https://example.com/missing.bin'), + httpClient: client, + ), + throwsA(isA()), + ); + }); } From c68dfb8e171f1ed62724e752101e6c7912cb96a1 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 12 Jul 2026 16:44:14 +0200 Subject: [PATCH 15/20] fix(genai): throw on a user-role ThinkingPart instead of silently staging nothing (#181) --- .../lib/core/genai/genai_input_converter.dart | 25 ++++++++++++------- .../genai/genai_input_converter_test.dart | 15 ++++++++++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart index b49fcb43..d1ef9166 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart @@ -11,9 +11,11 @@ import 'link_reader.dart'; /// /// One ChatMessage may yield >1 Message (tool results become sibling messages). /// The text/media parts collapse into a single [Message] emitted FIRST, then -/// one sibling [Message] per tool result in part order. A [ThinkingPart] is -/// stripped (thoughts aren't fed back as history). -/// Async because LinkPart resolution may need I/O. Throws — never silently drops. +/// one sibling [Message] per tool result in part order. A [ThinkingPart] on a +/// model turn is stripped by design (thoughts aren't fed back as history), so a +/// thought-only model turn yields no Message; a [ThinkingPart] on a user turn is +/// misuse and throws. Async because LinkPart resolution may need I/O. Other +/// unsupported content throws rather than being silently dropped. Future> messagesFromChatMessage( ChatMessage message, { http.Client? httpClient, @@ -56,13 +58,18 @@ Future> messagesFromChatMessage( text: jsonEncode({'name': toolName, 'parameters': arguments ?? {}}), ), ); - case TextPart(): case ThinkingPart(): - // TextPart text is read from `message.text` after the loop. A ThinkingPart - // is stripped from history: Gemma re-feeds the answer, not the reasoning - // (see the thinking docs' thought-stripping), and a model turn returned by - // the output converter carries its ThinkingPart, so dropping it here lets - // that turn round-trip back in as history. Both are no-ops in this loop. + // A model turn's thought is stripped from history: Gemma re-feeds the + // answer, not the reasoning (see the thinking docs' thought-stripping), + // so a model turn returned by the output converter round-trips back in. + // A user-role thought is misuse — thoughts are model output, fail loud. + if (isUser) { + throw UnsupportedError( + 'A ThinkingPart is model output, not user input.', + ); + } + case TextPart(): + // No-op: TextPart text is read from `message.text` after the loop. } } diff --git a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart index 8df7150d..01d8d7c6 100644 --- a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart +++ b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart @@ -43,7 +43,7 @@ void main() { ); }); - test('a ThinkingPart is stripped, sibling parts are kept', () async { + test('a model-turn ThinkingPart is stripped, sibling parts are kept', () async { // Gemma strips prior-turn thoughts from history; feeding a returned model // turn (which carries its ThinkingPart) back must not throw — drop the // thought, keep the answer. See genai_output_converter emitting ThinkingPart. @@ -65,6 +65,19 @@ void main() { expect(await messagesFromChatMessage(msg), isEmpty); }); + test('a user-role ThinkingPart throws (thoughts are model output)', () async { + // Stripping is only right for a returned model turn's history. A user + // constructing a thought is misuse — fail loud, don't stage nothing. + final msg = ChatMessage( + role: ChatMessageRole.user, + parts: [const ThinkingPart('t'), TextPart('hi')], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); + test('tool result → Message.toolResponse', () async { final msg = ChatMessage( role: ChatMessageRole.user, From 27b30e7de20c19492c9380ef5149c91ed4fa42af Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 12 Jul 2026 17:29:20 +0200 Subject: [PATCH 16/20] docs(website): add genai_primitives adoption guide + sidebar link (#181) --- website/content/docs/genai.md | 173 ++++++++++++++++++++++++++++++++++ website/lib/main.server.dart | 1 + 2 files changed, 174 insertions(+) create mode 100644 website/content/docs/genai.md diff --git a/website/content/docs/genai.md b/website/content/docs/genai.md new file mode 100644 index 00000000..a63356a7 --- /dev/null +++ b/website/content/docs/genai.md @@ -0,0 +1,173 @@ +--- +title: genai_primitives +description: Use the Flutter team's genai_primitives ChatMessage types with flutter_gemma — sendMessage, generateContent, streaming, tools, and thinking. +image: https://fluttergemma.dev/images/og-image.png +--- + +[genai_primitives](https://pub.dev/packages/genai_primitives) is the Flutter +team's set of standard chat types — `ChatMessage`, `TextPart`, `DataPart`, +`ToolPart`, and friends. flutter_gemma speaks them directly, so you can drive an +on-device chat with the same message types you'd use anywhere else in the +Flutter AI ecosystem, and move a conversation between providers without +rewriting it. + +The surface is a small extension on `InferenceChat`. Import the side barrel: + +```dart +import 'package:flutter_gemma/genai.dart'; +``` + +That one import re-exports the genai_primitives types too, so you don't need to +add `genai_primitives` to your own `pubspec.yaml` — `ChatMessage`, `TextPart`, +and the rest come with it. + +## The four entry points + +`genai.dart` adds these to any `InferenceChat`: + +- `sendMessage(ChatMessage)` — send one turn, get the whole model turn back as a + `role: model` `ChatMessage`. +- `sendMessageStream(ChatMessage)` — the same, streamed as partial + `ChatMessage`s, one per delta. +- `generateContent(List)` — stage a whole list of turns into the + chat, then generate once. +- `generateContentStream(List)` — the streamed variant. + +Every method returns (or yields) `ChatMessage`s with `role: model`. Read the +answer out of its parts. + +## Text + +```dart +final chat = await model.createChat(); + +final reply = await chat.sendMessage( + ChatMessage.user('Say hello in one word.'), +); + +final text = reply.parts.whereType().map((p) => p.text).join(); +print(text); +``` + +Streaming is the same call, awaited over: + +```dart +await for (final chunk in chat.sendMessageStream( + ChatMessage.user('Count to three.'), +)) { + final token = chunk.parts.whereType().map((p) => p.text).join(); + print(token); +} +``` + +## Images and audio + +Attach media as parts. Inline bytes go in a `DataPart` with a matching MIME +type; a `LinkPart` is fetched for you (`data:`, `file:`, and `http(s):` URLs are +supported). Create the chat with the capability the model needs, or the send +throws rather than silently dropping the media. + +```dart +final chat = await model.createChat(supportImage: true); + +final reply = await chat.sendMessage( + ChatMessage.user( + '', + parts: [ + TextPart('Describe this image briefly'), + DataPart(imageBytes, mimeType: 'image/jpeg'), + ], + ), +); +``` + +Audio is identical with `supportAudio: true` and an `audio/*` MIME type. One +audio part per message. + +## Thinking + +Create the chat with `isThinking: true`. The model turn comes back with its +reasoning in a `ThinkingPart` alongside the answer's `TextPart`: + +```dart +final chat = await model.createChat(isThinking: true); + +final reply = await chat.sendMessage( + ChatMessage.user('Why is the sky blue?'), +); + +final thinking = reply.parts.whereType().map((p) => p.text).join(); +final answer = reply.parts.whereType().map((p) => p.text).join(); +``` + +When you feed a prior model turn back in as history, its `ThinkingPart` is +stripped automatically — the model is re-fed the answer, not the reasoning, so +you can pass a returned turn straight back into `generateContent`. + +## Tool calls + +Create the chat with your tools, then read tool calls out of the reply's +`ToolPart`s and send the result back as a `ToolPart.result`: + +```dart +final chat = await model.createChat( + supportsFunctionCalls: true, + tools: const [getWeatherTool], + modelType: ModelType.gemma4, +); + +final reply = await chat.sendMessage( + ChatMessage.user('What is the weather like in Berlin?'), +); + +final calls = reply.parts + .whereType() + .where((p) => p.kind == ToolPartKind.call) + .toList(); + +if (calls.isNotEmpty) { + final call = calls.first; + // ... run the tool, then return its result as the next turn: + final followUp = await chat.sendMessage( + ChatMessage( + role: ChatMessageRole.user, + parts: [ + ToolPart.result( + callId: call.callId, + toolName: call.toolName, + result: const {'temperature_c': 18, 'condition': 'partly cloudy'}, + ), + ], + ), + ); +} +``` + +## Staging a batch + +`generateContent` stages a list of turns into the chat before generating — use +it to seed prior context in one call: + +```dart +final reply = await chat.generateContent([ + ChatMessage.user('Remember the number 7.'), + ChatMessage.model('Got it.'), + ChatMessage.user('What number did I say?'), +]); +``` + +## Rules and edge cases + +- `sendMessage` takes **user input**. Passing a `role: model` message throws — + a model turn is output; stage prior model turns with `generateContent`. +- A `ThinkingPart` on a **user** message throws (thoughts are model output); on + a **model** turn it is stripped as history. +- A `system`-role message throws — set the system prompt via + `createChat(systemInstruction: ...)` instead. +- Missing capability throws: sending an image to a chat created without + `supportImage`, audio without `supportAudio`, or a tool part without + `supportsFunctionCalls` fails loudly rather than dropping the content. + +This surface lives behind its own `genai.dart` barrel so genai_primitives' +pre-1.0 churn stays contained — the rest of flutter_gemma's API is unaffected by +it. diff --git a/website/lib/main.server.dart b/website/lib/main.server.dart index 472e02ee..25f4bac3 100644 --- a/website/lib/main.server.dart +++ b/website/lib/main.server.dart @@ -118,6 +118,7 @@ void main() { SidebarGroup( title: 'Integrations', links: [ + SidebarLink(text: 'genai_primitives', href: '/docs/genai'), SidebarLink(text: 'Genkit', href: '/docs/genkit'), ], ), From 67bec572f0f7fa9b368cb7cb70d8076cddb02218 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 12 Jul 2026 18:00:22 +0200 Subject: [PATCH 17/20] =?UTF-8?q?chore(release):=20flutter=5Fgemma=201.3.0?= =?UTF-8?q?=20=E2=80=94=20genai=5Fprimitives=20support=20(#181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 2 +- packages/flutter_gemma/CHANGELOG.md | 4 ++++ packages/flutter_gemma/DESKTOP_SUPPORT.md | 2 +- packages/flutter_gemma/ios/flutter_gemma.podspec | 2 +- packages/flutter_gemma/pubspec.yaml | 2 +- website/content/docs/genkit.md | 2 +- website/content/docs/migration.md | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 23dfa9cf..44700061 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,7 +137,7 @@ Core has NO pigeon (dropped at the 1.0 cut; its value types are hand-written in - **MediaPipe Web**: v0.10.27, Android/iOS: v0.10.33 - **LiteRT-LM**: native libs from `native-v0.13.1-a` GitHub Release. Android tarball bundles the Qualcomm QNN dispatch stack and Windows tarball bundles Intel NPU dispatch (`LiteRtDispatch.dll` + OpenVino runtime + TBB) for `PreferredBackend.npu` (Qualcomm Snapdragon / Intel LunarLake/PantherLake). MTP (speculative decoding) support for Gemma 4 (#318 MTP crash fixed). (native-v0.13.1-a restores the NPU dispatch libs accidentally omitted from native-v0.13.1 — #155.) - **large_file_handler**: `^0.5.0` (core dep; 0.5.0 declares all 6 platforms — needed for pana platform support + the dart2wasm-clean web graph) -- **Current Version**: core `flutter_gemma` `1.2.3`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.0.3`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.1`; `flutter_gemma_agent` `0.1.0` (new) +- **Current Version**: core `flutter_gemma` `1.3.0`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.0.3`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.1`; `flutter_gemma_agent` `0.1.0` (new) - **0.15.2**: embedding unified on LiteRT C API via Dart FFI on all native platforms (Android + iOS + Desktop). Drops `localagents-rag` JVM dep on Android and the separate TFLite C 0.12.7 tarball on Desktop; `TensorFlowLiteC` pod no longer needed on iOS. Single source of truth for `TaskType.prefix` in Dart, fixes cross-platform embedding drift (#264). ## Platform-Specific Setup diff --git a/packages/flutter_gemma/CHANGELOG.md b/packages/flutter_gemma/CHANGELOG.md index 0f768490..4220a175 100644 --- a/packages/flutter_gemma/CHANGELOG.md +++ b/packages/flutter_gemma/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.3.0 +- Add genai_primitives support via `package:flutter_gemma/genai.dart` — use `ChatMessage` with `InferenceChat` (#181). +- New `sendMessage`/`generateContent` (+ streams) covering text, vision, audio, thinking, and tool calls (#181). + ## 1.2.3 - Cancel native `.litertlm` decoding before session teardown to avoid an ANR (#364, #373). - Re-run macOS companion staging on incremental builds so LiteRtLm stays patched (#368). diff --git a/packages/flutter_gemma/DESKTOP_SUPPORT.md b/packages/flutter_gemma/DESKTOP_SUPPORT.md index e9ea70f5..aa0ec209 100644 --- a/packages/flutter_gemma/DESKTOP_SUPPORT.md +++ b/packages/flutter_gemma/DESKTOP_SUPPORT.md @@ -96,7 +96,7 @@ No Java/JVM/JRE required. ```yaml # pubspec.yaml dependencies: - flutter_gemma: ^1.2.3 # core + flutter_gemma: ^1.3.0 # core flutter_gemma_litertlm: ^1.0.3 # .litertlm engine — required on desktop ``` diff --git a/packages/flutter_gemma/ios/flutter_gemma.podspec b/packages/flutter_gemma/ios/flutter_gemma.podspec index b19481b2..0de44e1e 100644 --- a/packages/flutter_gemma/ios/flutter_gemma.podspec +++ b/packages/flutter_gemma/ios/flutter_gemma.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'flutter_gemma' - s.version = '1.2.3' + s.version = '1.3.0' s.summary = 'Flutter plugin for running Gemma and other LLMs locally on iOS.' s.description = <<-DESC Core runtime for running Gemma 4, Gemma3n, Gemma 3, FastVLM, Qwen3, diff --git a/packages/flutter_gemma/pubspec.yaml b/packages/flutter_gemma/pubspec.yaml index abd0d747..8a05cdbe 100644 --- a/packages/flutter_gemma/pubspec.yaml +++ b/packages/flutter_gemma/pubspec.yaml @@ -1,6 +1,6 @@ name: flutter_gemma description: "Run Gemma and other LLMs on-device in Flutter (Android, iOS, Web, Desktop). Multimodal vision/audio, function calling, thinking mode, GPU, embeddings, RAG." -version: 1.2.3 +version: 1.3.0 resolution: workspace homepage: https://fluttergemma.dev repository: https://github.com/DenisovAV/flutter_gemma diff --git a/website/content/docs/genkit.md b/website/content/docs/genkit.md index c75f6f40..7d819af4 100644 --- a/website/content/docs/genkit.md +++ b/website/content/docs/genkit.md @@ -21,7 +21,7 @@ with the on-device model exactly as it would with any cloud provider. ``` dependencies: genkit_flutter_gemma: ^0.4.3 - flutter_gemma: ^1.2.3 + flutter_gemma: ^1.3.0 # Add the inference engine(s) you need: flutter_gemma_litertlm: ^1.0.3 # .litertlm models (mobile + desktop) flutter_gemma_mediapipe: ^1.0.4 # .task / .bin models (mobile + web) diff --git a/website/content/docs/migration.md b/website/content/docs/migration.md index 8b476eb3..43fc6dd4 100644 --- a/website/content/docs/migration.md +++ b/website/content/docs/migration.md @@ -29,7 +29,7 @@ dependencies: ``` dependencies: - flutter_gemma: ^1.2.3 # core — always required + flutter_gemma: ^1.3.0 # core — always required flutter_gemma_litertlm: ^1.0.3 # add if you run .litertlm models flutter_gemma_mediapipe: ^1.0.4 # add if you run .task / .bin models flutter_gemma_embeddings: ^1.0.1 # add if you compute embeddings From d5f79cba310d63b971a5efde67b41118e18bad13 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 12 Jul 2026 20:21:22 +0200 Subject: [PATCH 18/20] fix(genai): _lockedStream must not release a mutex it doesn't own (#181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onCancel released genaiLock even while onListen was still queued on acquire() or inside stage(), freeing another turn's critical section (package:mutex is ownership-blind) — letting a second turn interleave into the shared session, or permanently wedging the chat. Lock is now owned by onListen: release() guards on lockHeld, and onCancel only tears down once generation is running. Also route a throw in the chunk-mapping into cleanup (was escaping to the Zone and leaking the lock), and log stopGeneration teardown failures. --- .../lib/core/genai/genai_chat_extension.dart | 83 +++++++--- .../test/genai/locked_stream_test.dart | 154 ++++++++++++++++++ 2 files changed, 215 insertions(+), 22 deletions(-) create mode 100644 packages/flutter_gemma/test/genai/locked_stream_test.dart diff --git a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart index aed57a90..a67b4a3b 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart @@ -5,6 +5,7 @@ import 'package:http/http.dart' as http; import '../chat.dart'; import '../model_response.dart'; +import '../utils/gemma_log.dart'; import 'genai_input_converter.dart'; import 'genai_output_converter.dart'; @@ -111,48 +112,83 @@ extension GenAiChat on InferenceChat { /// Wrap the generate stream, mapping each event to a ChatMessage and holding /// the chat mutex until a terminal event (done/error/cancel). + /// + /// The lock is owned by `onListen` (the flow that acquired it). `onCancel` + /// only *signals* cancellation and tears down when generation is actually + /// running; while we are still queued on `acquire()` or inside `stage()`, + /// `onListen` releases when it observes the flag. `release()` is guarded on + /// `lockHeld` so it can never free a lock this invocation doesn't hold — the + /// `package:mutex` mutex is ownership-blind, so an unguarded release from + /// `onCancel` could otherwise free another turn's critical section. Stream _lockedStream(Future Function() stage) { late final StreamController controller; StreamSubscription? sub; var released = false; - // Set by onCancel. onListen's async body awaits `stage()` BEFORE `sub` is - // assigned, so a subscriber that cancels in that window triggers onCancel - // while onListen is still suspended (sub == null → onCancel's sub?.cancel() - // is a no-op). Without this flag, onListen would resume and start generation - // on a chat whose lock onCancel already released — running a second turn - // concurrently and leaking an un-cancellable subscription. onListen checks - // this flag after `stage()` and bails before attaching the subscription. var cancelled = false; + var lockHeld = false; // true once THIS invocation's acquire() has resolved + var generating = false; // true once the generate subscription is attached + void release() { - if (!released) { + if (!released && lockHeld) { released = true; genaiLock.release(); } } + Future stopSafely() async { + try { + await stopGeneration(); + } catch (e, s) { + gemmaLog( + 'WARNING: genai stopGeneration during teardown failed: $e\n$s', + ); + } + } + controller = StreamController( onListen: () async { await genaiLock.acquire(); + lockHeld = true; + // Cancelled while queued on acquire(): we now own the lock — release it + // and bail without staging or generating. + if (cancelled) { + release(); + await controller.close(); + return; + } try { await stage(); + // Cancelled during stage(): we still hold the lock (onCancel does not + // release while onListen owns it) and stage() has finished mutating + // shared state, so releasing here can't let a second turn interleave. + // No generation started, so nothing to stop. if (cancelled) { - // Cancelled during stage(): onCancel already released the lock and - // called stopGeneration. Do NOT attach a subscription (it would be - // un-cancellable and would drive generation on an unlocked chat). + release(); await controller.close(); return; } + generating = true; sub = generateChatResponseAsync().listen( - (r) => controller.add(chatMessageFromChunk(r)), + (r) { + // A throw in the mapping would escape to the Zone and never + // release the lock — funnel it into the same cleanup. + try { + controller.add(chatMessageFromChunk(r)); + } catch (e, s) { + controller.addError(e, s); + sub?.cancel(); + stopSafely().whenComplete(() { + release(); + controller.close(); + }); + } + }, onError: (Object e, StackTrace s) async { controller.addError(e, s); await sub?.cancel(); - try { - await stopGeneration(); - } finally { - release(); - await controller.close(); - } + await stopSafely(); + release(); + await controller.close(); }, onDone: () { release(); @@ -167,10 +203,13 @@ extension GenAiChat on InferenceChat { }, onCancel: () async { cancelled = true; - await sub?.cancel(); - try { - await stopGeneration(); - } finally { + // Only tear down here once generation is running. While queued on + // acquire() or inside stage(), onListen owns the lock and releases it + // when it sees `cancelled`; releasing here would free a lock we don't + // hold yet, or free it mid-stage and let a second turn interleave. + if (generating) { + await sub?.cancel(); + await stopSafely(); release(); } }, diff --git a/packages/flutter_gemma/test/genai/locked_stream_test.dart b/packages/flutter_gemma/test/genai/locked_stream_test.dart new file mode 100644 index 00000000..1ec062e3 --- /dev/null +++ b/packages/flutter_gemma/test/genai/locked_stream_test.dart @@ -0,0 +1,154 @@ +import 'dart:async'; + +import 'package:flutter_gemma/flutter_gemma.dart'; +import 'package:flutter_gemma/genai.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// A no-engine [InferenceChat] whose staging (addQueryChunk) and generation +/// (generateChatResponseAsync) are driven by the test, so `_lockedStream`'s +/// mutex/cancel state machine can be exercised deterministically. +class _FakeChat extends InferenceChat { + _FakeChat() : super(sessionCreator: null, maxTokens: 1024); + + Duration stageDelay = Duration.zero; + final staged = []; + int stopGenerationCalls = 0; + StreamController? lastGen; + + @override + Future addQueryChunk( + Message message, [ + bool noTool = false, + bool prefix = false, + ]) async { + if (stageDelay > Duration.zero) await Future.delayed(stageDelay); + staged.add(message.text); + } + + @override + Stream generateChatResponseAsync() { + final c = StreamController(); + lastGen = c; + return c.stream; + } + + @override + Future stopGeneration() async { + stopGenerationCalls++; + } +} + +String _text(ChatMessage m) => + m.parts.whereType().map((p) => p.text).join(); + +void main() { + test( + 'cancel during stage keeps the lock held until stage() finishes', + () async { + // The C1 bug: onCancel releases the mutex mid-stage, while stage() is still + // mutating shared session state — so a second turn can interleave. + final chat = _FakeChat()..stageDelay = const Duration(milliseconds: 120); + final sub = chat.sendMessageStream(ChatMessage.user('hi')).listen((_) {}); + await Future.delayed(const Duration(milliseconds: 20)); + expect(chat.genaiLock.isLocked, isTrue, reason: 'lock held during stage'); + + await sub.cancel(); // cancel while stage() still has ~100ms to run + expect( + chat.genaiLock.isLocked, + isTrue, + reason: 'lock must NOT be released mid-stage', + ); + + await Future.delayed(const Duration(milliseconds: 160)); + expect( + chat.genaiLock.isLocked, + isFalse, + reason: 'lock released once stage() completes', + ); + }, + ); + + test( + 'a queued turn cancelled before acquiring never stages (no lock theft)', + () async { + // C1 variant: a stream cancelled while queued on acquire() must not + // release() the lock the running turn owns. If it did, this turn would be + // granted the stolen lock and proceed to stage() — an empty `staged` + // proves it stayed queued and never ran a critical section. + final chat = _FakeChat(); + await chat.genaiLock + .acquire(); // turn A holds the lock, does nothing else + + final sub = chat.sendMessageStream(ChatMessage.user('B')).listen((_) {}); + await Future.delayed(const Duration(milliseconds: 20)); // B queued + await sub.cancel(); + await Future.delayed(const Duration(milliseconds: 20)); + + expect( + chat.staged, + isEmpty, + reason: 'a queued+cancelled turn must not steal the lock and stage', + ); + expect(chat.genaiLock.isLocked, isTrue, reason: 'A still holds the lock'); + }, + ); + + test( + 'normal completion yields mapped chunks and releases the lock', + () async { + final chat = _FakeChat(); + final chunks = []; + final done = Completer(); + chat + .sendMessageStream(ChatMessage.user('hi')) + .listen(chunks.add, onDone: done.complete); + await Future.delayed(const Duration(milliseconds: 20)); + chat.lastGen!.add(const TextResponse('a')); + chat.lastGen!.add(const TextResponse('b')); + await chat.lastGen!.close(); + await done.future; + + expect(chunks.map(_text).toList(), ['a', 'b']); + expect(chat.genaiLock.isLocked, isFalse, reason: 'lock released on done'); + }, + ); + + test( + 'error mid-stream forwards it, stops generation, releases the lock', + () async { + final chat = _FakeChat(); + Object? err; + final done = Completer(); + chat + .sendMessageStream(ChatMessage.user('hi')) + .listen( + (_) {}, + onError: (Object e) => err = e, + onDone: done.complete, + ); + await Future.delayed(const Duration(milliseconds: 20)); + chat.lastGen!.addError(StateError('boom')); + await done.future; + + expect(err, isA()); + expect(chat.stopGenerationCalls, greaterThan(0)); + expect( + chat.genaiLock.isLocked, + isFalse, + reason: 'lock released on error', + ); + }, + ); + + test('cancel during generation stops it and releases the lock', () async { + final chat = _FakeChat(); + final sub = chat.sendMessageStream(ChatMessage.user('hi')).listen((_) {}); + await Future.delayed(const Duration(milliseconds: 20)); + chat.lastGen!.add(const TextResponse('a')); // generation is live + await Future.delayed(const Duration(milliseconds: 10)); + + await sub.cancel(); + expect(chat.stopGenerationCalls, greaterThan(0)); + expect(chat.genaiLock.isLocked, isFalse, reason: 'lock released on cancel'); + }); +} From a5a5675fc84517f8e9bf97e2168fb6b5ce354ee2 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Sun, 12 Jul 2026 20:27:58 +0200 Subject: [PATCH 19/20] fix(genai): address #363 review IMPORTANT cluster (#181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - keep preamble text alongside a tool call in the batch fold (was silently dropped, diverging from the streaming path) - link_reader: reject an empty 2xx body (blank media) and bound the fetch with a timeout so a hung server can't wedge the chat mutex - throw on a tool result carried by a model-role message (symmetric with the tool-call-on-user guard) - mark InferenceChat.genaiLock @internal — external acquire/release deadlocks the chat - document that callId mirrors toolName (flutter_gemma's tool protocol is name-keyed) --- packages/flutter_gemma/lib/core/chat.dart | 5 +++ .../lib/core/genai/genai_input_converter.dart | 5 +++ .../core/genai/genai_output_converter.dart | 22 ++++++++----- .../lib/core/genai/link_reader.dart | 18 +++++++++-- .../genai/genai_input_converter_test.dart | 15 +++++++++ .../genai/genai_output_converter_test.dart | 12 +++++-- .../test/genai/link_reader_test.dart | 32 +++++++++++++++++++ 7 files changed, 96 insertions(+), 13 deletions(-) diff --git a/packages/flutter_gemma/lib/core/chat.dart b/packages/flutter_gemma/lib/core/chat.dart index e495ef5a..d1f89610 100644 --- a/packages/flutter_gemma/lib/core/chat.dart +++ b/packages/flutter_gemma/lib/core/chat.dart @@ -8,6 +8,7 @@ import 'package:flutter_gemma/core/model_response.dart'; import 'package:flutter_gemma/core/parsing/sdk_response_parser.dart'; import 'package:flutter_gemma/core/tool.dart'; import 'package:flutter_gemma/flutter_gemma_interface.dart'; +import 'package:meta/meta.dart' show internal; import 'package:mutex/mutex.dart'; import 'model.dart'; @@ -34,6 +35,10 @@ class InferenceChat { /// Serializes genai_primitives sendMessage/generateContent calls so /// concurrent turns can't interleave staging into the shared session buffer. + /// + /// Internal to the `package:flutter_gemma/genai.dart` extension — not part of + /// the public API. External `acquire()`/`release()` would deadlock the chat. + @internal final Mutex genaiLock = Mutex(); final List _fullHistory = []; diff --git a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart index d1ef9166..906804ab 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart @@ -43,6 +43,11 @@ Future> messagesFromChatMessage( final bytes = await readLinkBytes(url, httpClient: httpClient); audio = _routeMedia(bytes, mimeType, images, audio); case ToolPart(kind: ToolPartKind.result, :final toolName, :final result): + if (!isUser) { + throw UnsupportedError( + 'A tool result is caller input, not model output.', + ); + } final resp = result is Map ? result : {'result': result}; diff --git a/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart index b7b156fe..bb8088ba 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_output_converter.dart @@ -23,7 +23,14 @@ ChatMessage chatMessageFromChunk(ModelResponse chunk) { } /// Coalesce a whole model turn into one [ChatMessage]. -/// Order: ThinkingPart → (ToolPart.call… OR TextPart). Tool calls suppress text. +/// +/// Parts follow generation order: ThinkingPart → TextPart → ToolPart.call…. A +/// preamble ("Let me check…") that accompanies a tool call is kept, not dropped +/// — matching what [chatMessageFromChunk] emits on the streaming path. +/// +/// `callId` mirrors `toolName`: flutter_gemma's tool protocol is name-keyed +/// ([FunctionCallResponse] carries no independent id), so parallel calls to the +/// same tool share a callId and callId is ignored when a result is fed back in. ChatMessage chatMessageFromParts({ String? text, List calls = const [], @@ -34,15 +41,14 @@ ChatMessage chatMessageFromParts({ if (thinking != null && thinking.isNotEmpty) { parts.add(ThinkingPart(thinking)); } - if (calls.isNotEmpty) { - for (final c in calls) { - parts.add( - ToolPart.call(callId: c.name, toolName: c.name, arguments: c.args), - ); - } - } else if (text != null && text.isNotEmpty) { + if (text != null && text.isNotEmpty) { parts.add(TextPart(text)); } + for (final c in calls) { + parts.add( + ToolPart.call(callId: c.name, toolName: c.name, arguments: c.args), + ); + } return ChatMessage( role: ChatMessageRole.model, parts: parts, diff --git a/packages/flutter_gemma/lib/core/genai/link_reader.dart b/packages/flutter_gemma/lib/core/genai/link_reader.dart index 15c563fc..51a7a120 100644 --- a/packages/flutter_gemma/lib/core/genai/link_reader.dart +++ b/packages/flutter_gemma/lib/core/genai/link_reader.dart @@ -7,7 +7,11 @@ import 'link_reader_stub.dart' if (dart.library.io) 'link_reader_io.dart'; /// - data: decoded in-memory (no I/O) /// - file:// dart:io on native; throws on web /// - http(s):: injected client, else default http.Client (wasm-safe BrowserClient on web) -Future readLinkBytes(Uri uri, {http.Client? httpClient}) async { +Future readLinkBytes( + Uri uri, { + http.Client? httpClient, + Duration timeout = const Duration(seconds: 30), +}) async { switch (uri.scheme) { case 'data': final data = uri.data; @@ -19,10 +23,20 @@ Future readLinkBytes(Uri uri, {http.Client? httpClient}) async { case 'https': final client = httpClient ?? http.Client(); try { - final resp = await client.get(uri); + // Bound the fetch: readLinkBytes is awaited while the chat mutex is + // held, so a hung server would otherwise wedge the whole genai surface. + final resp = await client.get(uri).timeout(timeout); if (resp.statusCode < 200 || resp.statusCode >= 300) { throw http.ClientException('HTTP ${resp.statusCode} for $uri', uri); } + // A 2xx with an empty body (204/205, or a truncated proxy response) + // would otherwise flow on as blank image/audio — a silent media drop. + if (resp.bodyBytes.isEmpty) { + throw http.ClientException( + 'Empty body (${resp.statusCode}) for $uri', + uri, + ); + } return resp.bodyBytes; } finally { if (httpClient == null) client.close(); diff --git a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart index 01d8d7c6..154a51fb 100644 --- a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart +++ b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart @@ -90,6 +90,21 @@ void main() { expect(out.single.toolName, 'calc'); }); + test('a tool result on a model-role message throws', () async { + // A tool result is caller/user input, not model output — mirror the + // tool-call-on-user guard rather than silently accepting it. + final msg = ChatMessage( + role: ChatMessageRole.model, + parts: [ + ToolPart.result(callId: '1', toolName: 'calc', result: {'v': 7}), + ], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); + test('second audio DataPart throws', () async { final msg = ChatMessage( role: ChatMessageRole.user, diff --git a/packages/flutter_gemma/test/genai/genai_output_converter_test.dart b/packages/flutter_gemma/test/genai/genai_output_converter_test.dart index 822b4d1d..02d5e465 100644 --- a/packages/flutter_gemma/test/genai/genai_output_converter_test.dart +++ b/packages/flutter_gemma/test/genai/genai_output_converter_test.dart @@ -30,12 +30,18 @@ void main() { expect(m.parts.whereType().single.text, 'answer'); }); - test('coalesced: tool calls suppress text', () { + test('coalesced: preamble text is kept alongside tool calls', () { + // The batch fold must not silently drop a "Let me check…" preamble when a + // tool call is present — the streaming path keeps it, so both must agree. final m = chatMessageFromParts( - text: 'ignored', + text: 'let me check', calls: const [FunctionCallResponse(name: 'f', args: {})], ); + expect(m.parts.whereType().single.text, 'let me check'); expect(m.parts.whereType(), hasLength(1)); - expect(m.parts.whereType(), isEmpty); + // Order follows generation: text precedes the tool call. + final textAt = m.parts.indexWhere((p) => p is TextPart); + final callAt = m.parts.indexWhere((p) => p is ToolPart); + expect(textAt, lessThan(callAt)); }); } diff --git a/packages/flutter_gemma/test/genai/link_reader_test.dart b/packages/flutter_gemma/test/genai/link_reader_test.dart index 781266f2..fb72ac7e 100644 --- a/packages/flutter_gemma/test/genai/link_reader_test.dart +++ b/packages/flutter_gemma/test/genai/link_reader_test.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:typed_data'; import 'package:flutter_gemma/core/genai/link_reader.dart'; @@ -43,4 +44,35 @@ void main() { throwsA(isA()), ); }); + + test( + 'throws on an empty 2xx body (blank media, not silently accepted)', + () async { + final client = MockClient( + (_) async => http.Response.bytes(Uint8List(0), 200), + ); + expect( + () => readLinkBytes( + Uri.parse('https://example.com/empty.png'), + httpClient: client, + ), + throwsA(isA()), + ); + }, + ); + + test('times out a hung server instead of holding forever', () async { + final client = MockClient((_) async { + await Future.delayed(const Duration(seconds: 5)); + return http.Response.bytes(Uint8List.fromList([1]), 200); + }); + expect( + () => readLinkBytes( + Uri.parse('https://example.com/slow.png'), + httpClient: client, + timeout: const Duration(milliseconds: 50), + ), + throwsA(isA()), + ); + }); } From 5915849dc5db4b4c9b40648b8c724c27e66bbaf9 Mon Sep 17 00:00:00 2001 From: Sasha Denisov Date: Fri, 17 Jul 2026 19:02:11 +0400 Subject: [PATCH 20/20] genai review fixes: onCancel double-fire + LinkPart scope - Guard onCancel teardown on !released: Dart fires onCancel after a normal or errored 'done' too, and the generating-only guard issued a spurious stopGeneration() on the shared session that could cancel the next back-to-back turn. - Scope LinkPart to bytes-only: the inference layer no longer fetches URLs or reads files (SSRF / arbitrary local-file read); LinkPart throws with guidance to resolve links caller-side into a DataPart. Removes link_reader and the http dependency; link handling belongs in flutter_gemma_agent. - Regression tests: no stopGeneration after a successful stream, back-to-back turn isolation, LinkPart throws. - migration.md core pin -> ^1.3.1. --- .../lib/core/genai/genai_chat_extension.dart | 59 ++++++-------- .../lib/core/genai/genai_input_converter.dart | 30 +++---- .../lib/core/genai/link_reader.dart | 52 ------------- .../lib/core/genai/link_reader_io.dart | 4 - .../lib/core/genai/link_reader_stub.dart | 7 -- packages/flutter_gemma/pubspec.yaml | 1 - .../genai/genai_input_converter_test.dart | 13 ++++ .../test/genai/link_reader_test.dart | 78 ------------------- .../test/genai/locked_stream_test.dart | 36 +++++++++ website/content/docs/migration.md | 2 +- 10 files changed, 90 insertions(+), 192 deletions(-) delete mode 100644 packages/flutter_gemma/lib/core/genai/link_reader.dart delete mode 100644 packages/flutter_gemma/lib/core/genai/link_reader_io.dart delete mode 100644 packages/flutter_gemma/lib/core/genai/link_reader_stub.dart delete mode 100644 packages/flutter_gemma/test/genai/link_reader_test.dart diff --git a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart index a67b4a3b..84f320ab 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_chat_extension.dart @@ -1,7 +1,6 @@ import 'dart:async'; import 'package:genai_primitives/genai_primitives.dart'; -import 'package:http/http.dart' as http; import '../chat.dart'; import '../model_response.dart'; @@ -23,58 +22,40 @@ void rejectModelRole(ChatMessage message) { extension GenAiChat on InferenceChat { /// Send one turn; returns the model turn as a role:model [ChatMessage] /// (text + tool calls + thinking as parts). - Future sendMessage( - ChatMessage message, { - http.Client? httpClient, - }) { + Future sendMessage(ChatMessage message) { rejectModelRole(message); return genaiLock.protect(() async { - await _stage([message], httpClient: httpClient); + await _stage([message]); return _foldToChatMessage(); }); } /// Streaming variant — partial role:model ChatMessages, one per delta. - Stream sendMessageStream( - ChatMessage message, { - http.Client? httpClient, - }) { + Stream sendMessageStream(ChatMessage message) { rejectModelRole(message); return _lockedStream(() async { - await _stage([message], httpClient: httpClient); + await _stage([message]); }); } /// STATEFUL batch: stage the whole list into THIS chat, then generate once. - Future generateContent( - List prompt, { - http.Client? httpClient, - }) { + Future generateContent(List prompt) { return genaiLock.protect(() async { - await _stage(prompt, httpClient: httpClient); + await _stage(prompt); return _foldToChatMessage(); }); } - Stream generateContentStream( - List prompt, { - http.Client? httpClient, - }) { + Stream generateContentStream(List prompt) { return _lockedStream(() async { - await _stage(prompt, httpClient: httpClient); + await _stage(prompt); }); } // --- internals --- - Future _stage( - List prompt, { - http.Client? httpClient, - }) async { - final messages = await messagesFromChatMessages( - prompt, - httpClient: httpClient, - ); + Future _stage(List prompt) async { + final messages = await messagesFromChatMessages(prompt); assertMessagesFitChat( messages, supportImage: supportImage, @@ -203,11 +184,21 @@ extension GenAiChat on InferenceChat { }, onCancel: () async { cancelled = true; - // Only tear down here once generation is running. While queued on - // acquire() or inside stage(), onListen owns the lock and releases it - // when it sees `cancelled`; releasing here would free a lock we don't - // hold yet, or free it mid-stage and let a second turn interleave. - if (generating) { + // Only tear down here once generation is running AND the turn hasn't + // already completed. Dart fires onCancel even after a normal `done` + // (closing the controller cancels the consumer subscription), and + // `generating` is never reset — so guarding on `generating` alone would + // issue a spurious stopGeneration() on the SHARED session after every + // successful stream, which (because onDone releases the lock first) can + // land on the NEXT queued turn and truncate it. `released` is set by + // onDone/onError before close(), so `!released` skips that late fire + // while a genuine mid-flight cancel (released still false) still tears + // down. Mirrors the FFI client's `mutexHeld` guard. + // + // While queued on acquire() or inside stage(), onListen owns the lock + // and releases it when it sees `cancelled`; releasing here would free a + // lock we don't hold yet, or free it mid-stage and let a turn interleave. + if (generating && !released) { await sub?.cancel(); await stopSafely(); release(); diff --git a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart index 906804ab..f25f7cf3 100644 --- a/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart +++ b/packages/flutter_gemma/lib/core/genai/genai_input_converter.dart @@ -2,10 +2,8 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:genai_primitives/genai_primitives.dart'; -import 'package:http/http.dart' as http; import '../message.dart'; -import 'link_reader.dart'; /// Converts a genai_primitives [ChatMessage] into flutter_gemma [Message]s. /// @@ -14,12 +12,11 @@ import 'link_reader.dart'; /// one sibling [Message] per tool result in part order. A [ThinkingPart] on a /// model turn is stripped by design (thoughts aren't fed back as history), so a /// thought-only model turn yields no Message; a [ThinkingPart] on a user turn is -/// misuse and throws. Async because LinkPart resolution may need I/O. Other -/// unsupported content throws rather than being silently dropped. -Future> messagesFromChatMessage( - ChatMessage message, { - http.Client? httpClient, -}) async { +/// misuse and throws. A [LinkPart] also throws: this inference layer does not +/// fetch URLs or read files — resolve links to bytes caller-side and pass a +/// [DataPart]. Other unsupported content throws rather than being silently +/// dropped. +Future> messagesFromChatMessage(ChatMessage message) async { if (message.role == ChatMessageRole.system) { throw ArgumentError( 'System messages are not input — set them via createChat(systemInstruction:).', @@ -39,9 +36,13 @@ Future> messagesFromChatMessage( switch (part) { case DataPart(:final bytes, :final mimeType): audio = _routeMedia(bytes, mimeType, images, audio); - case LinkPart(:final url, :final mimeType): - final bytes = await readLinkBytes(url, httpClient: httpClient); - audio = _routeMedia(bytes, mimeType, images, audio); + case LinkPart(): + throw UnsupportedError( + 'LinkPart is not resolved by flutter_gemma: an on-device model needs ' + 'the media bytes, and this inference layer does not fetch URLs or read ' + 'files. Resolve the link yourself and pass a DataPart with the bytes. ' + '(For URL/web content behind a permission gate, use flutter_gemma_agent.)', + ); case ToolPart(kind: ToolPartKind.result, :final toolName, :final result): if (!isUser) { throw UnsupportedError( @@ -97,12 +98,11 @@ Future> messagesFromChatMessage( /// Converts a list of [ChatMessage]s to flutter_gemma [Message]s in order, /// concatenating each message's expansion (see [messagesFromChatMessage]). Future> messagesFromChatMessages( - List messages, { - http.Client? httpClient, -}) async { + List messages, +) async { final out = []; for (final m in messages) { - out.addAll(await messagesFromChatMessage(m, httpClient: httpClient)); + out.addAll(await messagesFromChatMessage(m)); } return out; } diff --git a/packages/flutter_gemma/lib/core/genai/link_reader.dart b/packages/flutter_gemma/lib/core/genai/link_reader.dart deleted file mode 100644 index 51a7a120..00000000 --- a/packages/flutter_gemma/lib/core/genai/link_reader.dart +++ /dev/null @@ -1,52 +0,0 @@ -import 'dart:typed_data'; -import 'package:http/http.dart' as http; - -import 'link_reader_stub.dart' if (dart.library.io) 'link_reader_io.dart'; - -/// Resolves a genai_primitives LinkPart URI to bytes. -/// - data: decoded in-memory (no I/O) -/// - file:// dart:io on native; throws on web -/// - http(s):: injected client, else default http.Client (wasm-safe BrowserClient on web) -Future readLinkBytes( - Uri uri, { - http.Client? httpClient, - Duration timeout = const Duration(seconds: 30), -}) async { - switch (uri.scheme) { - case 'data': - final data = uri.data; - if (data == null) { - throw ArgumentError('Malformed data: URI: $uri'); - } - return data.contentAsBytes(); - case 'http': - case 'https': - final client = httpClient ?? http.Client(); - try { - // Bound the fetch: readLinkBytes is awaited while the chat mutex is - // held, so a hung server would otherwise wedge the whole genai surface. - final resp = await client.get(uri).timeout(timeout); - if (resp.statusCode < 200 || resp.statusCode >= 300) { - throw http.ClientException('HTTP ${resp.statusCode} for $uri', uri); - } - // A 2xx with an empty body (204/205, or a truncated proxy response) - // would otherwise flow on as blank image/audio — a silent media drop. - if (resp.bodyBytes.isEmpty) { - throw http.ClientException( - 'Empty body (${resp.statusCode}) for $uri', - uri, - ); - } - return resp.bodyBytes; - } finally { - if (httpClient == null) client.close(); - } - case 'file': - return readFileUri(uri); - default: - if (uri.scheme.isEmpty) return readFileUri(uri); // bare path - throw UnsupportedError( - 'Unsupported LinkPart scheme "${uri.scheme}": $uri', - ); - } -} diff --git a/packages/flutter_gemma/lib/core/genai/link_reader_io.dart b/packages/flutter_gemma/lib/core/genai/link_reader_io.dart deleted file mode 100644 index 9337befd..00000000 --- a/packages/flutter_gemma/lib/core/genai/link_reader_io.dart +++ /dev/null @@ -1,4 +0,0 @@ -import 'dart:io'; -import 'dart:typed_data'; - -Future readFileUri(Uri uri) => File.fromUri(uri).readAsBytes(); diff --git a/packages/flutter_gemma/lib/core/genai/link_reader_stub.dart b/packages/flutter_gemma/lib/core/genai/link_reader_stub.dart deleted file mode 100644 index 69e9da1d..00000000 --- a/packages/flutter_gemma/lib/core/genai/link_reader_stub.dart +++ /dev/null @@ -1,7 +0,0 @@ -import 'dart:typed_data'; - -/// Web: no filesystem. Callers must pass DataPart bytes, not file:// links. -Future readFileUri(Uri uri) => throw UnimplementedError( - 'file:// links are not readable on web ($uri). ' - 'Resolve to DataPart bytes before sending.', -); diff --git a/packages/flutter_gemma/pubspec.yaml b/packages/flutter_gemma/pubspec.yaml index 1e52a139..c8e9df8b 100644 --- a/packages/flutter_gemma/pubspec.yaml +++ b/packages/flutter_gemma/pubspec.yaml @@ -34,7 +34,6 @@ dependencies: code_assets: ^1.2.0 crypto: ^3.0.6 genai_primitives: ^0.2.3 - http: ^1.6.0 # Annotations (@immutable, @visibleForTesting) used across core, e.g. chat.dart. meta: ^1.15.0 # Serializes native LiteRT-LM conversation calls across concurrent diff --git a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart index 154a51fb..e217f9f0 100644 --- a/packages/flutter_gemma/test/genai/genai_input_converter_test.dart +++ b/packages/flutter_gemma/test/genai/genai_input_converter_test.dart @@ -149,4 +149,17 @@ void main() { throwsA(isA()), ); }); + + test('LinkPart throws — inference layer does not fetch URLs', () async { + // The on-device model needs bytes; resolving links (network/file I/O) is a + // caller/agent-layer concern, not core inference. Must throw, not fetch. + final msg = ChatMessage( + role: ChatMessageRole.user, + parts: [LinkPart(Uri.parse('https://example.com/cat.png'))], + ); + expect( + () => messagesFromChatMessage(msg), + throwsA(isA()), + ); + }); } diff --git a/packages/flutter_gemma/test/genai/link_reader_test.dart b/packages/flutter_gemma/test/genai/link_reader_test.dart deleted file mode 100644 index fb72ac7e..00000000 --- a/packages/flutter_gemma/test/genai/link_reader_test.dart +++ /dev/null @@ -1,78 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; -import 'dart:typed_data'; -import 'package:flutter_gemma/core/genai/link_reader.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:http/http.dart' as http; -import 'package:http/testing.dart'; - -void main() { - test('resolves a data: URI to bytes', () async { - final bytes = Uint8List.fromList([1, 2, 3, 4]); - final uri = Uri.parse( - 'data:application/octet-stream;base64,${base64Encode(bytes)}', - ); - expect(await readLinkBytes(uri), bytes); - }); - - test('throws on an unknown scheme', () async { - expect( - () => readLinkBytes(Uri.parse('ftp://x/y')), - throwsA(isA()), - ); - }); - - test('accepts a 2xx HTTP status other than 200', () async { - final bytes = Uint8List.fromList([9, 8, 7]); - final client = MockClient((_) async => http.Response.bytes(bytes, 206)); - expect( - await readLinkBytes( - Uri.parse('https://example.com/media.bin'), - httpClient: client, - ), - bytes, - ); - }); - - test('throws on a non-2xx HTTP status', () async { - final client = MockClient((_) async => http.Response('nope', 404)); - expect( - () => readLinkBytes( - Uri.parse('https://example.com/missing.bin'), - httpClient: client, - ), - throwsA(isA()), - ); - }); - - test( - 'throws on an empty 2xx body (blank media, not silently accepted)', - () async { - final client = MockClient( - (_) async => http.Response.bytes(Uint8List(0), 200), - ); - expect( - () => readLinkBytes( - Uri.parse('https://example.com/empty.png'), - httpClient: client, - ), - throwsA(isA()), - ); - }, - ); - - test('times out a hung server instead of holding forever', () async { - final client = MockClient((_) async { - await Future.delayed(const Duration(seconds: 5)); - return http.Response.bytes(Uint8List.fromList([1]), 200); - }); - expect( - () => readLinkBytes( - Uri.parse('https://example.com/slow.png'), - httpClient: client, - timeout: const Duration(milliseconds: 50), - ), - throwsA(isA()), - ); - }); -} diff --git a/packages/flutter_gemma/test/genai/locked_stream_test.dart b/packages/flutter_gemma/test/genai/locked_stream_test.dart index 1ec062e3..5a55803f 100644 --- a/packages/flutter_gemma/test/genai/locked_stream_test.dart +++ b/packages/flutter_gemma/test/genai/locked_stream_test.dart @@ -110,9 +110,45 @@ void main() { expect(chunks.map(_text).toList(), ['a', 'b']); expect(chat.genaiLock.isLocked, isFalse, reason: 'lock released on done'); + // Regression: Dart fires onCancel after a normal `done` too; the teardown + // there must NOT issue a stopGeneration() on the shared session. With the + // pre-fix `if (generating)` guard this was 1. + await Future.delayed(const Duration(milliseconds: 20)); + expect( + chat.stopGenerationCalls, + 0, + reason: 'no spurious stopGeneration after a successful stream', + ); }, ); + test('a turn completing does not stop the next back-to-back turn', () async { + // The race the spurious post-done onCancel opens: turn A's onDone releases + // the lock, turn B acquires and starts generating on the shared session, + // THEN A's late onCancel fires stopGeneration() — landing on B. + final chat = _FakeChat(); + + // Turn A: drive to completion. + final aDone = Completer(); + chat + .sendMessageStream(ChatMessage.user('A')) + .listen((_) {}, onDone: aDone.complete); + await Future.delayed(const Duration(milliseconds: 20)); + await chat.lastGen!.close(); + await aDone.future; + + // Turn B: start immediately, keep it live (do not close). + chat.sendMessageStream(ChatMessage.user('B')).listen((_) {}); + await Future.delayed(const Duration(milliseconds: 40)); + chat.lastGen!.add(const TextResponse('b')); // B is generating + + expect( + chat.stopGenerationCalls, + 0, + reason: "A's post-completion onCancel must not stop turn B", + ); + }); + test( 'error mid-stream forwards it, stops generation, releases the lock', () async { diff --git a/website/content/docs/migration.md b/website/content/docs/migration.md index 595909a8..1786acd6 100644 --- a/website/content/docs/migration.md +++ b/website/content/docs/migration.md @@ -29,7 +29,7 @@ dependencies: ``` dependencies: - flutter_gemma: ^1.3.0 # core — always required + flutter_gemma: ^1.3.1 # core — always required flutter_gemma_litertlm: ^1.1.0 # add if you run .litertlm models flutter_gemma_mediapipe: ^1.0.4 # add if you run .task / .bin models flutter_gemma_embeddings: ^1.0.2 # add if you compute embeddings