diff --git a/CLAUDE.md b/CLAUDE.md index a9a7f087..8b2ad85c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,7 +138,7 @@ Core has NO pigeon (dropped at the 1.0 cut; its value types are hand-written in - **MediaPipe Web**: v0.10.27, Android/iOS: v0.10.33 - **LiteRT-LM**: native libs from `native-v0.14.0` GitHub Release. Android tarball bundles the Qualcomm QNN dispatch stack and Windows tarball bundles Intel NPU dispatch (`LiteRtDispatch.dll` + OpenVino runtime + TBB) for `PreferredBackend.npu` (Qualcomm Snapdragon / Intel LunarLake/PantherLake). v0.14.0: native per-session sampler (opaque session-config) + #214 GPU output-garbage fix; Dawn split static→dynamic (Linux/Windows bundle `libwebgpu_dawn`). Windows discrete GPU (WebGPU/Dawn) regressed upstream — use CPU/NPU on Windows (LiteRT-LM #2957). - **large_file_handler**: `^0.5.0` (core dep; 0.5.0 declares all 6 platforms — needed for pana platform support + the dart2wasm-clean web graph) -- **Current Version**: core `flutter_gemma` `1.5.1`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.3.1`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.4`, `flutter_gemma_speech` `0.4.0`; `flutter_gemma_agent` `0.1.0`, `flutter_gemma_builtin_ai` `0.1.0` +- **Current Version**: core `flutter_gemma` `1.5.2`, `flutter_gemma_rag_sqlite` `1.1.0`, `flutter_gemma_rag_qdrant` `1.1.0`; `flutter_gemma_litertlm` `1.3.1`, `flutter_gemma_mediapipe` `1.0.4`, `flutter_gemma_embeddings` `1.0.4`, `flutter_gemma_speech` `0.4.1`; `flutter_gemma_agent` `0.1.0`, `flutter_gemma_builtin_ai` `0.1.0` - **0.15.2**: embedding unified on LiteRT C API via Dart FFI on all native platforms (Android + iOS + Desktop). Drops `localagents-rag` JVM dep on Android and the separate TFLite C 0.12.7 tarball on Desktop; `TensorFlowLiteC` pod no longer needed on iOS. Single source of truth for `TaskType.prefix` in Dart, fixes cross-platform embedding drift (#264). ## Platform-Specific Setup diff --git a/packages/flutter_gemma/CHANGELOG.md b/packages/flutter_gemma/CHANGELOG.md index 3f6c0beb..ff10221a 100644 --- a/packages/flutter_gemma/CHANGELOG.md +++ b/packages/flutter_gemma/CHANGELOG.md @@ -1,3 +1,6 @@ +## 1.5.2 +- Add TtsModelType.qwen3. + ## 1.5.1 - fix: namespace companion install files (tokenizers, TTS bundle aux) per model, fixing STT/embedding tokenizer collisions. - fix: migrate legacy (pre-namespacing) tokenizer installs on restore, so existing STT/embedding models keep working after upgrade. diff --git a/packages/flutter_gemma/example/integration_test/qwen3_tts_test.dart b/packages/flutter_gemma/example/integration_test/qwen3_tts_test.dart new file mode 100644 index 00000000..fc69d2ce --- /dev/null +++ b/packages/flutter_gemma/example/integration_test/qwen3_tts_test.dart @@ -0,0 +1,157 @@ +// On-device Qwen3-TTS e2e — installs Qwen3-TTS from HuggingFace through the +// REAL public flow (initialize -> installTts -> getActiveTts -> synthesize), +// exercising the plain-basename manifest -> getModelFilePaths -> +// Qwen3TtsCore.load consistency ON DEVICE (the artifact-gated unit tests +// build their own artifactPaths by directory scan, so they never touch the +// install path itself). +// +// Unlike `tts_matcha_test.dart` (byte-exact against a committed golden, +// fixed CFM seed), this test asserts PLAUSIBILITY only: the runtime uses the +// default int4 talker with `doSample: true`, so exact bytes are not +// reproducible across runs. The byte-exact / corr~=1.0 oracle for the shared +// AR pipeline lives in the fp32-greedy unit test (`qwen3_synthesize_test.dart`, +// artifact-gated, not a device test). +// +// Also asserts language selection actually changes the output end to end: +// synthesizing the same text with `language: 'german'` after closing the +// `'english'` synthesizer (required by the fail-loud language-cache guard — +// see Task 5.4) must NOT produce byte-identical audio to the english pass. +// +// The full 9-file bundle is ~1.9 GB; this test can take tens of minutes on a +// slow connection. Run: cd packages/flutter_gemma/example && \ +// flutter test integration_test/qwen3_tts_test.dart -d macos +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:flutter_gemma/flutter_gemma.dart' + show FlutterGemma, TtsModelType; +import 'package:flutter_gemma_speech/flutter_gemma_speech.dart' + show LiteRtTtsBackend; + +const _modelUrl = + 'https://huggingface.co/litert-community/Qwen3-TTS-12Hz-0.6B-Base/resolve/main/'; +const _text = 'Hello from on device text to speech.'; + +double _rms(Uint8List pcm) { + final samples = Int16List.sublistView(pcm); + if (samples.isEmpty) return 0; + var sumSquares = 0.0; + for (final s in samples) { + final n = s / 32768.0; + sumSquares += n * n; + } + return math.sqrt(sumSquares / samples.length); +} + +bool _bytesEqual(Uint8List a, Uint8List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} + +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + testWidgets( + 'Qwen3-TTS installs from HF, synthesizes plausible english audio, and ' + 'german differs from english', + (_) async { + await FlutterGemma.initialize(ttsBackends: const [LiteRtTtsBackend()]); + + await FlutterGemma.installTts() + .fromNetwork(_modelUrl) + .ofType(TtsModelType.qwen3) + .install(); + + final englishSynth = await FlutterGemma.getActiveTts(language: 'english'); + Uint8List englishPcm; + try { + englishPcm = await englishSynth.synthesize(_text); + + expect(englishPcm, isNotEmpty, reason: 'english PCM was empty'); + expect(englishSynth.sampleRate, 24000); + + final durationSeconds = englishPcm.length / 2 / englishSynth.sampleRate; + final rms = _rms(englishPcm); + + debugPrint( + 'QWEN3-TTS-DEVICE-GATE<<>>', + ); + + // Plausibility oracle (int4 + doSample:true is not byte-reproducible + // across runs/backends): non-silent, non-clipped-garbage, and a + // ~7-word sentence should take at least ~1s of audio. + expect( + durationSeconds, + greaterThanOrEqualTo(1.0), + reason: 'Synthesized audio implausibly short: ${durationSeconds}s', + ); + expect( + rms, + inInclusiveRange(0.005, 0.5), + reason: 'RMS out of the plausible speech band: $rms', + ); + } finally { + // REQUIRED before requesting a different language on the same + // active model — the language-cache guard (Task 5.4) throws + // StateError otherwise instead of silently reusing the english + // synthesizer for a german request. + await englishSynth.close(); + } + + final germanSynth = await FlutterGemma.getActiveTts(language: 'german'); + try { + final germanPcm = await germanSynth.synthesize(_text); + + expect(germanPcm, isNotEmpty, reason: 'german PCM was empty'); + expect(germanSynth.sampleRate, 24000); + + final durationSeconds = germanPcm.length / 2 / germanSynth.sampleRate; + final rms = _rms(germanPcm); + + debugPrint( + 'QWEN3-TTS-DEVICE-GATE<<>>', + ); + + expect( + durationSeconds, + greaterThanOrEqualTo(1.0), + reason: 'Synthesized audio implausibly short: ${durationSeconds}s', + ); + expect( + rms, + inInclusiveRange(0.005, 0.5), + reason: 'RMS out of the plausible speech band: $rms', + ); + + // Language selection actually changes the model's output, end to + // end through the public API: german must not equal english either + // in length or in bytes (a byte match at equal length would mean + // the language control token silently had no effect). + final identical = + germanPcm.length == englishPcm.length && + _bytesEqual(germanPcm, englishPcm); + expect( + identical, + isFalse, + reason: + 'german synthesis produced byte-identical audio to english ' + '(pcmBytes=${germanPcm.length} vs ${englishPcm.length}) — ' + 'language selection had no effect', + ); + } finally { + await germanSynth.close(); + } + }, + timeout: const Timeout(Duration(minutes: 90)), + ); +} diff --git a/packages/flutter_gemma/example/lib/models/tts_model.dart b/packages/flutter_gemma/example/lib/models/tts_model.dart index 0bc5cad0..34089f67 100644 --- a/packages/flutter_gemma/example/lib/models/tts_model.dart +++ b/packages/flutter_gemma/example/lib/models/tts_model.dart @@ -3,7 +3,11 @@ import 'package:flutter_gemma/flutter_gemma.dart' show TtsModelType; /// Catalog of on-device TTS models. SELECTABLE like STT — each entry carries /// the [TtsModelType] that tells the generic `LiteRtTtsBackend` which /// `TtsModelProfile` to run. Install uses one base URL + `.ofType`. -/// Only [matcha] is wired; kokoro/supertonic are follow-ons (isSupported false). +/// [matcha] and [qwen3] are wired; kokoro/supertonic are follow-ons +/// (isSupported false). [qwen3] additionally exposes 11 selectable +/// languages (`tts_screen.dart`'s language dropdown, populated from +/// `flutter_gemma_speech`'s `qwen3SupportedLanguages`) — [matcha] is +/// English-only (its locale comes from its bundle, not a runtime param). enum TtsModel { matcha( baseUrl: 'https://huggingface.co/litert-community/Matcha-TTS/resolve/main/', @@ -12,6 +16,17 @@ enum TtsModel { ttsModelType: TtsModelType.matcha, isSupported: true, ), + qwen3( + baseUrl: + 'https://huggingface.co/litert-community/Qwen3-TTS-12Hz-0.6B-Base/resolve/main/', + displayName: 'Qwen3-TTS 0.6B (11 languages)', + size: '~1.9GB', + ttsModelType: TtsModelType.qwen3, + isSupported: true, + notes: + 'CPU only, slow (RTF≈3 — ~3s of compute per 1s of audio), needs a ' + '6 GB-RAM-class device', + ), kokoro( baseUrl: 'https://huggingface.co/litert-community/Kokoro-82M/resolve/main/', displayName: 'Kokoro 82M', @@ -38,6 +53,7 @@ enum TtsModel { required this.ttsModelType, this.isSupported = true, this.unsupportedReason, + this.notes, }); /// HuggingFace repo base URL each manifest filename is resolved against. @@ -59,4 +75,9 @@ enum TtsModel { /// Why [isSupported] is false; null when supported. final String? unsupportedReason; + + /// UI-only performance/hardware caveat shown under the model info card + /// (e.g. CPU-only, expected RTF, RAM class) — null when there's nothing + /// notable to call out. + final String? notes; } diff --git a/packages/flutter_gemma/example/lib/tts_screen.dart b/packages/flutter_gemma/example/lib/tts_screen.dart index 5736a862..787c76a5 100644 --- a/packages/flutter_gemma/example/lib/tts_screen.dart +++ b/packages/flutter_gemma/example/lib/tts_screen.dart @@ -4,18 +4,27 @@ import 'package:flutter_gemma/flutter_gemma.dart'; import 'package:flutter_gemma_example/models/tts_model.dart'; import 'package:flutter_gemma_example/utils/audio_converter.dart'; import 'package:flutter_gemma_example/utils/platform_io_helper.dart'; +import 'package:flutter_gemma_speech/flutter_gemma_speech.dart' + show qwen3SupportedLanguages; import 'package:just_audio/just_audio.dart'; import 'package:path_provider/path_provider.dart'; /// TTS synthesize screen — mirrors [SttScreen]'s install-in-initState / /// `mounted`-after-await / reentrancy discipline, but the flow is simpler: /// no recording, just a text field → synthesize → play. Installs -/// (idempotent) and activates [TtsModel.matcha] — the only catalog entry -/// with a shipped `TtsModelProfile` today (see `models/tts_model.dart`) — -/// then lets the user type text and hear it spoken via +/// (idempotent) and activates the selected [TtsModel] (defaults to +/// [TtsModel.matcha]) then lets the user type text and hear it spoken via /// [SpeechSynthesizer.synthesize] + `just_audio` playback of the /// WAV-wrapped PCM (`AudioConverter.pcmToWav`). /// +/// [TtsModel.qwen3] additionally exposes a LANGUAGE dropdown (11 languages, +/// populated from `qwen3SupportedLanguages` — see Task 5.4's brief for why +/// language, not voice, is the v1-selectable dimension: the model ships +/// exactly one voice, no voice picker). Language is a create-time param +/// (`FlutterGemma.getActiveTts(language: ...)`) — changing it re-creates the +/// synthesizer (closes the old one, installs/activates again), reloading +/// the ~1.9 GB model. Switching the model dropdown does the same. +/// /// `createFile` (not raw `dart:io`) is used to write the WAV to a temp file /// so this screen still compiles for web, where `flutter_gemma_speech` has /// no TTS arm (the init call surfaces that as [_initError] instead). @@ -27,7 +36,8 @@ class TtsScreen extends StatefulWidget { } class _TtsScreenState extends State { - static const _model = TtsModel.matcha; + TtsModel _model = TtsModel.matcha; + String _language = 'english'; SpeechSynthesizer? _synth; bool _isInitializing = true; @@ -54,9 +64,25 @@ class _TtsScreenState extends State { super.dispose(); } + bool get _isQwen3 => _model.ttsModelType == TtsModelType.qwen3; + /// Install (idempotent) + activate [_model], then create the /// [SpeechSynthesizer]. Mirrors `SttScreen._initializeSttModel`. + /// + /// Closes any previously-created [_synth] first — `getActiveTts` reuses a + /// singleton for the same active model, so a language change alone (same + /// [_model], new [_language]) would otherwise silently keep serving the + /// OLD language unless the old instance is closed first (see + /// `FlutterGemma.getActiveTts`'s doc). Future _initializeTtsModel() async { + setState(() { + _isInitializing = true; + _initError = null; + _downloadPercent = null; + }); + final oldSynth = _synth; + _synth = null; + await oldSynth?.close(); try { await FlutterGemma.installTts() .fromNetwork(_model.baseUrl) @@ -67,7 +93,9 @@ class _TtsScreenState extends State { }) .install(); - final synth = await FlutterGemma.getActiveTts(); + final synth = await FlutterGemma.getActiveTts( + language: _isQwen3 ? _language : null, + ); if (!mounted) return; setState(() { @@ -86,6 +114,22 @@ class _TtsScreenState extends State { } } + /// Switch the active catalog entry and reinitialize. No-op if [model] is + /// already selected or a (re)initialization is already in flight. + void _selectModel(TtsModel model) { + if (model == _model || _isInitializing) return; + setState(() => _model = model); + _initializeTtsModel(); + } + + /// Switch the qwen3 language and reinitialize (see [_initializeTtsModel]'s + /// doc for why this must close+recreate rather than just updating state). + void _selectLanguage(String language) { + if (language == _language || _isInitializing) return; + setState(() => _language = language); + _initializeTtsModel(); + } + Future _speak() async { if (_synth == null || _isSpeaking) return; // reentrancy guard setState(() { @@ -143,6 +187,7 @@ class _TtsScreenState extends State { } Widget _buildModelInfoCard() { + final supported = TtsModel.values.where((m) => m.isSupported).toList(); return Card( color: const Color(0xFF1a3a5c), child: Padding( @@ -159,14 +204,79 @@ class _TtsScreenState extends State { ), ), const SizedBox(height: 12), + _buildDropdownRow( + label: 'Model:', + value: _model, + items: [ + for (final m in supported) + DropdownMenuItem(value: m, child: Text(m.displayName)), + ], + onChanged: (m) { + if (m != null) _selectModel(m); + }, + ), + const SizedBox(height: 8), _buildInfoRow('Size:', _model.size), _buildInfoRow('Type:', 'Text-to-Speech'), + if (_isQwen3) ...[ + const SizedBox(height: 8), + _buildDropdownRow( + label: 'Language:', + value: _language, + items: [ + for (final lang in qwen3SupportedLanguages) + DropdownMenuItem(value: lang, child: Text(lang)), + ], + onChanged: (lang) { + if (lang != null) _selectLanguage(lang); + }, + ), + ], + if (_model.notes != null) ...[ + const SizedBox(height: 12), + Text( + _model.notes!, + style: const TextStyle( + color: Colors.orangeAccent, + fontSize: 12, + ), + ), + ], ], ), ), ); } + /// A label + [DropdownButton] row, disabled while [_isInitializing] (a + /// model/language switch is already in flight — reentrancy guard mirrors + /// [_selectModel]/[_selectLanguage]'s own check). + Widget _buildDropdownRow({ + required String label, + required T value, + required List> items, + required ValueChanged onChanged, + }) { + return Row( + children: [ + SizedBox( + width: 80, + child: Text(label, style: const TextStyle(color: Colors.white70)), + ), + Expanded( + child: DropdownButton( + value: value, + items: items, + onChanged: _isInitializing ? null : onChanged, + isExpanded: true, + dropdownColor: const Color(0xFF1a3a5c), + style: const TextStyle(color: Colors.white), + ), + ), + ], + ); + } + Widget _buildInfoRow(String label, String value) { return Padding( padding: const EdgeInsets.symmetric(vertical: 4.0), diff --git a/packages/flutter_gemma/example/pubspec.lock b/packages/flutter_gemma/example/pubspec.lock index e20f00b0..8bec4fdd 100644 --- a/packages/flutter_gemma/example/pubspec.lock +++ b/packages/flutter_gemma/example/pubspec.lock @@ -241,7 +241,7 @@ packages: path: ".." relative: true source: path - version: "1.5.1" + version: "1.5.2" flutter_gemma_agent: dependency: "direct main" description: @@ -297,7 +297,7 @@ packages: path: "../../flutter_gemma_speech" relative: true source: path - version: "0.4.0" + version: "0.4.1" flutter_inappwebview: dependency: transitive description: diff --git a/packages/flutter_gemma/ios/flutter_gemma.podspec b/packages/flutter_gemma/ios/flutter_gemma.podspec index de1fdeb2..9269b776 100644 --- a/packages/flutter_gemma/ios/flutter_gemma.podspec +++ b/packages/flutter_gemma/ios/flutter_gemma.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'flutter_gemma' - s.version = '1.5.1' + s.version = '1.5.2' s.summary = 'Flutter plugin for running Gemma and other LLMs locally on iOS.' s.description = <<-DESC Core runtime for running Gemma 4, Gemma3n, Gemma 3, FastVLM, Qwen3, diff --git a/packages/flutter_gemma/lib/core/api/flutter_gemma.dart b/packages/flutter_gemma/lib/core/api/flutter_gemma.dart index 1d0e21c9..63d4504c 100644 --- a/packages/flutter_gemma/lib/core/api/flutter_gemma.dart +++ b/packages/flutter_gemma/lib/core/api/flutter_gemma.dart @@ -554,6 +554,13 @@ class FlutterGemma { /// /// Runtime parameters: /// - [preferredBackend]: CPU or GPU preference (optional) + /// - [language]: TTS-only (Qwen3; ignored by Matcha, whose locale comes + /// from its bundle instead) — one of `flutter_gemma_speech`'s + /// `qwen3SupportedLanguages`, or `'auto'`; defaults to `'english'` when + /// null. An unsupported value throws `ArgumentError` before the model + /// loads. A previously-created synthesizer for the same active model is + /// reused across calls (create-time singleton) — to pick up a new + /// [language], [close][SpeechSynthesizer.close] it first. /// /// Throws: /// - [StateError] if no active TTS model is set @@ -571,6 +578,7 @@ class FlutterGemma { /// ``` static Future getActiveTts({ PreferredBackend? preferredBackend, + String? language, }) async { final manager = FlutterGemmaPlugin.instance.modelManager; final activeSpec = manager.activeTtsModel; @@ -591,6 +599,7 @@ class FlutterGemma { // Create SpeechSynthesizer using active spec (paths resolved automatically) return await FlutterGemmaPlugin.instance.createTtsModel( preferredBackend: preferredBackend, + language: language, ); } diff --git a/packages/flutter_gemma/lib/core/api/tts_installation_builder.dart b/packages/flutter_gemma/lib/core/api/tts_installation_builder.dart index fd99e4a5..75e1dfc0 100644 --- a/packages/flutter_gemma/lib/core/api/tts_installation_builder.dart +++ b/packages/flutter_gemma/lib/core/api/tts_installation_builder.dart @@ -1,7 +1,6 @@ import 'package:flutter_gemma/core/utils/gemma_log.dart'; import 'package:flutter_gemma/core/di/service_registry.dart'; import 'package:flutter_gemma/core/model_management/model_specs.dart'; -import 'package:flutter_gemma/core/services/model_repository.dart' as repo; import 'package:flutter_gemma/flutter_gemma.dart'; /// Fluent builder for TTS (text-to-speech) model installation. @@ -87,8 +86,15 @@ class TtsInstallationBuilder { ); } - String joinUrl(String base, String fn) => - base.endsWith('/') ? '$base$fn' : '$base/$fn'; + // Most bundle members are fetched from `baseUrl/`, but a + // few (e.g. qwen3's embedding tables + demo voice) live under a + // subdirectory on the origin server even though their INSTALLED + // identity is the plain basename — see + // `TtsModelTypeManifest.urlSuffixFor`'s doc for why that split is safe. + String joinUrl(String base, String fn) { + final suffix = ttsModelType.urlSuffixFor(fn); + return base.endsWith('/') ? '$base$suffix' : '$base/$suffix'; + } final spec = TtsModelSpec.fromManifest( name: _name ?? ttsModelType.name, @@ -99,61 +105,27 @@ class TtsInstallationBuilder { final registry = ServiceRegistry.instance; final repository = registry.modelRepository; - final fileSystem = registry.fileSystemService; final handlerRegistry = registry.sourceHandlerRegistry; - final manifest = ttsModelType.manifest; - // Only a bundle basename that is UNIQUE across the ENTIRE TTS catalog is - // safe to adopt from a pre-refactor flat install: a shared basename (e.g. a - // generic `config.json`) carries no type marker, so blindly adopting it - // could hand THIS model another TTS model's file — the exact collision the - // namespacing refactor fixes (see FileSystemService.adoptLegacyFile's - // contract). Today matcha is the only type, so every basename is unique; - // this guard keeps the adoption correct the moment a second TTS model - // shares a basename (that basename then falls through to a fresh download). - final catalogBasenameCounts = {}; - for (final type in TtsModelType.values) { - final List typeManifest; - try { - typeManifest = type.manifest; - } on UnimplementedError { - // A not-yet-wired TTS type has no installed files, so it can't own a - // colliding plain file — skip it rather than let its manifest getter - // throw. - continue; - } - for (final fn in typeManifest) { - catalogBasenameCounts[fn] = (catalogBasenameCounts[fn] ?? 0) + 1; - } - } - + // NOTE: install-time legacy-file adoption (renaming an on-disk + // pre-1.5.1-namespacing plain file onto the namespaced identity) was + // removed here — a bundle basename can be unique within the TTS catalog + // yet still collide with a plain file left behind by an STT/embedding + // install (e.g. Qwen3's `tokenizer.json`), and adoption has no way to + // verify the on-disk file actually belongs to this model (no size/hash + // check available). A manifest file that is not already installed is + // downloaded fresh (never adopted from a legacy plain file); + // already-installed files are still skipped by the loop below. The safe + // migration path for genuinely-legacy TTS installs is restore-time, in + // MobileModelManager._migrateLegacyCompanionForRestore. final files = spec.files; var done = 0; for (var i = 0; i < files.length; i++) { _cancelToken?.throwIfCancelled(); final file = files[i]; - final legacyBasename = manifest[i]; - final adoptable = (catalogBasenameCounts[legacyBasename] ?? 0) == 1; if (await repository.isInstalled(file.filename)) { gemmaLog('ℹ️ TTS bundle file already installed: ${file.filename}'); - } else if (adoptable && - await fileSystem.adoptLegacyFile(legacyBasename, file.filename)) { - gemmaLog( - '♻️ Adopted legacy TTS bundle file: $legacyBasename -> ${file.filename}', - ); - final newPath = await fileSystem.getWriteTargetPath(file.filename); - final sizeBytes = await fileSystem.getFileSize(newPath); - await repository.saveModel( - repo.ModelInfo( - id: file.filename, - source: file.source, - installedAt: DateTime.now(), - sizeBytes: sizeBytes, - type: repo.ModelType.tts, - hasLoraWeights: false, - ), - ); } else { gemmaLog('📥 Installing TTS bundle file: ${file.filename}...'); final handler = handlerRegistry.getHandler(file.source); diff --git a/packages/flutter_gemma/lib/core/model_management/types/tts_model_spec.dart b/packages/flutter_gemma/lib/core/model_management/types/tts_model_spec.dart index 88a5259b..2ca9bf44 100644 --- a/packages/flutter_gemma/lib/core/model_management/types/tts_model_spec.dart +++ b/packages/flutter_gemma/lib/core/model_management/types/tts_model_spec.dart @@ -1,9 +1,24 @@ part of '../model_specs.dart'; /// Text-to-speech model families supported by the pluggable TTS backends. -/// Only [matcha] has a shipped [TtsModelProfile]/pipeline (`flutter_gemma_speech`); -/// [kokoro]/[supertonic] are documented follow-ons (fail-loud until wired). -enum TtsModelType { matcha, supertonic, kokoro } +/// [matcha] and [qwen3] have shipped [TtsModelProfile]/pipelines +/// (`flutter_gemma_speech`); [kokoro]/[supertonic] are documented follow-ons +/// (fail-loud until wired). +enum TtsModelType { matcha, supertonic, kokoro, qwen3 } + +/// [TtsModelType.qwen3]'s 4 embedding-table bundle members — the SINGLE +/// source of truth for which manifest basenames live under the HF repo's +/// `tables/` subdirectory. Spread into [TtsModelTypeManifest.manifest]'s +/// qwen3 entry AND consulted by [TtsModelTypeManifest.urlSuffixFor], so the +/// two can never desync (a prior version hardcoded this set a second time in +/// [TtsModelTypeManifest.urlSuffixFor]; a rename here used to require +/// remembering to update both places). +const _qwen3TableFiles = [ + 'text_embedding_fp16.npy', + 'text_projection_fp32.npz', + 'codec_embedding_fp32.npy', + 'mtp_embeddings_fp16.npy', +]; /// The filenames a given TTS model needs, installed together as a bundle from /// one source. Fail-loud for unwired families. @@ -19,10 +34,55 @@ extension TtsModelTypeManifest on TtsModelType { 'config.json', 'g2p_meta.json', ], + // litert-community/Qwen3-TTS-12Hz-0.6B-Base on HuggingFace. Entries are + // PLAIN basenames — deliberately NOT the `tables/`/`voices/` subpaths + // those 5 files actually live under on the HF repo (see [urlSuffixFor]). + // This is load-bearing, not a simplification: [TtsBundleFile.fromSource] + // derives a file's installed identity (== [TtsBundleFile.prefsKey] == + // the `artifactPaths` key `Qwen3TtsCore.load`/`Qwen3Tables.load` read) + // from the LAST path segment of its source, and + // `MobileModelManager._restoreActiveTtsModel` independently re-derives + // the SAME namespaced on-disk path directly from this raw manifest + // string (`FileNameUtils.namespaced(modelId, fn)`) — if an entry here + // contained a `/`, those two derivations would silently diverge (fresh + // install writes the URL-extracted flat basename; restore would look for + // a namespaced path with an embedded subdirectory) and active-model + // restore would fail to find the file after every relaunch. No + // `config.json` — Qwen3's sampling params (top_k=50, temperature=0.9, + // repetition_penalty=1.05, max_frames=512) are recipe constants threaded + // as Dart defaults, not model-file-driven. + TtsModelType.qwen3 => const [ + 'talker_int4.tflite', + 'mtp_fp32.tflite', + 'codec_decoder_fp32.tflite', + 'tokenizer.json', + ..._qwen3TableFiles, + 'demo_speaker.npy', + ], _ => throw UnimplementedError( - 'TTS manifest for $this is a follow-on (only matcha is wired)', + 'TTS manifest for $this is a follow-on (only matcha/qwen3 are wired)', ), }; + + /// The URL path segment a bundle member is actually fetched from under the + /// model's `resolve/main/` base — usually just [plainFilename] itself, but + /// [TtsModelType.qwen3]'s 4 embedding-table members and its 1 demo-voice + /// member live in the HF repo's `tables/`/`voices/` subdirectories even + /// though (per [manifest]'s doc) their INSTALLED identity is the bare + /// basename. Safe to add a directory prefix here: it changes ONLY where + /// [TtsInstallationBuilder] fetches the bytes from, never the on-disk + /// filename/prefsKey (which [TtsBundleFile.fromSource] derives from the + /// LAST path segment of the resulting URL — a `tables/` prefix earlier in + /// the path is dropped exactly like the plain-basename manifest entry + /// intends). A no-op (identity) for every other [TtsModelType]. + String urlSuffixFor(String plainFilename) { + if (this != TtsModelType.qwen3) return plainFilename; + if (_qwen3TableFiles.contains(plainFilename)) { + return 'tables/$plainFilename'; + } + if (plainFilename == 'demo_speaker.npy') return 'voices/$plainFilename'; + return plainFilename; + } } /// One file of a TTS model bundle. [filename] is namespaced by the owning @@ -88,9 +148,12 @@ class TtsBundleFile extends ModelFile { @override int? get minimumSizeBytes { // TTS bundles legitimately include small aux files (emb.bin ~137 KB, - // config/meta json, gzipped dict). Give those a 1 KB floor; the large - // .tflite graphs return null and keep the validator's 1 MB default. - const smallBundleExts = {'.bin', '.gz', '.json'}; + // config/meta json, gzipped dict, qwen3's demo_speaker.npy x-vector + // ~4 KB). Give those a 1 KB floor; the large .tflite graphs return null + // and keep the validator's 1 MB default. .npy/.npz also cover qwen3's + // larger embedding tables — a looser floor for those is accepted so the + // one genuinely small .npy (demo_speaker.npy) isn't rejected. + const smallBundleExts = {'.bin', '.gz', '.json', '.npy', '.npz'}; return smallBundleExts.contains(extension) ? 1024 : null; } } diff --git a/packages/flutter_gemma/lib/core/registry/runtime_config.dart b/packages/flutter_gemma/lib/core/registry/runtime_config.dart index 846e5a34..7b72caf6 100644 --- a/packages/flutter_gemma/lib/core/registry/runtime_config.dart +++ b/packages/flutter_gemma/lib/core/registry/runtime_config.dart @@ -17,6 +17,7 @@ class RuntimeConfig { this.maxConcurrentSessions, this.loraRanks, this.artifactPaths, + this.language, }) : assert(maxTokens >= 0, 'maxTokens must not be negative'), assert( maxNumImages == null || maxNumImages >= 0, @@ -61,4 +62,13 @@ class RuntimeConfig { /// backend resolves each graph/dictionary/embedding file by name from this /// map. Null for single-file models (inference/embedding/STT). final Map? artifactPaths; + + /// TTS-only (Qwen3): the language to condition generation on — one of + /// `flutter_gemma_speech`'s `qwen3SupportedLanguages`, or `'auto'`. Null + /// for non-TTS models, and for TTS models with no language parameter + /// (e.g. Matcha, whose locale comes from its `TtsModelProfile.locale` + /// instead) — the TTS backend defaults a null value to `'english'`. Not + /// validated here; the TTS backend rejects an unsupported value with + /// `ArgumentError` before loading the model. + final String? language; } diff --git a/packages/flutter_gemma/lib/desktop/flutter_gemma_desktop.dart b/packages/flutter_gemma/lib/desktop/flutter_gemma_desktop.dart index fa2ad9f9..3151bb8b 100644 --- a/packages/flutter_gemma/lib/desktop/flutter_gemma_desktop.dart +++ b/packages/flutter_gemma/lib/desktop/flutter_gemma_desktop.dart @@ -35,6 +35,18 @@ import '../mobile/flutter_gemma_mobile.dart' show MobileModelManager; import '../core/model_management/constants/preferences_keys.dart'; +/// Normalizes a `createTtsModel`/`getActiveTts` `language` argument for the +/// same-model reuse guard's store/compare — defaults `null` to `'english'` +/// (mirrors `LiteRtTtsBackend.createModel`'s `config.language ?? 'english'`) +/// and lowercases so `null`, `'english'`, and `'English'` all compare equal +/// (they build the SAME synthesizer either way — `Qwen3Prompt.build` already +/// lowercases the language it's given). Without this, storing/comparing the +/// raw argument tripped a spurious `StateError` for effectively-identical +/// requests (e.g. `getActiveTts()` then `getActiveTts(language: 'english')`). +/// Duplicated from `flutter_gemma_mobile.dart` (same shape, separate shell). +String _normalizeTtsLanguage(String? language) => + (language ?? 'english').toLowerCase(); + /// Desktop implementation of FlutterGemma plugin /// /// Uses dart:ffi to communicate directly with LiteRT-LM C API @@ -81,6 +93,14 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { SpeechSynthesizer? _initializedTtsModel; TtsModelSpec? _lastActiveTtsSpec; // Track which spec was used to create _initializedTtsModel + // The `language` the active singleton was built with, NORMALIZED + // ([_normalizeTtsLanguage] — defaulted + lowercased) so a same-effective- + // language request compared raw-to-raw (e.g. null vs. 'english', or + // 'English' vs. 'english') doesn't spuriously trip the guard below. + // Reusing the singleton for a genuinely DIFFERENT language would silently + // emit wrong-language audio with no error — see the same-model branch in + // createTtsModel below. + String? _lastActiveTtsLanguage; @override ModelFileManager get modelManager => _modelManager; @@ -500,6 +520,7 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { @override Future createTtsModel({ PreferredBackend? preferredBackend, + String? language, }) async { final activeModel = _modelManager.activeTtsModel; if (activeModel is! TtsModelSpec) { @@ -522,9 +543,25 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { _initTtsCompleter = null; _initializedTtsModel = null; _lastActiveTtsSpec = null; + _lastActiveTtsLanguage = null; await old?.close(); + } else if (_normalizeTtsLanguage(language) != _lastActiveTtsLanguage) { + // Same model, but a DIFFERENT language was requested — reusing the + // singleton here would silently emit WRONG-LANGUAGE audio with no + // error. Fail loud instead of reusing: the caller must close() the + // existing synthesizer first (tts_screen.dart already does this on + // every model/language switch — see LiteRtSpeechSynthesizer/ + // getActiveTts's docs). Both + // sides are normalized ([_normalizeTtsLanguage]) so this only fires + // for a GENUINELY different language, not e.g. null vs. 'english' + // or 'English' vs. 'english'. + throw StateError( + 'Active TTS synthesizer was created for language ' + "'$_lastActiveTtsLanguage'; call close() before requesting " + "'${_normalizeTtsLanguage(language)}'.", + ); } else { - // Same model - return existing singleton + // Same model, same language - return existing singleton return _initTtsCompleter!.future; } } @@ -550,6 +587,7 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { .first, // representative; TTS backend uses artifactPaths artifactPaths: filePaths, preferredBackend: preferredBackend, + language: language, ); final backend = TtsRegistry.instance.findFor(activeModel); if (backend == null) { @@ -570,6 +608,7 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { // package-built model fires this via CloseNotifier (addCloseListener). _initializedTtsModel = synth; _lastActiveTtsSpec = activeModel; + _lastActiveTtsLanguage = _normalizeTtsLanguage(language); synth.addCloseListener(() { // Only reset if this close-listener still belongs to the current // singleton — a newer model may already have replaced it (the @@ -578,6 +617,7 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { _initializedTtsModel = null; _initTtsCompleter = null; _lastActiveTtsSpec = null; + _lastActiveTtsLanguage = null; } }); @@ -588,6 +628,7 @@ class FlutterGemmaDesktop extends FlutterGemmaPlugin { _initTtsCompleter = null; _initializedTtsModel = null; _lastActiveTtsSpec = null; + _lastActiveTtsLanguage = null; // Return the completer's future (rather than a bare rethrow) so there // is exactly one Future in flight for this call — an unheeded // `completer.future` (left behind whenever no concurrent caller diff --git a/packages/flutter_gemma/lib/flutter_gemma_interface.dart b/packages/flutter_gemma/lib/flutter_gemma_interface.dart index 8d3417ad..1596c6af 100644 --- a/packages/flutter_gemma/lib/flutter_gemma_interface.dart +++ b/packages/flutter_gemma/lib/flutter_gemma_interface.dart @@ -122,8 +122,16 @@ abstract class FlutterGemmaPlugin extends PlatformInterface { /// /// Uses the active TTS model set via `FlutterGemma.installTts()` / /// `modelManager.setActiveModel()`. Native-only — throws on web. + /// + /// [language] is TTS-only (Qwen3; ignored by Matcha) — see + /// `RuntimeConfig.language`'s doc. Forwarded into the `RuntimeConfig` the + /// active model's backend builds from. A singleton for the same active + /// model is reused across calls, so switching languages requires + /// explicitly closing the previous [SpeechSynthesizer] first — a new + /// [language] alone does not force a rebuild. Future createTtsModel({ PreferredBackend? preferredBackend, + String? language, }); /// === RAG functionality === diff --git a/packages/flutter_gemma/lib/mobile/flutter_gemma_mobile.dart b/packages/flutter_gemma/lib/mobile/flutter_gemma_mobile.dart index 2b8e0b86..dae45735 100644 --- a/packages/flutter_gemma/lib/mobile/flutter_gemma_mobile.dart +++ b/packages/flutter_gemma/lib/mobile/flutter_gemma_mobile.dart @@ -38,6 +38,17 @@ part '../core/model_management/utils/file_system_manager.dart'; part '../core/model_management/utils/resume_checker.dart'; part '../core/model_management/managers/mobile_model_manager.dart'; +/// Normalizes a `createTtsModel`/`getActiveTts` `language` argument for the +/// same-model reuse guard's store/compare — defaults `null` to `'english'` +/// (mirrors `LiteRtTtsBackend.createModel`'s `config.language ?? 'english'`) +/// and lowercases so `null`, `'english'`, and `'English'` all compare equal +/// (they build the SAME synthesizer either way — `Qwen3Prompt.build` already +/// lowercases the language it's given). Without this, storing/comparing the +/// raw argument tripped a spurious `StateError` for effectively-identical +/// requests (e.g. `getActiveTts()` then `getActiveTts(language: 'english')`). +String _normalizeTtsLanguage(String? language) => + (language ?? 'english').toLowerCase(); + class FlutterGemmaMobile extends FlutterGemmaPlugin { Completer? _initCompleter; InferenceModel? _initializedModel; @@ -59,6 +70,14 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { SpeechSynthesizer? _initializedTtsModel; TtsModelSpec? _lastActiveTtsSpec; // Track which spec was used to create _initializedTtsModel + // The `language` the active singleton was built with, NORMALIZED + // ([_normalizeTtsLanguage] — defaulted + lowercased) so a same-effective- + // language request compared raw-to-raw (e.g. null vs. 'english', or + // 'English' vs. 'english') doesn't spuriously trip the guard below. + // Reusing the singleton for a genuinely DIFFERENT language would silently + // emit wrong-language audio with no error — see the same-model branch in + // createTtsModel below. + String? _lastActiveTtsLanguage; // Made public for example app integration late final MobileModelManager _unifiedManager = MobileModelManager(); @@ -581,6 +600,7 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { @override Future createTtsModel({ PreferredBackend? preferredBackend, + String? language, }) async { final manager = _unifiedManager; final activeModel = manager.activeTtsModel; @@ -609,9 +629,25 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { _initTtsCompleter = null; _initializedTtsModel = null; _lastActiveTtsSpec = null; + _lastActiveTtsLanguage = null; await old?.close(); + } else if (_normalizeTtsLanguage(language) != _lastActiveTtsLanguage) { + // Same model, but a DIFFERENT language was requested — reusing the + // singleton here would silently emit WRONG-LANGUAGE audio with no + // error. Fail loud instead of reusing: the caller must close() the + // existing synthesizer first (tts_screen.dart already does this on + // every model/language switch — see LiteRtSpeechSynthesizer/ + // getActiveTts's docs). Both + // sides are normalized ([_normalizeTtsLanguage]) so this only fires + // for a GENUINELY different language, not e.g. null vs. 'english' + // or 'English' vs. 'english'. + throw StateError( + 'Active TTS synthesizer was created for language ' + "'$_lastActiveTtsLanguage'; call close() before requesting " + "'${_normalizeTtsLanguage(language)}'.", + ); } else { - // Same model - return existing singleton + // Same model, same language - return existing singleton gemmaLog( 'ℹ️ Reusing existing TTS model instance for ${activeModel.name}', ); @@ -643,6 +679,7 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { .first, // representative; TTS backend uses artifactPaths artifactPaths: filePaths, preferredBackend: preferredBackend, + language: language, ); final backend = TtsRegistry.instance.findFor(activeModel); if (backend == null) { @@ -663,6 +700,7 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { // package-built model fires this via CloseNotifier (addCloseListener). _initializedTtsModel = synth; _lastActiveTtsSpec = activeModel; + _lastActiveTtsLanguage = _normalizeTtsLanguage(language); synth.addCloseListener(() { // Only reset if this close-listener still belongs to the current // singleton — a newer model may already have replaced it (the @@ -671,6 +709,7 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { _initializedTtsModel = null; _initTtsCompleter = null; _lastActiveTtsSpec = null; + _lastActiveTtsLanguage = null; } }); @@ -680,6 +719,7 @@ class FlutterGemmaMobile extends FlutterGemmaPlugin { _initTtsCompleter = null; _initializedTtsModel = null; _lastActiveTtsSpec = null; + _lastActiveTtsLanguage = null; // Complete the completer and return its future (rather than // rethrowing separately) so there is exactly one Future in flight for // this call — a second, unheeded `completer.future` (as a bare diff --git a/packages/flutter_gemma/lib/web/flutter_gemma_web.dart b/packages/flutter_gemma/lib/web/flutter_gemma_web.dart index 02788d04..ff9c8529 100644 --- a/packages/flutter_gemma/lib/web/flutter_gemma_web.dart +++ b/packages/flutter_gemma/lib/web/flutter_gemma_web.dart @@ -394,6 +394,7 @@ class FlutterGemmaWeb extends FlutterGemmaPlugin { @override Future createTtsModel({ PreferredBackend? preferredBackend, + String? language, }) async { throw UnsupportedError( 'On-device TTS is not supported on web (flutter_gemma_speech TTS is native-only).', diff --git a/packages/flutter_gemma/pubspec.yaml b/packages/flutter_gemma/pubspec.yaml index 83ef85fa..df0c6ce4 100644 --- a/packages/flutter_gemma/pubspec.yaml +++ b/packages/flutter_gemma/pubspec.yaml @@ -1,6 +1,6 @@ name: flutter_gemma description: "Run Gemma and other LLMs on-device in Flutter (Android, iOS, Web, Desktop). Multimodal vision/audio, function calling, thinking mode, GPU, embeddings, RAG." -version: 1.5.1 +version: 1.5.2 resolution: workspace homepage: https://fluttergemma.dev repository: https://github.com/DenisovAV/flutter_gemma diff --git a/packages/flutter_gemma/test/core/api/install_identity_namespacing_test.dart b/packages/flutter_gemma/test/core/api/install_identity_namespacing_test.dart index 8c1a1723..d3cd6208 100644 --- a/packages/flutter_gemma/test/core/api/install_identity_namespacing_test.dart +++ b/packages/flutter_gemma/test/core/api/install_identity_namespacing_test.dart @@ -216,62 +216,121 @@ void main() { ); }); - group('Task 6: migration fallback-probe (unique-basename files only)', () { - test( - 'an old flat TTS bundle file is adopted in place — no re-download', - () async { - final fixtureDownload = _FixtureDownloadService(_fakeCompanionBytes); - await ServiceRegistry.initialize(downloadService: fixtureDownload); + group('Task 6 (superseded 2026-08-04, whole-branch review): TTS install-time ' + 'adoption REMOVED — a not-yet-installed manifest file is always ' + 'downloaded fresh, never adopted from an on-disk plain file, even when ' + 'its basename is unique within the TTS catalog. Adoption at install time ' + 'could not distinguish "an old flat file this exact model installed ' + "before namespacing shipped\" from \"a same-named file some OTHER " + "model's catalog (e.g. an STT tokenizer.json) happened to leave " + 'behind" — no size/hash check was available to tell them apart. The ' + 'only remaining (safe) adoption path is restore-time, in ' + 'MobileModelManager._migrateLegacyCompanionForRestore, which operates on ' + 'a single KNOWN active model so the ambiguity cannot occur.', () { + test('an old flat TTS bundle file at the pre-refactor path is left ' + 'untouched — install() downloads the namespaced file fresh instead ' + 'of adopting it', () async { + final fixtureDownload = _FixtureDownloadService(_fakeCompanionBytes); + await ServiceRegistry.initialize(downloadService: fixtureDownload); - // Resolve the storage dir via the same FileSystemService the - // builder/adopt probe use rather than hardcoding fakeDocuments.path - // — on desktop hosts (this test typically runs as a native VM test) - // writes land under ApplicationSupport/flutter_gemma/, not - // Documents; on mobile they land directly under Documents. - final storageDir = await ServiceRegistry.instance.fileSystemService - .getModelStorageDirectory(); + // Resolve the storage dir via the same FileSystemService the + // builder uses rather than hardcoding fakeDocuments.path — on + // desktop hosts (this test typically runs as a native VM test) + // writes land under ApplicationSupport/flutter_gemma/, not + // Documents; on mobile they land directly under Documents. + final storageDir = await ServiceRegistry.instance.fileSystemService + .getModelStorageDirectory(); - // Pre-seed ONE bundle member at its OLD, pre-refactor flat path — - // simulating a matcha install from before this refactor shipped. - final oldPath = path.join(storageDir, 'matcha_textenc_fp16.tflite'); - await File(oldPath).writeAsBytes([7, 7, 7, 7]); + // Pre-seed ONE bundle member at its OLD, pre-refactor flat path — + // simulating a matcha install from before the namespacing refactor + // shipped. Its basename is unique within the TTS catalog, so + // pre-fix this WOULD have been adopted. + final oldPath = path.join(storageDir, 'matcha_textenc_fp16.tflite'); + await File(oldPath).writeAsBytes([7, 7, 7, 7]); - await FlutterGemma.installTts() - .fromNetwork('https://example.com/matcha/') - .ofType(TtsModelType.matcha) - .install(); + await FlutterGemma.installTts() + .fromNetwork('https://example.com/matcha/') + .ofType(TtsModelType.matcha) + .install(); - // Adopted in place: old path gone, new namespaced path holds the - // ORIGINAL bytes (proves it was renamed, not re-downloaded — a - // re-download would have overwritten it with _fakeCompanionBytes). - expect(await File(oldPath).exists(), isFalse); - final newPath = path.join( - storageDir, - 'matcha__matcha_textenc_fp16.tflite', - ); - expect(await File(newPath).readAsBytes(), [7, 7, 7, 7]); - expect( - fixtureDownload.requestedTargetPaths.contains(newPath), - isFalse, - reason: 'the adopted file must not have been (re-)downloaded', - ); + // NOT adopted: the old flat file is left exactly as it was... + expect(await File(oldPath).readAsBytes(), [7, 7, 7, 7]); + // ...and the namespaced file was downloaded fresh (fixture bytes), + // not renamed from the old path (which would carry [7,7,7,7]). + final newPath = path.join( + storageDir, + 'matcha__matcha_textenc_fp16.tflite', + ); + expect(await File(newPath).readAsBytes(), _fakeCompanionBytes); + expect( + fixtureDownload.requestedTargetPaths.contains(newPath), + isTrue, + reason: + 'a not-yet-installed bundle file must always be ' + 'downloaded, never adopted', + ); - // Every OTHER bundle member (no old file seeded) was downloaded - // normally under its namespaced name. - final otherPath = path.join(storageDir, 'matcha__config.json'); - expect( - fixtureDownload.requestedTargetPaths.contains(otherPath), - isTrue, - ); + // Every OTHER bundle member (no old file seeded) was also + // downloaded normally under its namespaced name. + final otherPath = path.join(storageDir, 'matcha__config.json'); + expect(fixtureDownload.requestedTargetPaths.contains(otherPath), isTrue); - final repository = ServiceRegistry.instance.modelRepository; - expect( - await repository.isInstalled('matcha__matcha_textenc_fp16.tflite'), - isTrue, - ); - }, - ); + final repository = ServiceRegistry.instance.modelRepository; + expect( + await repository.isInstalled('matcha__matcha_textenc_fp16.tflite'), + isTrue, + ); + }); + + test('a FOREIGN plain file left by another catalog (e.g. an STT ' + 'tokenizer.json) is NEVER adopted into a Qwen3 TTS install — the ' + 'exact collision this fix closes', () async { + final fixtureDownload = _FixtureDownloadService(_fakeCompanionBytes); + await ServiceRegistry.initialize(downloadService: fixtureDownload); + + final storageDir = await ServiceRegistry.instance.fileSystemService + .getModelStorageDirectory(); + + // Pre-seed a plain `tokenizer.json` at the real storage location — + // standing in for a leftover Moonshine/Whisper/Parakeet STT + // tokenizer. Distinct byte content from the qwen3 fixture bytes so + // adoption-vs-download is unambiguous. + final foreignPath = path.join(storageDir, 'tokenizer.json'); + await File(foreignPath).writeAsBytes([1, 2, 3, 4, 5]); + + final installation = await FlutterGemma.installTts() + .fromNetwork( + 'https://huggingface.co/litert-community/' + 'Qwen3-TTS-12Hz-0.6B-Base/resolve/main/', + ) + .ofType(TtsModelType.qwen3) + .install(); + + // The foreign file is completely untouched. + expect(await File(foreignPath).readAsBytes(), [1, 2, 3, 4, 5]); + + // qwen3's own tokenizer.json was downloaded fresh under ITS + // namespaced identity, carrying the fixture bytes — not the + // foreign file's bytes. + final qwen3TokenizerPath = path.join(storageDir, 'qwen3__tokenizer.json'); + expect(await File(qwen3TokenizerPath).readAsBytes(), _fakeCompanionBytes); + expect( + fixtureDownload.requestedTargetPaths.contains(qwen3TokenizerPath), + isTrue, + ); + + final repository = ServiceRegistry.instance.modelRepository; + expect(await repository.isInstalled('qwen3__tokenizer.json'), isTrue); + // The plain foreign key was never claimed by this install. + expect(await repository.isInstalled('tokenizer.json'), isFalse); + + // The installed identity is qwen3's, not the foreign file's. + expect(installation.spec.ttsModelType, TtsModelType.qwen3); + }); + }); + group('Task 6b: STT install-time collision guard (unaffected by the TTS ' + 'adoption removal — STT install() never adopted at all)', () { test( 'a colliding companion (tokenizer) NEVER triggers migration — it always ' 'installs fresh under the namespaced key (the mis-adoption guard)', diff --git a/packages/flutter_gemma/test/core/tts_installation_builder_test.dart b/packages/flutter_gemma/test/core/tts_installation_builder_test.dart index ca0abc9b..a1e71557 100644 --- a/packages/flutter_gemma/test/core/tts_installation_builder_test.dart +++ b/packages/flutter_gemma/test/core/tts_installation_builder_test.dart @@ -1,10 +1,42 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_gemma/core/di/service_registry.dart'; +import 'package:flutter_gemma/core/services/download_service.dart'; import 'package:flutter_gemma/flutter_gemma.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; import 'package:shared_preferences/shared_preferences.dart'; void main() { TestWidgetsFlutterBinding.ensureInitialized(); - setUp(() => SharedPreferences.setMockInitialValues({})); + + late Directory fakeDocuments; + late Directory fakeAppSupport; + + setUp(() async { + SharedPreferences.setMockInitialValues({}); + fakeDocuments = await Directory.systemTemp.createTemp( + 'flutter_gemma_docs_', + ); + fakeAppSupport = await Directory.systemTemp.createTemp( + 'flutter_gemma_appsupport_', + ); + PathProviderPlatform.instance = _FixedPathProviderPlatform( + documentsPath: fakeDocuments.path, + appSupportPath: fakeAppSupport.path, + ); + ServiceRegistry.reset(); + }); + + tearDown(() async { + ServiceRegistry.reset(); + for (final dir in [fakeDocuments, fakeAppSupport]) { + if (await dir.exists()) { + await dir.delete(recursive: true); + } + } + }); test('TtsModelType is exported from the public barrel', () { // Compile-level check that users can reference the type + install API. @@ -25,4 +57,180 @@ void main() { test('getActiveTts throws with no active model', () { expect(FlutterGemma.getActiveTts(), throwsA(isA())); }); + + test( + 'qwen3: install() wires the 4 embedding tables + demo voice through ' + 'urlSuffixFor (the real joinUrl path, not a re-derivation) — their ' + 'NetworkSource URL carries the HF tables/voices subdirectory, but the ' + 'installed identity (TtsBundleFile.prefsKey, via TtsModelSpec.files) ' + 'stays the PLAIN basename. Exercises TtsInstallationBuilder.install() ' + 'end to end, so reverting the joinUrl/urlSuffixFor wiring in ' + 'tts_installation_builder.dart fails this test (unlike the ' + 'tts_model_spec_test.dart round-trip test, which builds sourceFor ' + 'itself and would keep passing even if the builder wiring regressed).', + () async { + final fixtureDownload = _RecordingDownloadService( + Uint8List.fromList(List.filled(2048, 7)), + ); + await ServiceRegistry.initialize(downloadService: fixtureDownload); + + final installation = await FlutterGemma.installTts() + .fromNetwork( + 'https://huggingface.co/litert-community/' + 'Qwen3-TTS-12Hz-0.6B-Base/resolve/main/', + ) + .ofType(TtsModelType.qwen3) + .install(); + + final spec = installation.spec; + expect(spec.sources.length, 9); + expect(spec.files.length, 9); + + // Map prefsKey (the installed identity / artifactPaths key) -> the + // NetworkSource actually built by install()'s sourceFor/joinUrl. + final sourceByKey = { + for (var i = 0; i < spec.files.length; i++) + spec.files[i].prefsKey: spec.sources[i], + }; + + String urlOf(String prefsKey) { + final source = sourceByKey[prefsKey]; + expect(source, isNotNull, reason: 'no bundle file for "$prefsKey"'); + expect(source, isA()); + return (source as NetworkSource).url; + } + + // Fetch URL carries the HF subdirectory... + expect( + urlOf('text_embedding_fp16.npy'), + endsWith('/tables/text_embedding_fp16.npy'), + ); + expect( + urlOf('text_projection_fp32.npz'), + endsWith('/tables/text_projection_fp32.npz'), + ); + expect( + urlOf('codec_embedding_fp32.npy'), + endsWith('/tables/codec_embedding_fp32.npy'), + ); + expect( + urlOf('mtp_embeddings_fp16.npy'), + endsWith('/tables/mtp_embeddings_fp16.npy'), + ); + expect(urlOf('demo_speaker.npy'), endsWith('/voices/demo_speaker.npy')); + + // ...but the installed identity (the map's KEYS, i.e. every + // TtsBundleFile.prefsKey) is the plain basename — no leaked + // tables/voices subdirectory anywhere in artifactPaths. + for (final key in sourceByKey.keys) { + expect( + key.contains('/'), + isFalse, + reason: '"$key" must be a plain basename (no subdirectory)', + ); + } + expect(sourceByKey.containsKey('text_embedding_fp16.npy'), isTrue); + expect(sourceByKey.containsKey('demo_speaker.npy'), isTrue); + + // Top-level members (no HF subdirectory) fetch from the plain name. + expect(urlOf('talker_int4.tflite'), endsWith('/talker_int4.tflite')); + expect(urlOf('tokenizer.json'), endsWith('/tokenizer.json')); + + // Every requested download went through the real URL built above — + // proves the recorded urls above are what install() actually fetched + // from, not just what the returned spec claims. + expect( + fixtureDownload.requestedUrls.any( + (u) => u.endsWith('/tables/text_embedding_fp16.npy'), + ), + isTrue, + ); + expect( + fixtureDownload.requestedUrls.any( + (u) => u.endsWith('/voices/demo_speaker.npy'), + ), + isTrue, + ); + }, + ); + + test( + 'matcha spot-check: install() keeps identity URLs for its top-level-only ' + 'manifest (no subdirectory, urlSuffixFor is a no-op for non-qwen3 types)', + () async { + final fixtureDownload = _RecordingDownloadService( + Uint8List.fromList(List.filled(2048, 3)), + ); + await ServiceRegistry.initialize(downloadService: fixtureDownload); + + final installation = await FlutterGemma.installTts() + .fromNetwork('https://example.com/matcha/') + .ofType(TtsModelType.matcha) + .install(); + + final spec = installation.spec; + final sourceByKey = { + for (var i = 0; i < spec.files.length; i++) + spec.files[i].prefsKey: spec.sources[i], + }; + final configUrl = (sourceByKey['config.json']! as NetworkSource).url; + expect(configUrl, 'https://example.com/matcha/config.json'); + }, + ); +} + +/// PathProviderPlatform stub mirroring +/// test/core/api/install_identity_namespacing_test.dart's fixture. +class _FixedPathProviderPlatform extends PathProviderPlatform { + final String documentsPath; + final String appSupportPath; + + _FixedPathProviderPlatform({ + required this.documentsPath, + required this.appSupportPath, + }); + + @override + Future getApplicationDocumentsPath() async => documentsPath; + + @override + Future getApplicationSupportPath() async => appSupportPath; + + @override + Future getTemporaryPath() async => Directory.systemTemp.path; +} + +/// A real (non-mock) DownloadService fake that writes [bytes] to whatever +/// targetPath it's asked to download to, instead of making a real HTTP +/// request — and records every requested URL so a test can assert exactly +/// what install() fetched from, not just what the returned spec claims. +class _RecordingDownloadService implements DownloadService { + final Uint8List bytes; + final List requestedUrls = []; + _RecordingDownloadService(this.bytes); + + @override + Future download( + String url, + String targetPath, { + String? token, + CancelToken? cancelToken, + }) async { + requestedUrls.add(url); + await File(targetPath).writeAsBytes(bytes); + } + + @override + Stream downloadWithProgress( + String url, + String targetPath, { + String? token, + int maxRetries = 10, + CancelToken? cancelToken, + bool? foreground, + }) async* { + requestedUrls.add(url); + await File(targetPath).writeAsBytes(bytes); + yield 100; + } } diff --git a/packages/flutter_gemma/test/core/tts_model_spec_test.dart b/packages/flutter_gemma/test/core/tts_model_spec_test.dart index 2ac700ee..d4c86c07 100644 --- a/packages/flutter_gemma/test/core/tts_model_spec_test.dart +++ b/packages/flutter_gemma/test/core/tts_model_spec_test.dart @@ -17,6 +17,28 @@ void main() { expect(m.length, 8); }); + test('every wired TtsModelType manifest (matcha, qwen3 — those whose ' + '.manifest does not throw) contains only plain basenames: a "/" would ' + 'make MobileModelManager._restoreActiveTtsModel derive a different ' + 'on-disk path than a fresh install did, silently breaking active-model ' + 'restore after relaunch', () { + for (final type in TtsModelType.values) { + final List manifest; + try { + manifest = type.manifest; + } on UnimplementedError { + continue; // Unwired family (kokoro/supertonic) — nothing to check. + } + for (final fn in manifest) { + expect( + fn.contains('/'), + isFalse, + reason: '$type manifest entry "$fn" must be a plain basename', + ); + } + } + }); + test('unwired families throw UnimplementedError from manifest', () { expect( () => TtsModelType.kokoro.manifest, @@ -28,6 +50,130 @@ void main() { ); }); + test('qwen3 manifest lists its 9 runtime files (plain basenames — no ' + 'config.json, no tables/voices subdirectory in the manifest itself; ' + 'see urlSuffixFor for the network-fetch mapping)', () { + final m = TtsModelType.qwen3.manifest; + expect(m, contains('talker_int4.tflite')); + expect(m, contains('mtp_fp32.tflite')); + expect(m, contains('codec_decoder_fp32.tflite')); + expect(m, contains('tokenizer.json')); + expect(m, contains('text_embedding_fp16.npy')); + expect(m, contains('text_projection_fp32.npz')); + expect(m, contains('codec_embedding_fp32.npy')); + expect(m, contains('mtp_embeddings_fp16.npy')); + expect(m, contains('demo_speaker.npy')); + expect(m.length, 9); + expect(m, isNot(contains('config.json'))); + for (final fn in m) { + expect(fn.contains('/'), isFalse, reason: '$fn must be a plain basename'); + } + }); + + test( + 'qwen3 urlSuffixFor maps the 4 embedding tables + demo voice to their ' + 'HF tables/voices subdirectory; the other 4 members map to themselves', + () { + expect( + TtsModelType.qwen3.urlSuffixFor('text_embedding_fp16.npy'), + 'tables/text_embedding_fp16.npy', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('text_projection_fp32.npz'), + 'tables/text_projection_fp32.npz', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('codec_embedding_fp32.npy'), + 'tables/codec_embedding_fp32.npy', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('mtp_embeddings_fp16.npy'), + 'tables/mtp_embeddings_fp16.npy', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('demo_speaker.npy'), + 'voices/demo_speaker.npy', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('talker_int4.tflite'), + 'talker_int4.tflite', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('mtp_fp32.tflite'), + 'mtp_fp32.tflite', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('codec_decoder_fp32.tflite'), + 'codec_decoder_fp32.tflite', + ); + expect( + TtsModelType.qwen3.urlSuffixFor('tokenizer.json'), + 'tokenizer.json', + ); + }, + ); + + test('urlSuffixFor is the identity mapping for every other TtsModelType ' + '(matcha has no subdirectories)', () { + for (final fn in TtsModelType.matcha.manifest) { + expect(TtsModelType.matcha.urlSuffixFor(fn), fn); + } + }); + + test('flat<->url round trip (I5): building a qwen3 spec with sourceFor ' + 'wired through urlSuffixFor (as TtsInstallationBuilder does) produces ' + 'TtsBundleFiles whose prefsKey set is EXACTLY the manifest — the ' + "tables/voices subdirectory in the fetch URL never leaks into the " + 'installed identity / artifactPaths key Qwen3TtsCore.load reads', () { + final spec = TtsModelSpec.fromManifest( + name: 'qwen3', + ttsModelType: TtsModelType.qwen3, + sourceFor: (fn) => ModelSource.network( + 'https://huggingface.co/litert-community/' + 'Qwen3-TTS-12Hz-0.6B-Base/resolve/main/' + '${TtsModelType.qwen3.urlSuffixFor(fn)}', + ), + ); + final prefsKeys = spec.files.map((f) => f.prefsKey).toSet(); + expect(prefsKeys, TtsModelType.qwen3.manifest.toSet()); + expect(spec.files.length, 9); + + // The 4 table files + demo voice namespace to a flat qwen3__ + // filename — no embedded subdirectory made it through. + final tablesFile = spec.files.firstWhere( + (f) => f.prefsKey == 'text_embedding_fp16.npy', + ); + expect(tablesFile.filename, 'qwen3__text_embedding_fp16.npy'); + final voiceFile = spec.files.firstWhere( + (f) => f.prefsKey == 'demo_speaker.npy', + ); + expect(voiceFile.filename, 'qwen3__demo_speaker.npy'); + }); + + test('qwen3 minimumSizeBytes: .npy/.npz bundle members (incl. the small ' + 'demo_speaker.npy voice) get the 1 KB floor; .tflite graphs keep the ' + 'validator default (null)', () { + final spec = TtsModelSpec.fromManifest( + name: 'qwen3', + ttsModelType: TtsModelType.qwen3, + sourceFor: (fn) => ModelSource.network( + 'https://x/${TtsModelType.qwen3.urlSuffixFor(fn)}', + ), + ); + final byName = {for (final f in spec.files) f.prefsKey: f}; + + expect(byName['text_embedding_fp16.npy']!.minimumSizeBytes, 1024); + expect(byName['text_projection_fp32.npz']!.minimumSizeBytes, 1024); + expect(byName['codec_embedding_fp32.npy']!.minimumSizeBytes, 1024); + expect(byName['mtp_embeddings_fp16.npy']!.minimumSizeBytes, 1024); + expect(byName['demo_speaker.npy']!.minimumSizeBytes, 1024); + expect(byName['tokenizer.json']!.minimumSizeBytes, 1024); + + expect(byName['talker_int4.tflite']!.minimumSizeBytes, isNull); + expect(byName['mtp_fp32.tflite']!.minimumSizeBytes, isNull); + expect(byName['codec_decoder_fp32.tflite']!.minimumSizeBytes, isNull); + }); + test( 'fromManifest builds one source+file per manifest entry; type is tts', () { @@ -115,6 +261,37 @@ void main() { expect(configFile.prefsKey, 'config.json'); }); + test('qwen3 restore-safe: MobileModelManager._restoreActiveTtsModel derives ' + 'namespaced paths DIRECTLY from the raw manifest string (no URL ' + 'involved) — this must land on the SAME namespaced filename a fresh ' + 'network install produces via urlSuffixFor, for every one of the 4 ' + 'table + 1 voice member whose fetch URL has a tables/voices ' + 'subdirectory the plain manifest entry does not', () { + final installed = TtsModelSpec.fromManifest( + name: 'qwen3', + ttsModelType: TtsModelType.qwen3, + sourceFor: (fn) => ModelSource.network( + 'https://x/${TtsModelType.qwen3.urlSuffixFor(fn)}', + ), + ); + for (final fn in TtsModelType.qwen3.manifest) { + // Mirrors _restoreActiveTtsModel's `FileNameUtils.namespaced(modelId, + // fn)` — computed from the raw manifest entry, independent of any URL. + final restoreDerived = FileNameUtils.namespaced('qwen3', fn); + final installDerived = installed.files + .firstWhere((f) => f.prefsKey == fn) + .filename; + expect( + restoreDerived, + installDerived, + reason: + 'restore would look for "$restoreDerived" but install wrote ' + '"$installDerived" — active TTS model restore would silently ' + 'fail to find "$fn" after relaunch', + ); + } + }); + test('two different ttsModelTypes namespace the same generic basename ' 'distinctly (the latent config.json/emb.bin collision)', () { final matcha = TtsModelSpec.fromManifest( diff --git a/packages/flutter_gemma/test/mobile/tts_language_singleton_test.dart b/packages/flutter_gemma/test/mobile/tts_language_singleton_test.dart new file mode 100644 index 00000000..dadac1da --- /dev/null +++ b/packages/flutter_gemma/test/mobile/tts_language_singleton_test.dart @@ -0,0 +1,258 @@ +// Task 5.4 review, Important #1: the TTS singleton cache +// (FlutterGemmaMobile.createTtsModel) used to key its "reuse the existing +// synthesizer" branch ONLY on the active model's name, not on the requested +// `language`. So a second `getActiveTts(language: 'french')` for the SAME +// active qwen3 model silently returned the FIRST call's (e.g. English) +// synthesizer — wrong-language audio through the primary public API, with +// no error. This is a real end-to-end test against the ACTUAL +// FlutterGemmaMobile/FlutterGemma facade (not a hand-rolled simulation of +// the completer logic) — it uses the exact `_FixtureDownloadService` + +// `_FixedPathProviderPlatform` fixture pattern already established in +// test/core/api/install_identity_namespacing_test.dart to get a REAL +// install() + getActiveTts() flow working without a device or network. +// +// Run: flutter test test/mobile/tts_language_singleton_test.dart + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; +import 'package:shared_preferences/shared_preferences.dart'; + +import 'package:flutter_gemma/core/di/service_registry.dart'; +import 'package:flutter_gemma/core/lifecycle/close_notifier.dart'; +import 'package:flutter_gemma/core/registry/runtime_config.dart'; +import 'package:flutter_gemma/core/registry/tts_backend_provider.dart'; +import 'package:flutter_gemma/core/registry/tts_registry.dart'; +import 'package:flutter_gemma/core/services/download_service.dart'; +import 'package:flutter_gemma/flutter_gemma.dart'; + +// FileSourceHandler enforces a minimum size per extension — the qwen3 +// manifest's 9 files clear both the 1KB (json/npy/npz) and 1MB (tflite) +// floors with this single fixture, mirroring the matcha fixture in +// install_identity_namespacing_test.dart. +final _fakeBundleBytes = Uint8List(1024 * 1024 + 16); + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late Directory fakeDocuments; + late Directory fakeAppSupport; + + setUp(() async { + fakeDocuments = await Directory.systemTemp.createTemp( + 'flutter_gemma_docs_', + ); + fakeAppSupport = await Directory.systemTemp.createTemp( + 'flutter_gemma_appsupport_', + ); + PathProviderPlatform.instance = _FixedPathProviderPlatform( + documentsPath: fakeDocuments.path, + appSupportPath: fakeAppSupport.path, + ); + SharedPreferences.setMockInitialValues({}); + ServiceRegistry.reset(); + TtsRegistry.instance.reset(); + }); + + tearDown(() async { + ServiceRegistry.reset(); + TtsRegistry.instance.reset(); + for (final dir in [fakeDocuments, fakeAppSupport]) { + if (await dir.exists()) { + await dir.delete(recursive: true); + } + } + }); + + test('getActiveTts throws StateError (not a silent wrong-language reuse) ' + 'when a second call requests a different language for the same active ' + 'model, without closing the first synthesizer', () async { + final fixtureDownload = _FixtureDownloadService(_fakeBundleBytes); + await ServiceRegistry.initialize(downloadService: fixtureDownload); + final fakeBackend = _FakeTtsBackend(); + TtsRegistry.instance.registerAll([fakeBackend]); + + await FlutterGemma.installTts() + .fromNetwork('https://example.com/qwen3/') + .ofType(TtsModelType.qwen3) + .install(); + + final synth1 = await FlutterGemma.getActiveTts(language: 'english'); + expect(fakeBackend.lastConfig?.language, 'english'); + expect(fakeBackend.createModelCallCount, 1); + + // Same language again -> reuses the singleton (no new backend call, + // no error) — the guard must not be overly strict. + final synth1Again = await FlutterGemma.getActiveTts(language: 'english'); + expect(identical(synth1Again, synth1), isTrue); + expect(fakeBackend.createModelCallCount, 1); + + // Different language, SAME active model, WITHOUT closing first -> + // must fail loud. Before the fix this returned synth1 (English) + // silently; after the fix it throws instead. + await expectLater( + FlutterGemma.getActiveTts(language: 'french'), + throwsA(isA()), + ); + // The rejected request must not have built a second synthesizer — + // this is a fail-FAST guard, not a fallback that still constructs + // something wrong. + expect(fakeBackend.createModelCallCount, 1); + + // After the caller closes the existing synthesizer, a new language is + // allowed and actually rebuilds against the new language. + await synth1.close(); + final synth2 = await FlutterGemma.getActiveTts(language: 'french'); + expect(fakeBackend.lastConfig?.language, 'french'); + expect(fakeBackend.createModelCallCount, 2); + expect(identical(synth2, synth1), isFalse); + + // Close synth2 too — FlutterGemmaMobile's TTS singleton fields + // (_lastActiveTtsSpec/_lastActiveTtsLanguage/_initializedTtsModel) live + // on the plugin instance, not this test's fixtures, so a leftover + // active synthesizer here would leak into the NEXT test in this file. + await synth2.close(); + }); + + test( + 'getActiveTts treats null, "english", and "English" as the SAME ' + 'effective language (default + case normalized before the reuse-guard ' + 'store/compare) — a genuinely different language still fails loud', + () async { + final fixtureDownload = _FixtureDownloadService(_fakeBundleBytes); + await ServiceRegistry.initialize(downloadService: fixtureDownload); + final fakeBackend = _FakeTtsBackend(); + TtsRegistry.instance.registerAll([fakeBackend]); + + await FlutterGemma.installTts() + .fromNetwork('https://example.com/qwen3/') + .ofType(TtsModelType.qwen3) + .install(); + + // No `language:` argument at all -> the backend defaults it to + // 'english' (LiteRtTtsBackend.createModel), and the guard normalizes + // the same way when storing. + final synth1 = await FlutterGemma.getActiveTts(); + expect(fakeBackend.createModelCallCount, 1); + + // Same effective language, explicit lowercase -> reuse, no new backend + // call, no StateError. Before normalization this compared `null != + // 'english'` and threw spuriously. + final synth1Again = await FlutterGemma.getActiveTts(language: 'english'); + expect(identical(synth1Again, synth1), isTrue); + expect(fakeBackend.createModelCallCount, 1); + + // Same effective language, different case -> still reused. Before + // normalization this compared `'english' != 'English'` and threw. + final synth1Cased = await FlutterGemma.getActiveTts(language: 'English'); + expect(identical(synth1Cased, synth1), isTrue); + expect(fakeBackend.createModelCallCount, 1); + + // A GENUINELY different language must still fail loud, without closing + // the existing synthesizer first. + await expectLater( + FlutterGemma.getActiveTts(language: 'german'), + throwsA(isA()), + ); + expect(fakeBackend.createModelCallCount, 1); + + // Close the still-active synthesizer so FlutterGemmaMobile's TTS + // singleton fields don't leak into a later test in this file (see the + // matching cleanup at the end of the test above). + await synth1.close(); + }, + ); +} + +class _FakeTtsBackend implements TtsBackendProvider { + RuntimeConfig? lastConfig; + int createModelCallCount = 0; + + @override + String get name => 'FakeTTS'; + + @override + int get priority => 0; + + @override + bool canHandle(TtsModelSpec spec) => true; + + @override + Future createModel( + TtsModelSpec spec, + RuntimeConfig config, + ) async { + createModelCallCount++; + lastConfig = config; + return _FakeSpeechSynthesizer(); + } +} + +class _FakeSpeechSynthesizer extends SpeechSynthesizer with CloseNotifier { + @override + int get sampleRate => 24000; + + @override + Future synthesize(String text) async => Uint8List(4); + + @override + Future close() async { + fireCloseListeners(); + } +} + +/// PathProviderPlatform stub that returns fixed, distinct paths for +/// Documents and ApplicationSupport (mirrors +/// test/core/api/install_identity_namespacing_test.dart). +class _FixedPathProviderPlatform extends PathProviderPlatform { + final String documentsPath; + final String appSupportPath; + + _FixedPathProviderPlatform({ + required this.documentsPath, + required this.appSupportPath, + }); + + @override + Future getApplicationDocumentsPath() async => documentsPath; + + @override + Future getApplicationSupportPath() async => appSupportPath; + + @override + Future getTemporaryPath() async => Directory.systemTemp.path; +} + +/// A real (non-mock) DownloadService fake that writes [bytes] to whatever +/// targetPath it's asked to download to, instead of making a real HTTP +/// request (mirrors install_identity_namespacing_test.dart's +/// _FixtureDownloadService). +class _FixtureDownloadService implements DownloadService { + final Uint8List bytes; + _FixtureDownloadService(this.bytes); + + @override + Future download( + String url, + String targetPath, { + String? token, + CancelToken? cancelToken, + }) async { + await File(targetPath).writeAsBytes(bytes); + } + + @override + Stream downloadWithProgress( + String url, + String targetPath, { + String? token, + int maxRetries = 10, + CancelToken? cancelToken, + bool? foreground, + }) async* { + await File(targetPath).writeAsBytes(bytes); + yield 100; + } +} diff --git a/packages/flutter_gemma_speech/CHANGELOG.md b/packages/flutter_gemma_speech/CHANGELOG.md index 847e81f7..bdde8746 100644 --- a/packages/flutter_gemma_speech/CHANGELOG.md +++ b/packages/flutter_gemma_speech/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.4.1 +- Add Qwen3-TTS (multilingual AR codec-LM, 11 languages) — 2nd selectable TTS family. + ## 0.4.0 - feat: on-device Whisper-tiny STT (English-only) — log-mel frontend + GPT-2 BPE decode. - feat: on-device Parakeet-CTC STT (desktop) — NeMo mel frontend + greedy CTC decode. diff --git a/packages/flutter_gemma_speech/lib/flutter_gemma_speech.dart b/packages/flutter_gemma_speech/lib/flutter_gemma_speech.dart index 05a825c6..79985ded 100644 --- a/packages/flutter_gemma_speech/lib/flutter_gemma_speech.dart +++ b/packages/flutter_gemma_speech/lib/flutter_gemma_speech.dart @@ -25,3 +25,10 @@ export 'src/litert_tts_backend_stub.dart' export 'src/voice/voice_event.dart'; export 'src/voice/voice_responder.dart'; export 'src/voice/voice_session.dart'; + +// Qwen3-TTS supported-language list — dependency-free, safe on every +// platform. The single source of truth for both the create-time +// language validator (`LiteRtSpeechSynthesizer.create`) and a UI language +// picker; see `qwen3_languages.dart`'s header for why it's a separate file +// from `qwen3_prompt.dart` (which is native-only, via `dart:io`). +export 'src/qwen3/qwen3_languages.dart'; diff --git a/packages/flutter_gemma_speech/lib/src/litert/litert_graph.dart b/packages/flutter_gemma_speech/lib/src/litert/litert_graph.dart new file mode 100644 index 00000000..fecefdd1 --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/litert/litert_graph.dart @@ -0,0 +1,371 @@ +// Shared LiteRT graph-loading + forward-pass helpers, factored out of +// Matcha's `TtsCore` so the same load/run machinery can drive Qwen3's +// talker/codec graphs too. This is a pure, behavior-preserving extraction: +// `TtsCore` now delegates to [loadLiteRtGraph]/[runLiteRtGraph] instead of +// its own former private `_loadGraph`/`_runGraph` — the tensor-buffer +// create -> run -> lock(Read) -> copy-through-locked-ptr -> unlock -> +// destroy sequence and the partial-failure cleanup are unchanged, only +// renamed and moved. See `tts_core.dart`'s file header for the original +// provenance (a verbatim port of `matcha_synth.dart`'s `runGraph`, FFI-styled +// after `stt_core.dart`). +// +// Generalization over the Matcha-only original: Qwen3's talker `decode` +// signature mixes 58 F32 inputs with 1 I32 input (`input_pos`), and the +// codec graph's input is I32 token ids — so a F32-only `_runGraph` can't be +// reused verbatim. [GraphInput] tags each input with its dtype so +// [runLiteRtGraph] creates the right kind of tensor buffer per input; +// outputs stay F32-only (Qwen3's KV/logits/codec-PCM outputs and Matcha's +// mel/PCM outputs are all f32). The hardcoded signature index `0` is now a +// parameter ([runLiteRtGraph]'s `signatureIndex`) — Matcha call sites keep +// passing `0`. + +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; +// Public, native-only bindings library (not the package barrel) — see the +// equivalent comment in `tts_core.dart`/`stt_core.dart` for why this import +// (not the `if (dart.library.ffi)` barrel) is correct in a native-only file. +import 'package:flutter_gemma_litertlm/litert_bindings.dart'; + +/// One compiled graph's handles (model/options/compiledModel), freed +/// together by the owner's dispose or by [loadLiteRtGraph]'s caller's +/// partial-failure cleanup. Was `TtsCore`'s private `_LoadedGraph`. +class LoadedGraph { + LoadedGraph(this.model, this.options, this.compiledModel); + final LiteRtModel model; + final LiteRtOptions options; + final LiteRtCompiledModel compiledModel; +} + +/// Loads + compiles one `.tflite` graph at [path], mirroring the +/// model/options/compiled sequence `SttCore.load` runs for its single model +/// — generalized so callers (e.g. `TtsCore.load`, the upcoming Qwen3 loader) +/// can call it once per graph. On any failure partway through, frees +/// whatever handles it already created for THIS graph before rethrowing; +/// the caller is responsible for freeing any earlier, already-succeeded +/// graphs + the shared environment. Was `TtsCore`'s private `_loadGraph`. +LoadedGraph loadLiteRtGraph( + LiteRtBindings bindings, + LiteRtEnvironment environment, + String path, + int accelerator, +) { + LiteRtModel? model; + LiteRtOptions? options; + LiteRtCompiledModel? compiled; + try { + final pathC = path.toNativeUtf8(); + final modelPtr = calloc(); + try { + bindings + .createModelFromFile(environment, pathC, modelPtr) + .check('LiteRtCreateModelFromFile($path)'); + } finally { + calloc.free(pathC); + } + model = modelPtr.value; + calloc.free(modelPtr); + + final optsPtr = calloc(); + bindings.createOptions(optsPtr).check('LiteRtCreateOptions($path)'); + options = optsPtr.value; + calloc.free(optsPtr); + bindings + .setOptionsHardwareAccelerators(options, accelerator) + .check('LiteRtSetOptionsHardwareAccelerators($path)'); + + final compiledPtr = calloc(); + bindings + .createCompiledModel(environment, model, options, compiledPtr) + .check('LiteRtCreateCompiledModel($path)'); + compiled = compiledPtr.value; + calloc.free(compiledPtr); + + return LoadedGraph(model, options, compiled); + } catch (_) { + if (compiled != null) bindings.destroyCompiledModel(compiled); + if (options != null) bindings.destroyOptions(options); + if (model != null) bindings.destroyModel(model); + rethrow; + } +} + +/// One tensor buffer's raw host allocation + its LiteRT wrapper handle, +/// freed together in [runLiteRtGraph]'s `finally`. Mirrors +/// `matcha_synth.dart`'s `_TensorHandle`. Was `TtsCore`'s private +/// `_TensorHandle`. +class TensorHandle { + TensorHandle(this.raw, this.buffer); + final Pointer raw; + final LiteRtTensorBuffer buffer; +} + +/// Tagged graph input: pairs a shape with its dtype-specific payload so +/// [runLiteRtGraph] can create the right kind of tensor buffer per input. +/// Matcha's graphs are F32-only ([F32Input] everywhere); Qwen3's talker +/// `decode` signature mixes 58 F32 inputs with 1 I32 input (`input_pos`), +/// and the codec graph's input is I32 token ids. +sealed class GraphInput { + List get shape; +} + +class F32Input extends GraphInput { + F32Input(this.shape, this.data); + @override + final List shape; + final Float32List data; +} + +class I32Input extends GraphInput { + I32Input(this.shape, this.data); + @override + final List shape; + final Int32List data; +} + +/// Generic multi-input/multi-output forward pass over one compiled [graph] +/// at [signatureIndex]. Mirrors `SttCore._encode`/`_decodeLoop`'s tensor- +/// buffer create -> run -> lock(Read) -> copy-through-locked-ptr -> unlock +/// -> destroy sequence, generalized to N inputs / M outputs and to mixed +/// F32/I32 input dtypes via [GraphInput] — this replaces `TtsCore`'s former +/// private `_runGraph` (F32-only, signature index hardcoded to 0). [inputs] +/// and [outputShapes] must list tensors in the model's declared +/// [signatureIndex] argument order. Outputs are always read back as F32. +List runLiteRtGraph( + LiteRtBindings bindings, + LiteRtCompiledModel graph, + int signatureIndex, + List inputs, + List> outputShapes, +) { + final inHandles = []; + final outHandles = []; + try { + for (final input in inputs) { + inHandles.add(switch (input) { + F32Input(:final shape, :final data) => _createTensorBuffer( + bindings, + shape, + kLiteRtElementTypeFloat32, + data.length, + (alloc, count) { + alloc.aligned.cast().asTypedList(count).setAll(0, data); + }, + ), + I32Input(:final shape, :final data) => _createTensorBuffer( + bindings, + shape, + kLiteRtElementTypeInt32, + data.length, + (alloc, count) { + alloc.aligned.cast().asTypedList(count).setAll(0, data); + }, + ), + }); + } + for (final shape in outputShapes) { + outHandles.add( + _createTensorBuffer( + bindings, + shape, + kLiteRtElementTypeFloat32, + null, + null, + ), + ); + } + + // Combined arrays are built fresh right before the call and freed + // right after, mirroring `SttCore._decodeLoop`'s 3-input decode + // (stt_core.dart:504-527): each buffer was created into its own + // single-slot pointer above; only `.value` is copied in here. + final inPtrs = calloc(inHandles.length); + final outPtrs = calloc(outHandles.length); + try { + for (var i = 0; i < inHandles.length; i++) { + inPtrs[i] = inHandles[i].buffer; + } + for (var i = 0; i < outHandles.length; i++) { + outPtrs[i] = outHandles[i].buffer; + } + bindings + .runCompiledModel( + graph, + signatureIndex, + inHandles.length, + inPtrs, + outHandles.length, + outPtrs, + ) + .check('LiteRtRunCompiledModel(signature=$signatureIndex)'); + } finally { + calloc.free(inPtrs); + calloc.free(outPtrs); + } + + // Lock(Read) triggers the device->host sync on GPU/NPU; read each + // output THROUGH the locked pointer into an owned Float32List copy + // before unlocking/destroying the buffer — mirrors + // `SttCore._encode`'s locked-pointer copy (stt_core.dart:322-351). + final results = []; + for (var i = 0; i < outHandles.length; i++) { + final count = outputShapes[i].fold(1, (a, b) => a * b); + final lockedPtr = calloc>(); + try { + bindings + .lockTensorBuffer( + outHandles[i].buffer, + lockedPtr, + kLiteRtTensorBufferLockModeRead, + ) + .check('LiteRtLockTensorBuffer(out $i)'); + final locked = lockedPtr.value.cast(); + results.add(Float32List.fromList(locked.asTypedList(count))); + bindings + .unlockTensorBuffer(outHandles[i].buffer) + .check('LiteRtUnlockTensorBuffer(out $i)'); + } finally { + calloc.free(lockedPtr); + } + } + return results; + } finally { + for (final h in inHandles) { + bindings.destroyTensorBuffer(h.buffer); + calloc.free(h.raw); + } + for (final h in outHandles) { + bindings.destroyTensorBuffer(h.buffer); + calloc.free(h.raw); + } + } +} + +/// Creates a F32 tensor buffer of [shape], optionally seeded with [data]. +/// [env] is accepted for signature symmetry with [loadLiteRtGraph] and to +/// leave room for a future host-buffer variant that needs the environment; +/// `LiteRtCreateTensorBufferFromHostMemory` itself does not take one, so it +/// is unused today. Was `TtsCore`'s private `_createF32TensorBuffer`. +TensorHandle createF32TensorBuffer( + LiteRtBindings bindings, + LiteRtEnvironment env, + List shape, + Float32List? data, +) { + return _createTensorBuffer( + bindings, + shape, + kLiteRtElementTypeFloat32, + data?.length, + data == null + ? null + : (alloc, count) { + alloc.aligned.cast().asTypedList(count).setAll(0, data); + }, + ); +} + +/// Creates an I32 tensor buffer of [shape], optionally seeded with [data]. +/// Same as [createF32TensorBuffer] but `elementType = +/// kLiteRtElementTypeInt32` over an [Int32List] payload — used for the +/// talker's `input_pos` and the codec's token-id inputs. [env] is unused +/// today; see [createF32TensorBuffer]'s doc. +TensorHandle createI32TensorBuffer( + LiteRtBindings bindings, + LiteRtEnvironment env, + List shape, + Int32List? data, +) { + return _createTensorBuffer( + bindings, + shape, + kLiteRtElementTypeInt32, + data?.length, + data == null + ? null + : (alloc, count) { + alloc.aligned.cast().asTypedList(count).setAll(0, data); + }, + ); +} + +/// Guards every [GraphInput]/[createF32TensorBuffer]/[createI32TensorBuffer] +/// payload: a `data` shorter than [count] (= product of the tensor's shape) +/// would otherwise be silently zero-padded by `setAll` into a +/// wrong-but-not-obviously-wrong tensor, and a longer one would throw a +/// confusing `RangeError` from `setAll` itself instead of naming the actual +/// mismatch. Called from [_createTensorBuffer] itself — BEFORE any native +/// allocation (`allocAligned`/the `type` view) — so a mismatch fails loud +/// with zero native allocation instead of leaking the aligned buffer + type +/// struct that a throw from inside the old write-callback used to leave +/// behind (both were allocated before the `try`/`finally` that frees them). +/// THROWS (not `assert` — asserts are stripped in release builds and this +/// must fail loud in production too). +void _checkPayloadLength(int dataLength, int count) { + if (dataLength != count) { + throw ArgumentError( + 'GraphInput payload length $dataLength != product(shape) $count', + ); + } +} + +/// Shared allocate + (optionally) write + wrap-as-tensor-buffer sequence +/// behind [createF32TensorBuffer]/[createI32TensorBuffer]/[runLiteRtGraph]'s +/// per-input dispatch. [elementType] is one of `kLiteRtElementTypeFloat32`/ +/// `kLiteRtElementTypeInt32` (both 4-byte elements, so `bytes = count * 4` +/// unconditionally — matches the original F32-only arithmetic byte-for-byte +/// when [elementType] is float32). [dataLength], when non-null (a payload is +/// being written), is validated against `count` (= product of [shape]) via +/// [_checkPayloadLength] BEFORE any native allocation — [writeData], if +/// non-null, is called once with the fresh aligned allocation to copy the +/// caller's payload in AFTER that check passes, so a length mismatch throws +/// with zero native allocation instead of leaking the aligned buffer + the +/// `type` view (both used to be allocated before the length was checked, +/// which sat inside [writeData] itself, past the point either could still +/// be freed on that throw). +TensorHandle _createTensorBuffer( + LiteRtBindings bindings, + List shape, + int elementType, + int? dataLength, + void Function(AlignedAlloc alloc, int count)? writeData, +) { + final count = shape.fold(1, (a, b) => a * b); + if (dataLength != null) { + _checkPayloadLength(dataLength, count); + } + final bytes = count * 4; + final type = LiteRtRankedTensorTypeView.calloc() + ..elementType = elementType + ..rank = shape.length; + for (var i = 0; i < shape.length; i++) { + type.setDimension(i, shape[i]); + } + final alloc = allocAligned(bytes); + writeData?.call(alloc, count); + final bufPtr = calloc(); + // On success alloc.raw is handed off to the caller inside the returned + // TensorHandle — runLiteRtGraph frees it once the tensor buffer is + // destroyed. On ANY throw before the return it must be freed here or it + // leaks native heap permanently (~bytes per failed call) — mirrors + // SttCore._encode's `returning` flag (stt_core.dart:288). + var returning = false; + try { + bindings + .createTensorBufferFromHostMemory( + type.pointer, + alloc.aligned.cast(), + bytes, + nullptr, + bufPtr, + ) + .check('CreateTensorBufferFromHostMemory(shape=$shape)'); + returning = true; + return TensorHandle(alloc.raw, bufPtr.value); + } finally { + calloc.free(bufPtr); + type.free(); + if (!returning) calloc.free(alloc.raw); + } +} diff --git a/packages/flutter_gemma_speech/lib/src/litert/litert_speech_synthesizer.dart b/packages/flutter_gemma_speech/lib/src/litert/litert_speech_synthesizer.dart index 36dd1319..673c3899 100644 --- a/packages/flutter_gemma_speech/lib/src/litert/litert_speech_synthesizer.dart +++ b/packages/flutter_gemma_speech/lib/src/litert/litert_speech_synthesizer.dart @@ -1,13 +1,13 @@ // `SpeechSynthesizer` facade over a background isolate, mirroring -// `litert_speech_recognizer.dart` (Task 2.5 direct STT→TTS mirror). The -// blocking LiteRT text-frontend + CFM/vocoder forward passes run on a -// dedicated [TtsWorker] isolate — spawned once, reused for every call — so -// the UI isolate stays free. +// `litert_speech_recognizer.dart` (a direct STT→TTS mirror). The blocking +// LiteRT text-frontend + CFM/vocoder forward passes run on a dedicated +// [TtsWorker] isolate — spawned once, reused for every call — so the UI +// isolate stays free. // // The native code lives in `tts_core.dart` (driven inside the worker -// isolate, Task 2.3) plus `tts_text_frontend.dart` (Task 2.2); this file is -// the public, async, main-isolate API generic over [TtsModelProfile] — -// matcha/kokoro/supertonic select a profile, not a synthesizer subclass. +// isolate) plus `tts_text_frontend.dart`; this file is the public, async, +// main-isolate API generic over [TtsModelProfile] — matcha/kokoro/supertonic +// select a profile, not a synthesizer subclass. import 'dart:typed_data'; @@ -18,6 +18,8 @@ import 'package:flutter_gemma/flutter_gemma_interface.dart' show SpeechSynthesizer; import '../model/tts_model_profile.dart'; +import '../qwen3/qwen3_languages.dart' + show assertQwen3LanguageSupported, normalizeQwen3Language; import 'tts_worker.dart'; /// Signature for the `onClose` callback. Same name Flutter uses. @@ -41,19 +43,47 @@ class LiteRtSpeechSynthesizer extends SpeechSynthesizer with CloseNotifier { /// [artifactPaths] maps each of [profile]'s bundle filenames (config, /// dict, embedding, and the `.tflite` graphs) to their resolved on-disk /// paths. [preferredBackend] selects the LiteRT hardware accelerator - /// (defaults to CPU). + /// (defaults to CPU). [language] is Qwen3-only (ignored by Matcha, which + /// has no language parameter — its locale comes from + /// [TtsModelProfile.locale] instead); defaults to `'english'`. For a + /// [profile] whose [TtsModelProfile.pipeline] is + /// [TtsPipelineKind.qwen3ArCodec], [language] is validated against + /// `qwen3SupportedLanguages` (`qwen3_languages.dart`) and this throws + /// [ArgumentError] for an unknown value BEFORE spawning the worker — + /// fail-fast, ahead of the ~1.9 GB model load. Once validated, + /// [language] is normalized to lowercase (`normalizeQwen3Language`) so the + /// whole downstream pipeline (the worker, `Qwen3TtsCore.synthesizePcm16`, + /// `Qwen3Prompt.build`'s case-SENSITIVE `'auto'` comparison) sees one + /// consistent value — `'Auto'`/`'AUTO'` are accepted here exactly like + /// `'auto'`, not just at validation time. + /// + /// [voice] is a forward-compat speaker x-vector override (`[1024]`, + /// Qwen3-only): when non-null it replaces the bundle's single demo voice + /// (`voices/demo_speaker.npy`) for every `synthesize` call on the returned + /// instance. v1 ships exactly one voice and does not surface a voice + /// picker anywhere — this param exists purely so a future multi-voice + /// release doesn't need a breaking signature change. /// /// Caller owns the returned instance and must call [close] when done. static Future create({ required TtsModelProfile profile, required Map artifactPaths, PreferredBackend? preferredBackend, + String language = 'english', + Float32List? voice, VoidCallback? onClose, }) async { + var effectiveLanguage = language; + if (profile.pipeline == TtsPipelineKind.qwen3ArCodec) { + assertQwen3LanguageSupported(language); + effectiveLanguage = normalizeQwen3Language(language); + } final worker = await TtsWorker.spawn( profile: profile, artifactPaths: artifactPaths, backend: preferredBackend, + language: effectiveLanguage, + voice: voice, ); return LiteRtSpeechSynthesizer._(worker, onClose ?? () {}); } diff --git a/packages/flutter_gemma_speech/lib/src/litert/tts_core.dart b/packages/flutter_gemma_speech/lib/src/litert/tts_core.dart index 5c6ec669..3d8f4c16 100644 --- a/packages/flutter_gemma_speech/lib/src/litert/tts_core.dart +++ b/packages/flutter_gemma_speech/lib/src/litert/tts_core.dart @@ -23,19 +23,26 @@ // outputs (the multi-input pattern mirrors `SttCore._decodeLoop`'s 3-input // decode at stt_core.dart:504-527: each buffer is created into its own // single-slot pointer, then `.value` is copied into a fresh combined array -// right before the run call). +// right before the run call). The load/run/tensor-buffer machinery itself +// now lives in `litert_graph.dart` (`loadLiteRtGraph`/`runLiteRtGraph`), +// generalized to mixed F32/I32 inputs for the upcoming Qwen3 talker/codec +// graphs; this file delegates to it and stays behavior-identical. // // Generic over [TtsModelProfile] — only `TtsPipelineKind.matchaCfm` is -// implemented; kokoro/supertonic profiles are documented follow-ons. The -// `dp_g2p` graph IS loaded here (4th compiled graph) and exposed via +// implemented. `qwen3ArCodec` profiles are dispatched to `Qwen3TtsCore` by +// the worker and fail-loud if they ever reach [load]'s guard instead; +// kokoro/supertonic profiles are documented follow-ons with no wired +// pipeline kind at all. The `dp_g2p` graph IS loaded here (4th compiled +// graph) and exposed via // `TtsCore.neuralG2p` — it's the neural OOV fallback used when a word is // missing from the dictionary; the dictionary-only path stays the golden -// path via `TtsTextFrontend` (Task 2.2). +// path via `TtsTextFrontend`. // // Determinism: the Euler CFM decoder's initial noise is drawn from a FIXED // seed ([ttsCfmSeed]) via Box-Muller ([nextGaussian]), so [TtsCore.synthesize] -// is byte-reproducible run-to-run — the Phase-3 golden depends on this. Do -// NOT change the seed or the Box-Muller formula without regenerating it. +// is byte-reproducible run-to-run — the golden in `tts_core_test.dart` +// depends on this. Do NOT change the seed or the Box-Muller formula +// without regenerating it. // // Leak-safety: buffer create/lock/read/unlock/destroy mirrors // `litert_embedding_core.dart`'s forward-pass pattern; `load`'s partial- @@ -63,10 +70,11 @@ import 'package:flutter_gemma_litertlm/litert_bindings.dart'; import '../model/tts_model_profile.dart'; import '../tts/neural_g2p_decode.dart'; import '../tts/tts_frontend_input.dart'; +import 'litert_graph.dart'; /// The Euler CFM decoder's fixed Gaussian-noise seed. Fixed (not /// time-derived) so [TtsCore.synthesize] is byte-reproducible run-to-run — -/// the Phase-3 golden depends on this. Verified value from +/// the golden in `tts_core_test.dart` depends on this. Verified value from /// `matcha_synth.dart:565`. const int ttsCfmSeed = 1234; @@ -135,78 +143,6 @@ String _artifactPath(Map artifactPaths, String file) { return path; } -/// One compiled Matcha graph's handles (model/options/compiledModel), freed -/// together by [TtsCore.dispose] or by [TtsCore.load]'s partial-failure -/// cleanup. -class _LoadedGraph { - _LoadedGraph(this.model, this.options, this.compiledModel); - final LiteRtModel model; - final LiteRtOptions options; - final LiteRtCompiledModel compiledModel; -} - -/// Loads + compiles one `.tflite` graph at [path], mirroring the -/// model/options/compiled sequence `SttCore.load` runs for its single model -/// — generalized so [TtsCore.load] can call it 4 times (text-encoder, -/// decoder, vocoder, dp_g2p). On any failure partway through, frees whatever -/// handles it already created for THIS graph before rethrowing; the caller -/// is responsible for freeing any earlier, already-succeeded graphs + the -/// shared environment. -_LoadedGraph _loadGraph( - LiteRtBindings bindings, - LiteRtEnvironment environment, - String path, - int accelerator, -) { - LiteRtModel? model; - LiteRtOptions? options; - LiteRtCompiledModel? compiled; - try { - final pathC = path.toNativeUtf8(); - final modelPtr = calloc(); - try { - bindings - .createModelFromFile(environment, pathC, modelPtr) - .check('LiteRtCreateModelFromFile($path)'); - } finally { - calloc.free(pathC); - } - model = modelPtr.value; - calloc.free(modelPtr); - - final optsPtr = calloc(); - bindings.createOptions(optsPtr).check('LiteRtCreateOptions($path)'); - options = optsPtr.value; - calloc.free(optsPtr); - bindings - .setOptionsHardwareAccelerators(options, accelerator) - .check('LiteRtSetOptionsHardwareAccelerators($path)'); - - final compiledPtr = calloc(); - bindings - .createCompiledModel(environment, model, options, compiledPtr) - .check('LiteRtCreateCompiledModel($path)'); - compiled = compiledPtr.value; - calloc.free(compiledPtr); - - return _LoadedGraph(model, options, compiled); - } catch (_) { - if (compiled != null) bindings.destroyCompiledModel(compiled); - if (options != null) bindings.destroyOptions(options); - if (model != null) bindings.destroyModel(model); - rethrow; - } -} - -/// One tensor buffer's raw host allocation + its LiteRT wrapper handle, -/// freed together in `TtsCore._runGraph`'s `finally`. Mirrors -/// `matcha_synth.dart`'s `_TensorHandle`. -class _TensorHandle { - _TensorHandle(this.raw, this.buffer); - final Pointer raw; - final LiteRtTensorBuffer buffer; -} - /// Synchronous native TTS core. NOT safe to share across isolates — the FFI /// handles it holds are owned by the isolate that called [load]. class TtsCore { @@ -238,10 +174,10 @@ class TtsCore { final LiteRtBindings _bindings; final LiteRtEnvironment _environment; - final _LoadedGraph _textEncoder; - final _LoadedGraph _decoder; - final _LoadedGraph _vocoder; - final _LoadedGraph _dpG2p; + final LoadedGraph _textEncoder; + final LoadedGraph _decoder; + final LoadedGraph _vocoder; + final LoadedGraph _dpG2p; final int _nFeats; final int _nChannels; @@ -277,8 +213,9 @@ class TtsCore { }) async { if (profile.pipeline != TtsPipelineKind.matchaCfm) { throw UnimplementedError( - 'TtsCore: only TtsPipelineKind.matchaCfm is implemented ' - '(kokoro/supertonic are follow-ons; see the design spec).', + 'TtsCore: only TtsPipelineKind.matchaCfm is implemented here ' + '(qwen3ArCodec is handled by Qwen3TtsCore, not the Matcha TtsCore ' + 'path; kokoro/supertonic have no wired pipeline kind yet).', ); } @@ -306,7 +243,7 @@ class TtsCore { // LiteRT native heap is process-global and is NOT reclaimed by the // isolate dying. Mirrors `SttCore.load`, generalized to 4 graphs. LiteRtEnvironment? environment; - final loadedGraphs = <_LoadedGraph>[]; + final loadedGraphs = []; try { final envPtr = calloc(); bindings @@ -315,7 +252,7 @@ class TtsCore { environment = envPtr.value; calloc.free(envPtr); - final textEncoder = _loadGraph( + final textEncoder = loadLiteRtGraph( bindings, environment, _artifactPath(artifactPaths, profile.textEncoderFile), @@ -323,7 +260,7 @@ class TtsCore { ); loadedGraphs.add(textEncoder); - final decoder = _loadGraph( + final decoder = loadLiteRtGraph( bindings, environment, _artifactPath(artifactPaths, profile.decoderFile), @@ -331,7 +268,7 @@ class TtsCore { ); loadedGraphs.add(decoder); - final vocoder = _loadGraph( + final vocoder = loadLiteRtGraph( bindings, environment, _artifactPath(artifactPaths, profile.vocoderFile), @@ -339,7 +276,7 @@ class TtsCore { ); loadedGraphs.add(vocoder); - final dpG2p = _loadGraph( + final dpG2p = loadLiteRtGraph( bindings, environment, _artifactPath(artifactPaths, profile.g2pFile), @@ -409,8 +346,8 @@ class TtsCore { /// phoneme string. Synchronous (no file I/O; `g2p_meta.json` was already /// parsed in [load]), so it's safe to call per-word from the frontend (the /// `NeuralG2pResolver` adapter is `core.neuralG2p`). Framing/decoding is - /// [encodeG2pInput]/[decodeG2pOutput] (Task 6) — this method only runs the - /// native forward pass and picks the per-position argmax over the + /// [encodeG2pInput]/[decodeG2pOutput] — this method only runs the native + /// forward pass and picks the per-position argmax over the /// `n_phonemes`-wide logits. String neuralG2p(String word) { if (_disposed) { @@ -424,10 +361,12 @@ class TtsCore { end: _g2pEnd, maxT: _g2pMaxT, ); - final logits = _runGraph( + final logits = runLiteRtGraph( + _bindings, _dpG2p.compiledModel, + 0, [ - ([1, _g2pMaxT], x), + F32Input([1, _g2pMaxT], x), ], [ [1, _g2pMaxT, _g2pNPhonemes], @@ -453,10 +392,10 @@ class TtsCore { /// into MAX_MEL-sized windows -> per-window N-step Euler CFM decoder -> /// mel denorm -> HiFi-GAN vocoder -> 16-bit PCM, concatenated across /// windows. Verbatim port of `matcha_synth.dart:518-635`'s math for the - /// single-window case; every graph run goes through [_runGraph] instead of - /// that script's raw-`dlopen` `runGraph`. Returns 16-bit little-endian - /// mono PCM with NO WAV header — the example's `pcmToWav` adds a header in - /// Phase 3. + /// single-window case; every graph run goes through [runLiteRtGraph] + /// instead of that script's raw-`dlopen` `runGraph`. Returns 16-bit + /// little-endian mono PCM with NO WAV header — the example's `pcmToWav` + /// adds a header. /// /// When the predicted duration fits one window (`rawYlen <= _maxMel`, the /// common case), this is byte-identical to the pre-chunking implementation @@ -480,11 +419,13 @@ class TtsCore { // --- text encoder: symbolEmbeddings[1,maxText,nChannels] + // textMask[1,1,maxText] -> mu[1,nFeats,maxText], logw[1,1,maxText] --- - final teOut = _runGraph( + final teOut = runLiteRtGraph( + _bindings, _textEncoder.compiledModel, + 0, [ - ([1, _maxText, _nChannels], input.symbolEmbeddings), - ([1, 1, _maxText], input.textMask), + F32Input([1, _maxText, _nChannels], input.symbolEmbeddings), + F32Input([1, 1, _maxText], input.textMask), ], [ [1, _nFeats, _maxText], @@ -605,13 +546,15 @@ class TtsCore { } for (var k = 0; k < _nTimesteps; k++) { final tEmb = tSin(k / _nTimesteps); - final decOut = _runGraph( + final decOut = runLiteRtGraph( + _bindings, _decoder.compiledModel, + 0, [ - ([1, _nFeats, _maxMel], x), - ([1, _nFeats, _maxMel], muY), - ([1, 160], tEmb), - ([1, 1, _maxMel], ymask), + F32Input([1, _nFeats, _maxMel], x), + F32Input([1, _nFeats, _maxMel], muY), + F32Input([1, 160], tEmb), + F32Input([1, 1, _maxMel], ymask), ], [ [1, _nFeats, _maxMel], @@ -632,10 +575,12 @@ class TtsCore { } // vocoder -> wav -> 16-bit LE PCM - final vocOut = _runGraph( + final vocOut = runLiteRtGraph( + _bindings, _vocoder.compiledModel, + 0, [ - ([1, _nFeats, _maxMel], mel), + F32Input([1, _nFeats, _maxMel], mel), ], [ [1, 1, _maxMel * _hop], @@ -693,134 +638,6 @@ class TtsCore { return windows; } - /// Generic multi-input/multi-output f32 forward pass over one compiled - /// [graph], signature index 0. Mirrors `SttCore._encode`/`_decodeLoop`'s - /// tensor-buffer create -> run -> lock(Read) -> copy-through-locked-ptr -> - /// unlock -> destroy sequence, generalized to N inputs / M outputs — this - /// replaces `matcha_synth.dart`'s raw-`dlopen` `runGraph`. [inputs] and - /// [outputShapes] must list tensors in the model's declared signature-0 - /// argument order. - List _runGraph( - LiteRtCompiledModel graph, - List<(List shape, Float32List data)> inputs, - List> outputShapes, - ) { - final inHandles = <_TensorHandle>[]; - final outHandles = <_TensorHandle>[]; - try { - for (final (shape, data) in inputs) { - inHandles.add(_createF32TensorBuffer(shape, data)); - } - for (final shape in outputShapes) { - outHandles.add(_createF32TensorBuffer(shape, null)); - } - - // Combined arrays are built fresh right before the call and freed - // right after, mirroring `SttCore._decodeLoop`'s 3-input decode - // (stt_core.dart:504-527): each buffer was created into its own - // single-slot pointer above; only `.value` is copied in here. - final inPtrs = calloc(inHandles.length); - final outPtrs = calloc(outHandles.length); - try { - for (var i = 0; i < inHandles.length; i++) { - inPtrs[i] = inHandles[i].buffer; - } - for (var i = 0; i < outHandles.length; i++) { - outPtrs[i] = outHandles[i].buffer; - } - _bindings - .runCompiledModel( - graph, - 0, - inHandles.length, - inPtrs, - outHandles.length, - outPtrs, - ) - .check('LiteRtRunCompiledModel(tts)'); - } finally { - calloc.free(inPtrs); - calloc.free(outPtrs); - } - - // Lock(Read) triggers the device->host sync on GPU/NPU; read each - // output THROUGH the locked pointer into an owned Float32List copy - // before unlocking/destroying the buffer — mirrors - // `SttCore._encode`'s locked-pointer copy (stt_core.dart:322-351). - final results = []; - for (var i = 0; i < outHandles.length; i++) { - final count = outputShapes[i].fold(1, (a, b) => a * b); - final lockedPtr = calloc>(); - try { - _bindings - .lockTensorBuffer( - outHandles[i].buffer, - lockedPtr, - kLiteRtTensorBufferLockModeRead, - ) - .check('LiteRtLockTensorBuffer(tts out $i)'); - final locked = lockedPtr.value.cast(); - results.add(Float32List.fromList(locked.asTypedList(count))); - _bindings - .unlockTensorBuffer(outHandles[i].buffer) - .check('LiteRtUnlockTensorBuffer(tts out $i)'); - } finally { - calloc.free(lockedPtr); - } - } - return results; - } finally { - for (final h in inHandles) { - _bindings.destroyTensorBuffer(h.buffer); - calloc.free(h.raw); - } - for (final h in outHandles) { - _bindings.destroyTensorBuffer(h.buffer); - calloc.free(h.raw); - } - } - } - - _TensorHandle _createF32TensorBuffer(List shape, Float32List? data) { - final count = shape.fold(1, (a, b) => a * b); - final bytes = count * 4; - final type = LiteRtRankedTensorTypeView.calloc() - ..elementType = kLiteRtElementTypeFloat32 - ..rank = shape.length; - for (var i = 0; i < shape.length; i++) { - type.setDimension(i, shape[i]); - } - final alloc = allocAligned(bytes); - if (data != null) { - final view = alloc.aligned.cast().asTypedList(count); - view.setAll(0, data); - } - final bufPtr = calloc(); - // On success alloc.raw is handed off to the caller inside the returned - // _TensorHandle — _runGraph frees it once the tensor buffer is destroyed - // (tts_core.dart:556-563). On ANY throw before the return it must be - // freed here or it leaks native heap permanently (~bytes per failed - // call) — mirrors SttCore._encode's `returning` flag (stt_core.dart:288). - var returning = false; - try { - _bindings - .createTensorBufferFromHostMemory( - type.pointer, - alloc.aligned.cast(), - bytes, - nullptr, - bufPtr, - ) - .check('CreateTensorBufferFromHostMemory(shape=$shape)'); - returning = true; - return _TensorHandle(alloc.raw, bufPtr.value); - } finally { - calloc.free(bufPtr); - type.free(); - if (!returning) calloc.free(alloc.raw); - } - } - /// Destroys all 4 compiled graphs' handles + the shared environment. /// Mirrors `SttCore.dispose`, x4. void dispose() { diff --git a/packages/flutter_gemma_speech/lib/src/litert/tts_worker.dart b/packages/flutter_gemma_speech/lib/src/litert/tts_worker.dart index 49fd3b95..73d6c04d 100644 --- a/packages/flutter_gemma_speech/lib/src/litert/tts_worker.dart +++ b/packages/flutter_gemma_speech/lib/src/litert/tts_worker.dart @@ -1,12 +1,13 @@ // Long-lived background isolate that owns the entire Matcha-TTS pipeline: -// both the text frontend (dictionary G2P + host embedding gather, Task 2.2) -// and the native LiteRT core (4 compiled graphs + the CFM/vocoder forward -// passes, Task 2.3). The forward passes are blocking synchronous FFI calls; -// running them (and the frontend's dictionary lookups + 275k-entry load) -// here keeps the UI isolate's event loop free. Direct analog of -// `stt_worker.dart` — see that file's header for the "why a long-lived -// worker and not `Isolate.run`" rationale (FFI handles can't cross isolate -// boundaries; the compiled models are expensive to build but cheap to run). +// both the text frontend (`TtsTextFrontend`: dictionary G2P + host embedding +// gather) and the native LiteRT core (`TtsCore`: 4 compiled graphs + the +// CFM/vocoder forward passes). The forward passes are blocking synchronous +// FFI calls; running them (and the frontend's dictionary lookups + +// 275k-entry load) here keeps the UI isolate's event loop free. Direct +// analog of `stt_worker.dart` — see that file's header for the "why a +// long-lived worker and not `Isolate.run`" rationale (FFI handles can't +// cross isolate boundaries; the compiled models are expensive to build but +// cheap to run). // // Unlike `SttWorker` (which owns only `SttCore`, tokenizer included inside // the core), this worker owns TWO objects: `TtsTextFrontend` and `TtsCore`. @@ -30,6 +31,18 @@ // Only sendable values cross the port: file paths + profile + backend // (setup), a `String` (request), and a `Uint8List` of 16-bit PCM samples // (reply). +// +// `_workerEntry` branches on `init.profile.pipeline` into `_runMatchaWorker` +// (the above, unchanged) or `_runQwen3Worker`: Qwen3-TTS is a from-scratch +// autoregressive codec-token LM (`Qwen3TtsCore`) with its own KV cache and +// no CFM step, so it needs neither +// `TtsTextFrontend`/`TtsTextNormalizer` (Matcha's dictionary G2P + host +// embedding gather + clause splitter) nor `TtsCore` (Matcha's native +// core) — it does its own byte-level BPE tokenization +// (`Qwen2BpeEncoder`, inside `Qwen3TtsCore`) and consumes a request's full +// text in ONE AR pass (no clause-splitting, no per-clause CFM seed, no +// inter-clause silence — see `_runQwen3Worker`'s doc for why those three +// Matcha-specific behaviors don't apply here). import 'dart:async'; import 'dart:isolate'; @@ -40,6 +53,8 @@ import 'package:flutter_gemma/core/domain/platform_types.dart' import 'package:flutter_gemma/core/utils/gemma_log.dart'; import '../model/tts_model_profile.dart'; +import '../qwen3/npy_reader.dart'; +import '../qwen3/qwen3_tts_core.dart'; import '../tts/tts_text_frontend.dart'; import '../tts/tts_text_normalizer.dart'; import 'tts_chunk.dart'; @@ -47,7 +62,8 @@ import 'tts_core.dart'; /// Handshake payload the worker sends back once the frontend + native model /// are loaded. Carries [sampleRate] (read from `TtsCore` after load) so the -/// facade (Task 2.5) can expose it synchronously without a round-trip. +/// `LiteRtSpeechSynthesizer` facade can expose it synchronously without a +/// round-trip. class _Ready { _Ready(this.commandPort, this.sampleRate); final SendPort commandPort; @@ -88,6 +104,8 @@ class _WorkerInit { required this.artifactPaths, required this.backend, required this.logLevel, + required this.language, + required this.voice, }); final SendPort replyTo; final TtsModelProfile profile; @@ -98,6 +116,22 @@ class _WorkerInit { /// isolate gets its own copy of the per-isolate top-level (default info), /// so it must be seeded explicitly. final GemmaLogLevel logLevel; + + /// Qwen3-only: the `Qwen3Prompt.languageIds` key (case-insensitive) or + /// `'auto'`, forwarded verbatim to every `Qwen3TtsCore.synthesizePcm16` + /// call. Ignored by [_runMatchaWorker] (Matcha has no language parameter — + /// its locale comes from [TtsModelProfile.locale] instead). Validated by + /// `LiteRtSpeechSynthesizer.create` before the worker is even spawned, so + /// by the time it reaches here it is always a supported value. + final String language; + + /// Qwen3-only: forward-compat speaker x-vector override (`[1024]`); null + /// uses the bundle's single demo voice (`voices/demo_speaker.npy`, read in + /// [_runQwen3Worker]). Ignored by [_runMatchaWorker]. See + /// `LiteRtSpeechSynthesizer.create`'s [voice] doc — v1 does not surface a + /// voice picker anywhere; this only exists so a future multi-voice release + /// doesn't need a breaking signature change. + final Float32List? voice; } /// Main-isolate handle to the TTS worker. Spawns the isolate, performs the @@ -121,10 +155,17 @@ class TtsWorker { Completer? _closeAck; /// Spawn the worker and wait until the frontend + native model are loaded. + /// + /// [language] is Qwen3-only (see [_WorkerInit.language]'s doc); Matcha + /// ignores it. Defaults to `'english'` so every existing (Matcha) caller + /// is unaffected. [voice] is Qwen3-only (see [_WorkerInit.voice]'s doc); + /// null (the default) uses the bundle's demo voice. static Future spawn({ required TtsModelProfile profile, required Map artifactPaths, PreferredBackend? backend, + String language = 'english', + Float32List? voice, }) async { final fromWorker = ReceivePort(); final readyCompleter = Completer<_Ready>(); @@ -163,6 +204,8 @@ class TtsWorker { artifactPaths: artifactPaths, backend: backend, logLevel: gemmaLogLevel, + language: language, + voice: voice, ), // onExit posts `null` to fromWorker so we never wait on a dead isolate. onExit: fromWorker.sendPort, @@ -255,11 +298,26 @@ class TtsWorker { } } -/// Isolate entry point. Loads the frontend + model, then serves requests -/// until _Close. +/// Isolate entry point. Seeds the per-isolate log level, then dispatches to +/// the pipeline-specific arm ([_runMatchaWorker] / [_runQwen3Worker]) — +/// see this file's header for why the two pipelines don't share a request +/// loop. Future _workerEntry(_WorkerInit init) async { // Seed this isolate's per-isolate log level from the main-isolate snapshot. gemmaLogLevel = init.logLevel; + switch (init.profile.pipeline) { + case TtsPipelineKind.matchaCfm: + await _runMatchaWorker(init); + case TtsPipelineKind.qwen3ArCodec: + await _runQwen3Worker(init); + } +} + +/// Matcha-CFM arm of [_workerEntry] — loads `TtsTextFrontend` + `TtsCore` +/// and serves requests via the clause-split / per-clause-CFM-seed / +/// inter-clause-silence loop described in this file's header. Qwen3-TTS +/// dispatches to [_runQwen3Worker] instead. +Future _runMatchaWorker(_WorkerInit init) async { final TtsCore core; final TtsTextFrontend frontend; final TtsTextNormalizer normalizer; @@ -338,3 +396,114 @@ Future _workerEntry(_WorkerInit init) async { init.replyTo.send(const _CloseAck()); } } + +/// Qwen3-TTS arm of [_workerEntry]. Loads `Qwen3TtsCore` (the talker/MTP/ +/// codec graphs + host tables + BPE encoder) and the bundle's one +/// demo-voice x-vector, then serves requests with exactly ONE +/// `core.synthesizePcm16` call per request. +/// +/// Deliberately does NOT reuse [_runMatchaWorker]'s clause-split / CFM-seed +/// / inter-clause-silence loop — none of those three apply here: +/// - No clause-splitting (`TtsTextNormalizer.splitClauses`): that exists +/// only to keep each chunk under the CFM decoder's `MAX_MEL` cap. Qwen3 is +/// autoregressive over its own KV cache and has no such per-call length +/// ceiling — it consumes a request's full text in one AR pass +/// (`Qwen3TtsCore.synthesize`'s `maxFrames`, not a text-length limit). +/// - No CFM seed (`ttsCfmSeed + clauseIndex`): Qwen3 has no CFM step; its +/// only randomness is `pickSampled`'s token sampling (see +/// [Qwen3TtsCore.synthesizePcm16]'s sampling path), seeded once per call +/// via `Qwen3TtsCore.synthesizePcm16`'s own `seed` param (left unset +/// here — see `doSample`'s doc below). +/// - No `concatPcmWithSilence`: there is only ever one segment per request +/// (no per-clause splitting to splice back together). +/// +/// [Qwen3TtsCore.load] uses its default `talkerFileName` +/// (`talker_int4.tflite`, the quantized runtime artifact — NOT the fp32 +/// artifact the golden-gate tests load) — this is the runtime path real +/// users hit, not a correctness gate. +/// +/// `doSample: true` on every call (the runtime default is varied, natural +/// prosody; a fixed `seed` would make every synthesis of the same text +/// sound identical). The fp32-greedy +/// byte-for-byte golden gate lives in `qwen3_synthesize_test.dart` +/// (`Qwen3TtsCore.synthesize` driven directly with `doSample: false`), not +/// here — this worker never runs greedy. +/// +/// v1 ships exactly one voice: the bundle's `demo_speaker.npy` x-vector +/// (the model bundle's asset manifest), read once at load time via +/// [readNpyF32] unless [_WorkerInit.voice] overrides it — no voice PICKER +/// yet (`init.voice` is forward-compat-only; the UI never sets it, see +/// [_WorkerInit.voice]'s doc). `init.language` is the per-call knob this +/// worker threads through to `Qwen3TtsCore.synthesizePcm16`, validated up +/// front by `LiteRtSpeechSynthesizer.create` (`assertQwen3LanguageSupported`), +/// which is also what surfaces a real language picker (via +/// `qwen3SupportedLanguages`); see [_WorkerInit.language]'s doc. +/// +/// Fail-loud (per Global Constraints / the project's no-masking-fallback +/// rule): if [Qwen3TtsCore.load] or the demo-voice read throws, this sends +/// the error back to the main isolate exactly like [_runMatchaWorker]'s +/// load failure does — it NEVER falls back to the Matcha path. +Future _runQwen3Worker(_WorkerInit init) async { + final Qwen3TtsCore core; + final Float32List demoVoice; + // Same "track the core once load succeeds" rationale as + // [_runMatchaWorker]'s `loadedCore` — a fully-loaded native core (3 + // compiled graphs + environment + tables) must not be leaked if the + // demo-voice read fails right after. + Qwen3TtsCore? loadedCore; + try { + core = await Qwen3TtsCore.load( + artifactPaths: init.artifactPaths, + backend: init.backend, + ); + loadedCore = core; + final demoVoicePath = init.artifactPaths['demo_speaker.npy']; + if (demoVoicePath == null) { + throw StateError( + 'TtsWorker: qwen3 bundle is missing "demo_speaker.npy" in ' + 'artifactPaths', + ); + } + // [_WorkerInit.voice] overrides the bundle's demo voice when set + // (forward-compat only — v1's UI never sets it). The manifest presence + // check above stays unconditional even then: the bundle always ships + // demo_speaker.npy in v1, so its absence is still a load-time bug worth + // surfacing regardless of whether an override happens to be in play. + demoVoice = init.voice ?? readNpyF32(demoVoicePath); + } catch (e, st) { + loadedCore?.dispose(); + gemmaLog('[TtsWorker] qwen3 load failed: $e\n$st'); + init.replyTo.send('TTS worker failed to load: $e'); + return; + } + + final commandPort = ReceivePort(); + init.replyTo.send(_Ready(commandPort.sendPort, core.sampleRate)); + + try { + await for (final msg in commandPort) { + if (msg is _SynthRequest) { + try { + final pcm = core.synthesizePcm16( + msg.text, + speaker: demoVoice, + language: init.language, + doSample: true, + ); + init.replyTo.send(_SynthReply(msg.id, pcm, null)); + } catch (e) { + init.replyTo.send(_SynthReply(msg.id, null, e.toString())); + } + } else if (msg is _Close) { + commandPort.close(); + break; + } + } + } finally { + // Qwen3TtsCore.dispose() also frees the tables/tokenizer it owns + // internally (unlike Matcha, where the frontend is separate pure-Dart + // state) — only the core needs disposing here. + core.dispose(); + init.replyTo.send(const _CloseAck()); + } +} diff --git a/packages/flutter_gemma_speech/lib/src/litert_tts_backend.dart b/packages/flutter_gemma_speech/lib/src/litert_tts_backend.dart index 880b91b1..0df0d591 100644 --- a/packages/flutter_gemma_speech/lib/src/litert_tts_backend.dart +++ b/packages/flutter_gemma_speech/lib/src/litert_tts_backend.dart @@ -36,10 +36,19 @@ class LiteRtTtsBackend implements TtsBackendProvider { } // spec.ttsModelType (e.g. TtsModelType.matcha) selects the runtime // profile — this backend never hardcodes a model. + // + // config.language is Qwen3-only — threaded from + // `FlutterGemma.getActiveTts(language: ...)` through `RuntimeConfig`; + // null falls back to `LiteRtSpeechSynthesizer.create`'s own `'english'` + // default. Matcha ignores it (no language parameter). No `voice` here: + // v1 never surfaces a voice picker anywhere in the public API, so this + // backend always takes `LiteRtSpeechSynthesizer.create`'s default + // (the bundle's single demo voice) — see that method's `voice` doc. return LiteRtSpeechSynthesizer.create( profile: TtsModelProfile.forType(spec.ttsModelType), artifactPaths: artifactPaths, preferredBackend: config.preferredBackend, + language: config.language ?? 'english', onClose: () {}, ); } diff --git a/packages/flutter_gemma_speech/lib/src/model/tts_model_profile.dart b/packages/flutter_gemma_speech/lib/src/model/tts_model_profile.dart index 906b59ab..91a77ebc 100644 --- a/packages/flutter_gemma_speech/lib/src/model/tts_model_profile.dart +++ b/packages/flutter_gemma_speech/lib/src/model/tts_model_profile.dart @@ -1,7 +1,7 @@ /// Per-model runtime descriptor for the generic Matcha-CFM TTS pipeline. /// /// This is what makes the TTS pipeline SELECTABLE without per-model classes: -/// `TtsCore` (Task 2.3) is generic over [TtsModelProfile] — matcha/kokoro/ +/// `TtsCore` is generic over [TtsModelProfile] — matcha/kokoro/ /// supertonic are data (a profile + a catalog entry), not separate synthesizer /// classes. Mirrors `SttModelProfile`. Unlike STT, the numeric synthesis /// params (mel dims, CFM steps, sample rate, …) are NOT baked in here — Matcha @@ -20,6 +20,15 @@ enum TtsPipelineKind { /// Glow-TTS length regulator (256→512 frames) → N-step Euler CFM decoder → /// HiFi-GAN vocoder. Numeric params come from the bundle's config.json. matchaCfm, + + /// Qwen3-TTS: autoregressive codec-token LM (talker, prefill+decode over a + /// threaded KV cache) → 15-step MTP residual-codebook inner loop → + /// windowed codec decoder → 24 kHz PCM. Dispatched to `Qwen3TtsCore` + /// (`flutter_gemma_speech/lib/src/qwen3/qwen3_tts_core.dart`) by the + /// background worker — this pipeline kind never reaches the Matcha-only + /// `TtsCore`/`TtsTextFrontend` path, which fail-loud on it instead of + /// silently running Matcha behavior against a Qwen3 bundle. + qwen3ArCodec, } /// How text becomes the model's encoder input. @@ -33,8 +42,8 @@ enum G2pStrategy { dictionary, dictionaryPlusNeural, neuralOnly, none } /// `.tflite` to load for which stage; it does not auto-detect roles from the /// compiled models' tensor layouts. class TtsModelProfile { - /// Matcha-TTS bundle: filenames match `TtsModelType.matcha.manifest` - /// (Task 1.3), here they get roles. + /// Matcha-TTS bundle: filenames match `TtsModelType.matcha.manifest`; + /// here they get roles. const TtsModelProfile.matcha() : pipeline = TtsPipelineKind.matchaCfm, textEncoderFile = 'matcha_textenc_fp16.tflite', @@ -49,6 +58,33 @@ class TtsModelProfile { g2p = G2pStrategy.dictionaryPlusNeural, locale = 'en_us'; + /// Qwen3-TTS bundle: unlike [matcha], `Qwen3TtsCore.load` does NOT read + /// these file-role fields — it hardcodes its own manifest basenames + /// (`talker_int4.tflite`, `mtp_fp32.tflite`, `codec_decoder_fp32.tflite`, + /// `tokenizer.json`, plus the 4 `Qwen3Tables` npy/npz basenames; see + /// `TtsModelTypeManifest.manifest` for `TtsModelType.qwen3` in + /// `flutter_gemma`'s `model_specs.dart`). So for this profile [pipeline] + /// is the ONLY load-bearing field; the file-role fields below are left + /// `''` (non-nullable, so empty stands in for "not used") rather than + /// mapped onto Qwen3 basenames — none of Matcha's roles (a single + /// text-encoder/decoder/vocoder trio) has a clean 1:1 equivalent in the + /// AR-codec pipeline's talker/MTP/codec-decoder split, and inventing one + /// here would be misleading. Qwen3 does its own byte-level BPE + /// tokenization (`Qwen2BpeEncoder` over `tokenizer.json`), not G2P. + const TtsModelProfile.qwen3() + : pipeline = TtsPipelineKind.qwen3ArCodec, + textEncoderFile = '', + decoderFile = '', + vocoderFile = '', + g2pFile = '', + g2pMetaFile = '', + configFile = '', + dictFile = '', + embeddingFile = '', + representation = TextRepresentation.subwordTokens, + g2p = G2pStrategy.none, + locale = 'en_us'; + /// Which end-to-end synthesis pipeline this profile drives. final TtsPipelineKind pipeline; @@ -86,16 +122,17 @@ class TtsModelProfile { /// token models. final G2pStrategy? g2p; - /// Selects the `TtsTextNormalizer` (Task 4); matcha → `'en_us'`. + /// Selects the `TtsTextNormalizer`; matcha → `'en_us'`. final String locale; - /// Resolve the runtime profile for [t]. Only [TtsModelType.matcha] is - /// wired; kokoro/supertonic are follow-ons and throw (fail-loud — never - /// run text through the wrong pipeline). + /// Resolve the runtime profile for [t]. Only [TtsModelType.matcha] and + /// [TtsModelType.qwen3] are wired; kokoro/supertonic are follow-ons and + /// throw (fail-loud — never run text through the wrong pipeline). factory TtsModelProfile.forType(TtsModelType t) => switch (t) { TtsModelType.matcha => const TtsModelProfile.matcha(), + TtsModelType.qwen3 => const TtsModelProfile.qwen3(), _ => throw UnimplementedError( - 'TTS profile for $t is a follow-on (only matcha is wired)', + 'TTS profile for $t is a follow-on (only matcha/qwen3 are wired)', ), }; } diff --git a/packages/flutter_gemma_speech/lib/src/qwen3/npy_reader.dart b/packages/flutter_gemma_speech/lib/src/qwen3/npy_reader.dart new file mode 100644 index 00000000..e6210054 --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/qwen3/npy_reader.dart @@ -0,0 +1,336 @@ +// .npy / .npz reader for the Qwen3-TTS host tables (embedding tables, +// projection matrices, prompt caches) shipped as NumPy arrays. +// +// Supports: +// - whole-array reads of ` shape; + final String dtype; + final int dataOffset; +} + +const int _npyMagicAndVersionLength = 8; // 6-byte magic + major + minor. + +/// Parses the `.npy` v1 header from an already-open [RandomAccessFile]. +/// Leaves the file position at [NpyHeader.dataOffset] on return. +NpyHeader parseNpyHeader(RandomAccessFile f) { + f.setPositionSync(0); + final magicAndVersion = f.readSync(_npyMagicAndVersionLength); + if (magicAndVersion.length != _npyMagicAndVersionLength || + magicAndVersion[0] != 0x93 || + String.fromCharCodes(magicAndVersion.sublist(1, 6)) != 'NUMPY') { + throw const FormatException('not a .npy file (bad magic)'); + } + final major = magicAndVersion[6]; + if (major != 1) { + throw FormatException('unsupported .npy version: $major'); + } + + final headerLenBytes = f.readSync(2); + final headerLen = ByteData.sublistView( + headerLenBytes, + ).getUint16(0, Endian.little); + final headerBytes = f.readSync(headerLen); + final headerStr = ascii.decode(headerBytes); + + final dtype = _extractDictString(headerStr, 'descr'); + final shape = _extractShape(headerStr); + + final dataOffset = _npyMagicAndVersionLength + 2 + headerLen; + f.setPositionSync(dataOffset); + return NpyHeader(shape: shape, dtype: dtype, dataOffset: dataOffset); +} + +String _extractDictString(String header, String key) { + final match = RegExp("'$key'\\s*:\\s*'([^']*)'").firstMatch(header); + if (match == null) { + throw FormatException(".npy header missing '$key': $header"); + } + return match.group(1)!; +} + +List _extractShape(String header) { + final match = RegExp(r"'shape'\s*:\s*\(([^)]*)\)").firstMatch(header); + if (match == null) { + throw FormatException('.npy header missing shape: $header'); + } + final inner = match.group(1)!.trim(); + if (inner.isEmpty) return const []; + return inner + .split(',') + .map((s) => s.trim()) + .where((s) => s.isNotEmpty) + .map(int.parse) + .toList(growable: false); +} + +int _elementCount(List shape) => + shape.isEmpty ? 1 : shape.reduce((a, b) => a * b); + +/// Reads a whole `= rows) { + throw RangeError.range(row, 0, rows - 1, 'row'); + } + final offset = h.dataOffset + row * cols * 2; + f.setPositionSync(offset); + final bytes = f.readSync(cols * 2); + if (bytes.length != cols * 2) { + throw StateError( + 'npyRowF16AsF32: truncated file at row $row (expected ${cols * 2} ' + 'bytes, got ${bytes.length})', + ); + } + final view = ByteData.sublistView(_copyToAlignedBuffer(bytes)); + final out = Float32List(cols); + for (var i = 0; i < cols; i++) { + out[i] = _halfToFloat(view.getUint16(i * 2, Endian.little)); + } + return out; +} + +/// Copies [bytes] into a fresh, 0-offset [Uint8List] so it is safe to wrap +/// with [Float32List.view] / [ByteData.sublistView] regardless of the +/// underlying buffer's alignment or offset (`RandomAccessFile.readSync` +/// results are not guaranteed to start at a buffer offset divisible by 4). +Uint8List _copyToAlignedBuffer(Uint8List bytes) => Uint8List.fromList(bytes); + +/// IEEE-754 binary16 -> IEEE-754 binary64 (Dart `double`) conversion, +/// handling zero, subnormals, normals, infinities, and NaN. +double _halfToFloat(int h) { + final s = (h >> 15) & 1; + final e = (h >> 10) & 0x1f; + final m = h & 0x3ff; + if (e == 0) { + // Zero (m == 0) or subnormal: value = m * 2^-24. + return (s == 1 ? -1 : 1) * m * 5.9604644775390625e-8; + } + if (e == 31) { + if (m == 0) { + return s == 1 ? double.negativeInfinity : double.infinity; + } + return double.nan; + } + return (s == 1 ? -1 : 1) * (1 + m / 1024.0) * _pow2(e - 15); +} + +double _pow2(int exp) => exp >= 0 ? (1 << exp).toDouble() : 1.0 / (1 << -exp); + +// --- Minimal ZIP reader for .npz (STORED entries only) ---------------- + +/// Reads every member of a `.npz` archive (a ZIP of `.npy` files, as +/// produced by `numpy.savez`) and up-casts each to a flat [Float32List], +/// keyed by member name without the `.npy` suffix. +/// +/// Only STORED (uncompressed) entries are supported — `numpy.savez` (as +/// opposed to `savez_compressed`) always writes STORED entries. A DEFLATE +/// member throws a [StateError] rather than being silently misread. +Map readNpzF32(String path) { + final bytes = File(path).readAsBytesSync(); + final entries = _readZipStoredEntries(bytes); + final out = {}; + for (final entry in entries.entries) { + var name = entry.key; + if (name.endsWith('.npy')) { + name = name.substring(0, name.length - '.npy'.length); + } + out[name] = _npyBytesToF32(entry.value, name); + } + return out; +} + +Float32List _npyBytesToF32(Uint8List npyBytes, String memberName) { + final byteData = ByteData.sublistView(npyBytes); + if (npyBytes.length < 10 || + byteData.getUint8(0) != 0x93 || + String.fromCharCodes(npyBytes.sublist(1, 6)) != 'NUMPY') { + throw FormatException('npz member $memberName: not a .npy file'); + } + final headerLen = byteData.getUint16(8, Endian.little); + final headerStr = ascii.decode(npyBytes.sublist(10, 10 + headerLen)); + final dtype = _extractDictString(headerStr, 'descr'); + final shape = _extractShape(headerStr); + final dataOffset = 10 + headerLen; + final count = _elementCount(shape); + + if (dtype == ' _readZipStoredEntries(Uint8List bytes) { + const eocdSignature = 0x06054b50; + const centralDirSignature = 0x02014b50; + const localHeaderSignature = 0x04034b50; + const storedMethod = 0; + + // The EOCD record is at least 22 bytes and sits at the end of the file, + // optionally preceded by a variable-length comment — scan backward for + // its signature (comment is capped at 65535 bytes by the ZIP spec, so this + // terminates quickly). + var eocdOffset = -1; + final maxCommentLen = bytes.length < 65557 ? bytes.length - 22 : 65535; + for (var back = 0; back <= maxCommentLen; back++) { + final pos = bytes.length - 22 - back; + if (pos < 0) break; + final data = ByteData.sublistView(bytes, pos, pos + 4); + if (data.getUint32(0, Endian.little) == eocdSignature) { + eocdOffset = pos; + break; + } + } + if (eocdOffset < 0) { + throw const FormatException('.npz: End Of Central Directory not found'); + } + + final eocd = ByteData.sublistView(bytes, eocdOffset, eocdOffset + 22); + final entryCount = eocd.getUint16(10, Endian.little); + final centralDirOffset = eocd.getUint32(16, Endian.little); + + final out = {}; + var pos = centralDirOffset; + for (var i = 0; i < entryCount; i++) { + final header = ByteData.sublistView(bytes, pos, pos + 46); + if (header.getUint32(0, Endian.little) != centralDirSignature) { + throw const FormatException('.npz: malformed central directory entry'); + } + final compressionMethod = header.getUint16(10, Endian.little); + final compressedSize = header.getUint32(20, Endian.little); + final nameLen = header.getUint16(28, Endian.little); + final extraLen = header.getUint16(30, Endian.little); + final commentLen = header.getUint16(32, Endian.little); + final localHeaderOffset = header.getUint32(42, Endian.little); + final name = utf8.decode(bytes.sublist(pos + 46, pos + 46 + nameLen)); + + if (compressionMethod != storedMethod) { + throw StateError('npz member $name is DEFLATE; STORED expected'); + } + + // Parse the local file header to find where the raw data actually + // starts (its filename/extra-field lengths can differ in principle from + // the central directory's, even though numpy writes them identically). + final localHeader = ByteData.sublistView( + bytes, + localHeaderOffset, + localHeaderOffset + 30, + ); + if (localHeader.getUint32(0, Endian.little) != localHeaderSignature) { + throw FormatException('.npz: malformed local file header for $name'); + } + final localNameLen = localHeader.getUint16(26, Endian.little); + final localExtraLen = localHeader.getUint16(28, Endian.little); + final dataStart = localHeaderOffset + 30 + localNameLen + localExtraLen; + out[name] = bytes.sublist(dataStart, dataStart + compressedSize); + + pos += 46 + nameLen + extraLen + commentLen; + } + return out; +} diff --git a/packages/flutter_gemma_speech/lib/src/qwen3/qwen2_bpe_encoder.dart b/packages/flutter_gemma_speech/lib/src/qwen3/qwen2_bpe_encoder.dart new file mode 100644 index 00000000..8e8f2f8f --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/qwen3/qwen2_bpe_encoder.dart @@ -0,0 +1,304 @@ +// Qwen2 byte-level BPE ENCODER (text -> token ids) for the Qwen3-TTS text +// frontend. Companion to `Gpt2ByteLevelBpeTokenizer` +// (`../tokenizer/gpt2_byte_level_bpe_tokenizer.dart`), which is DECODE-only +// for Whisper's ids -> text; `HfTokenizer`/that decoder has no encoder, so +// this is net-new. Same byte-level BPE family (GPT-2's byte<->unicode map + +// rank-ordered merges), but reproduces Qwen2's specific pre-tokenizer +// pipeline as recorded in `tokenizer.json`: +// +// NFC normalize +// -> split out added/special tokens atomically (longest match first) +// -> for each remaining span, the Qwen2 pre-tokenizer `Split` regex +// (behavior "Isolated", `add_prefix_space=false`) +// -> ByteLevel: map each UTF-8 byte of every piece to its GPT-2 +// byte<->unicode character +// -> BPE merge each byte-level piece (rank-ordered, from `model.merges`) +// -> vocab lookup (`model.vocab`) for the final merged symbols. +// +// Pure Dart (`dart:convert` + `dart:io`) -- no Flutter import -- so this runs +// headless in `dart test` / `flutter test` without a device. +// +// NFC: this encoder does NOT perform explicit Unicode NFC normalization. +// Dart's core libraries have no built-in NFC implementation, and adding one +// would mean a new third-party dependency -- not done unilaterally here. In +// practice this is a pass-through: text authored in a normal editor/source +// file is already NFC, and the golden coverage (contractions, German/French +// accented Latin, CJK, whitespace runs, digits, a literal special-token +// string) all encode correctly without it. Known v1 limitation: arbitrary +// non-NFC input (e.g. combining-character sequences from some IMEs or +// pasted text) is not renormalized before encoding and may diverge from the +// reference HF tokenizer for that specific input. +library; + +import 'dart:convert'; +import 'dart:io'; + +/// Byte-level BPE encoder for the Qwen2 tokenizer family, driving the +/// Qwen3-TTS text frontend (text prompt -> token ids fed to the LLM core). +class Qwen2BpeEncoder { + Qwen2BpeEncoder._({ + required this._vocab, + required this._mergeRanks, + required List<_AddedToken> addedTokens, + }) : _addedTokenById = {for (final t in addedTokens) t.content: t.id}, + _addedTokenPattern = _buildAddedTokenPattern(addedTokens); + + final Map _vocab; + final Map _mergeRanks; + final Map _addedTokenById; + final RegExp? _addedTokenPattern; + + /// GPT-2's byte<->unicode map: every raw byte 0..255 -> a printable + /// unicode code point, so arbitrary UTF-8 bytes round-trip through a BPE + /// vocab of "characters". Ports the same table as + /// `Gpt2ByteLevelBpeTokenizer._gpt2ByteToUnicode()` (private there, so + /// duplicated here rather than reused across libraries); see that file's + /// doc comment for the byte-range rationale. + static final Map _byteToUnicode = _buildByteToUnicode(); + + static Map _buildByteToUnicode() { + final bytes = [ + for (var b = 0x21; b <= 0x7E; b++) b, + for (var b = 0xA1; b <= 0xAC; b++) b, + for (var b = 0xAE; b <= 0xFF; b++) b, + ]; + final selfMapped = bytes.toSet(); + final codepoints = [...bytes]; + var n = 0; + for (var b = 0; b < 256; b++) { + if (!selfMapped.contains(b)) { + bytes.add(b); + codepoints.add(256 + n); + n++; + } + } + return {for (var i = 0; i < bytes.length; i++) bytes[i]: codepoints[i]}; + } + + /// The Qwen2 pre-tokenizer's `Split` regex, taken verbatim from + /// `tokenizer.json`'s `pre_tokenizer.pretokenizers[0].pattern.Regex` + /// (a `Sequence` of `Split(behavior=Isolated, invert=false)` then + /// `ByteLevel(add_prefix_space=false, use_regex=false)`). The pattern + /// alone tiles the whole input (contractions / letter runs / lone digits + /// / punctuation runs / newline-terminated whitespace / trailing + /// whitespace / any other whitespace), so `allMatches` over it is + /// equivalent to applying the `Split` step. + /// + /// `(?i:...)` (inline case-insensitive group) and `\p{L}`/`\p{N}` (Unicode + /// property escapes, needing `unicode: true`) are both supported by + /// Dart's `RegExp` as of the 3.12 SDK. + static final RegExp _splitPattern = RegExp( + r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+", + unicode: true, + ); + + /// Loads an uncompressed `tokenizer.json` from disk (the form the model + /// bundle installs at runtime). + static Future fromTokenizerJson(String path) async { + final raw = await File(path).readAsString(); + return Qwen2BpeEncoder.fromTokenizerJsonString(raw); + } + + /// Same as [fromTokenizerJson], but from an already-in-memory JSON string + /// (e.g. a test that gunzips the golden fixture first). + factory Qwen2BpeEncoder.fromTokenizerJsonString(String jsonStr) { + return Qwen2BpeEncoder.fromTokenizerJsonMap( + jsonDecode(jsonStr) as Map, + ); + } + + /// Same as [fromTokenizerJson], but from an already-parsed `tokenizer.json` + /// document. + factory Qwen2BpeEncoder.fromTokenizerJsonMap(Map doc) { + final model = doc['model'] as Map; + + final vocabJson = model['vocab'] as Map; + final vocab = { + for (final entry in vocabJson.entries) entry.key: entry.value as int, + }; + + final mergesJson = model['merges'] as List; + final mergeRanks = {}; + for (var i = 0; i < mergesJson.length; i++) { + final entry = mergesJson[i]; + final String a; + final String b; + if (entry is List) { + a = entry[0] as String; + b = entry[1] as String; + } else if (entry is String) { + final sp = entry.indexOf(' '); + if (sp < 0) { + throw FormatException('malformed merges entry: $entry'); + } + a = entry.substring(0, sp); + b = entry.substring(sp + 1); + } else { + throw FormatException('unexpected merges entry type: $entry'); + } + mergeRanks[_pairKey(a, b)] = i; + } + + final addedTokensJson = doc['added_tokens'] as List? ?? const []; + final addedTokens = + [ + for (final t in addedTokensJson) + _AddedToken( + content: (t as Map)['content'] as String, + id: t['id'] as int, + ), + ] + // Longest-match-first: see `_buildAddedTokenPattern`. + ..sort((x, y) => y.content.length.compareTo(x.content.length)); + + return Qwen2BpeEncoder._( + vocab: vocab, + mergeRanks: mergeRanks, + addedTokens: addedTokens, + ); + } + + static String _pairKey(String a, String b) => '$a $b'; + + /// Builds a single alternation over every added-token content string, + /// longest-first, so that `allMatches` performs longest-match-at-position + /// scanning (regex alternation picks the first alternative that matches + /// at a given start position; since alternatives are sorted by + /// descending length, that first success is always the longest possible + /// added token starting there). Mirrors the HF `AddedVocabulary` trie's + /// longest-match behavior for `added_tokens` (both `special: true` and + /// `special: false` entries participate -- HF's fast tokenizer splits on + /// *all* added tokens before pre-tokenization, not just the "special" + /// subset). + static RegExp? _buildAddedTokenPattern(List<_AddedToken> addedTokens) { + if (addedTokens.isEmpty) return null; + final alternatives = addedTokens + .map((t) => RegExp.escape(t.content)) + .join('|'); + return RegExp(alternatives); + } + + /// Encodes [text] into token ids, reproducing the HF `tokenizers` + /// fast-tokenizer pipeline for this `tokenizer.json`. + List encode(String text) { + final ids = []; + for (final segment in _splitOnAddedTokens(text)) { + final addedId = segment.addedTokenId; + if (addedId != null) { + ids.add(addedId); + continue; + } + for (final match in _splitPattern.allMatches(segment.text!)) { + final piece = match[0]!; + if (piece.isEmpty) continue; + final byteLevel = _byteLevelEncode(piece); + for (final symbol in _bpe(byteLevel)) { + final id = _vocab[symbol]; + if (id == null) { + throw StateError( + 'Qwen2BpeEncoder: no vocab entry for BPE symbol ' + '${symbol.runes.toList()} (from piece "$piece")', + ); + } + ids.add(id); + } + } + } + return ids; + } + + List<_Segment> _splitOnAddedTokens(String text) { + final pattern = _addedTokenPattern; + if (pattern == null) return [_Segment.text(text)]; + + final segments = <_Segment>[]; + var lastEnd = 0; + for (final m in pattern.allMatches(text)) { + if (m.start > lastEnd) { + segments.add(_Segment.text(text.substring(lastEnd, m.start))); + } + final content = m[0]!; + segments.add(_Segment.added(_addedTokenById[content]!)); + lastEnd = m.end; + } + if (lastEnd < text.length) { + segments.add(_Segment.text(text.substring(lastEnd))); + } + return segments; + } + + /// UTF-8-encodes [piece] and maps each raw byte to its GPT-2 + /// byte<->unicode character, producing the byte-level string the BPE + /// merge table and vocab operate on. + String _byteLevelEncode(String piece) { + final bytes = utf8.encode(piece); + final buf = StringBuffer(); + for (final b in bytes) { + buf.writeCharCode(_byteToUnicode[b]!); + } + return buf.toString(); + } + + /// Standard byte-pair-encoding merge loop (the same algorithm as OpenAI's + /// reference `gpt2/encoder.py` `bpe()` / HF's slow-tokenizer `bpe()`): + /// each iteration finds the single lowest-rank pair present anywhere in + /// the symbol list, then merges *every* non-overlapping occurrence of + /// that exact pair in one left-to-right pass, and repeats until no known + /// pair remains. (Naively merging only the first occurrence found per + /// iteration -- and immediately reconsidering newly formed pairs -- can + /// diverge from this on inputs with repeated bigrams; merging the whole + /// pass before recomputing ranks is required for exact parity with the + /// reference tokenizer.) + List _bpe(String word) { + var symbols = [for (final r in word.runes) String.fromCharCode(r)]; + if (symbols.length < 2) return symbols; + + while (true) { + int? bestRank; + String? bestFirst; + String? bestSecond; + for (var i = 0; i < symbols.length - 1; i++) { + final rank = _mergeRanks[_pairKey(symbols[i], symbols[i + 1])]; + if (rank != null && (bestRank == null || rank < bestRank)) { + bestRank = rank; + bestFirst = symbols[i]; + bestSecond = symbols[i + 1]; + } + } + if (bestFirst == null || bestSecond == null) break; + final first = bestFirst; + final second = bestSecond; + + final merged = []; + var i = 0; + while (i < symbols.length) { + if (i < symbols.length - 1 && + symbols[i] == first && + symbols[i + 1] == second) { + merged.add(first + second); + i += 2; + } else { + merged.add(symbols[i]); + i += 1; + } + } + symbols = merged; + if (symbols.length == 1) break; + } + return symbols; + } +} + +class _AddedToken { + const _AddedToken({required this.content, required this.id}); + final String content; + final int id; +} + +class _Segment { + const _Segment.text(this.text) : addedTokenId = null; + const _Segment.added(int id) : addedTokenId = id, text = null; + + final String? text; + final int? addedTokenId; +} diff --git a/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_languages.dart b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_languages.dart new file mode 100644 index 00000000..290529c9 --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_languages.dart @@ -0,0 +1,75 @@ +// Qwen3-TTS language-id map + the public supported-language list. +// +// Single source of truth for THREE consumers that must never drift apart: +// - `Qwen3Prompt.build` (`qwen3_prompt.dart`), which maps a language name to +// its control-token id for the talker prompt. +// - `LiteRtSpeechSynthesizer.create`'s create-time language validator — +// fails loud with an `ArgumentError` for an unknown language BEFORE +// spawning the ~1.9 GB `TtsWorker`, rather than deep inside the worker +// isolate on the first `synthesize` call. +// - the example's language picker (`tts_screen.dart`), which populates its +// dropdown from [qwen3SupportedLanguages] instead of hardcoding a second +// copy of the list. +// +// This file is deliberately dependency-free (no imports at all, not even +// `dart:typed_data`) so it is safe to export UNCONDITIONALLY from the +// package barrel, including on platforms without `dart:io`/`dart:ffi` +// (`qwen3_prompt.dart` itself pulls in `qwen3_tables.dart`, which uses +// `dart:io` — that stays package-private). +library; + +/// Language id map from the model config (`talker_config.codec_language_id`). +/// Ported from `text_to_speech_lm/python/qwen3_tts_pipeline.py LANGUAGE_IDS`. +const Map languageIds = { + 'chinese': 2055, + 'english': 2050, + 'german': 2053, + 'italian': 2070, + 'portuguese': 2071, + 'spanish': 2054, + 'japanese': 2058, + 'korean': 2064, + 'french': 2061, + 'russian': 2069, +}; + +/// The full set of `language` values Qwen3-TTS accepts: [languageIds]'s +/// keys (alphabetical) plus `'auto'` (automatic language detection — no +/// language-id control token is emitted; see `Qwen3Prompt.build`'s `'auto'` +/// branch). `'auto'` is appended last since it isn't a language, it's a +/// detection mode. +final List qwen3SupportedLanguages = List.unmodifiable([ + ...(languageIds.keys.toList()..sort()), + 'auto', +]); + +/// Throws [ArgumentError] if [language] (case-insensitive) is not in +/// [qwen3SupportedLanguages]. Factored out as a standalone pure function +/// (rather than inlined at the call site) so it is unit-testable without +/// spawning a `TtsWorker` — which needs the ~1.9 GB Qwen3-TTS model bundle +/// on disk to construct at all. +void assertQwen3LanguageSupported(String language) { + if (!qwen3SupportedLanguages.contains(language.toLowerCase())) { + throw ArgumentError.value( + language, + 'language', + 'Unsupported Qwen3-TTS language. Must be one of: ' + '${qwen3SupportedLanguages.join(', ')}', + ); + } +} + +/// Canonicalizes an already-[assertQwen3LanguageSupported]-validated +/// [language] to the lowercase form every downstream consumer expects. +/// +/// `assertQwen3LanguageSupported`'s membership check is case-insensitive +/// (`language.toLowerCase()`), but `Qwen3Prompt.build`'s `'auto'` branch +/// compares case-SENSITIVELY (`language == 'auto'`; its non-`'auto'` branch +/// already lowercases via `languageIds[language.toLowerCase()]`). Without +/// normalizing once here, `'Auto'`/`'AUTO'` would pass validation at +/// `LiteRtSpeechSynthesizer.create` but then miss `Qwen3Prompt.build`'s +/// `'auto'` comparison and throw `ArgumentError` AFTER the ~1.9 GB model +/// load, at the first `synthesize` call — defeating create-time fail-fast +/// for exactly that value. Call this once, at the `create` boundary, +/// immediately after validation succeeds. +String normalizeQwen3Language(String language) => language.toLowerCase(); diff --git a/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_prompt.dart b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_prompt.dart new file mode 100644 index 00000000..bfb4227e --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_prompt.dart @@ -0,0 +1,212 @@ +// Prompt-embedding assembly for the Qwen3-TTS talker. +// +// `Qwen3Prompt.build` turns a text-token id sequence + a speaker x-vector + +// a language into: +// - `prefill` — the fixed-length (~10-row) prompt embeddings fed to the +// talker's `prefill_32` signature (role text + control/x-vector rows + +// the first streamed text token), +// - `trailing` — the rest of the text, one 1024-d embedding per decoded +// audio frame, consumed during the per-frame decode loop, +// - `ttsPad` — the fallback embedding used once `trailing` is exhausted. +// +// Ports `Qwen3TtsPipeline._build_prompt` verbatim from the recipe's +// `qwen3_tts_pipeline.py` (see `qwen3_tts_core.dart`'s file header for +// where it's vendored; plus its module-level token/language constants — +// `_CODEC_*`, `_TTS_*`, `LANGUAGE_IDS`), reading `ids` (already tokenized +// by the frontend, see `Qwen2BpeEncoder`) instead of re-tokenizing raw +// text. +// +// This file is pure Dart (`dart:typed_data` only) — no Flutter import — so +// it can be unit-tested without a device. + +import 'dart:typed_data'; + +import 'qwen3_languages.dart' show languageIds; +import 'qwen3_tables.dart'; + +/// Talker hidden size — width of every embedding row this module handles. +const int _hidden = 1024; + +// Talker codec-token vocabulary layout (ids >= 2048 are control tokens). +// Matches the recipe's codec-token constants block (`_CODEC_VOCAB` .. +// `_CODEC_NOTHINK` in `qwen3_tts_pipeline.py`). +const int _codecPad = 2148; +const int _codecBos = 2149; +const int _codecThink = 2154; +const int _codecNothink = 2155; +const int _codecThinkBos = 2156; +const int _codecThinkEos = 2157; + +// Text-side special tokens (Qwen2 BPE vocabulary). Matches the recipe's +// `_TTS_BOS`/`_TTS_EOS`/`_TTS_PAD` in `qwen3_tts_pipeline.py`. +const int _ttsBos = 151672; +const int _ttsEos = 151673; +const int _ttsPad = 151671; + +/// Result of [Qwen3Prompt.build]: the talker's fixed-length prefill +/// embeddings plus the per-frame streamed text conditioning consumed while +/// decoding audio frames. +class Qwen3PromptResult { + const Qwen3PromptResult({ + required this.prefill, + required this.promptLen, + required this.trailing, + required this.ttsPad, + }); + + /// Flattened prefill embeddings, row-major `[promptLen * 1024]`. Fed to + /// the talker's `prefill_32` signature (right-padded to 32 rows by the + /// caller — see `Qwen3TtsCore.runPrefill`, which ports `_run_prefill`). + final Float32List prefill; + + /// Number of rows in [prefill] (`prefill.length == promptLen * 1024`). + /// Fixed at `3 (role) + (codec_pre.length - 1) (body) + 1 (first_text)` — + /// decoupled from text length (only `ids[:4]` feed the prefill; the rest + /// of the text streams via [trailing]). + final int promptLen; + + /// Per-frame streamed text-conditioning rows, each `[1024]`: the + /// remaining text embeddings (`ids[4:-5]`) followed by the `tts_eos` + /// embedding. Consumed one row per decoded audio frame; once exhausted, + /// [ttsPad] is added instead (see the decode loop, `synthesize`, in + /// `qwen3_tts_core.dart`). + final List trailing; + + /// The `[1024]` TTS pad embedding, added once [trailing] is exhausted. + final Float32List ttsPad; +} + +/// Builds the Qwen3-TTS talker prompt. +/// +/// Ports `Qwen3TtsPipeline._build_prompt` verbatim, given already-tokenized +/// [ids] (the frontend's chat-template encoding — +/// see `Qwen2BpeEncoder`), a `[1024]` speaker x-vector [speaker], and a +/// [language] (one of [languageIds]' keys, case-insensitive, or `'auto'`). +class Qwen3Prompt { + Qwen3Prompt._(); + + static Qwen3PromptResult build({ + required List ids, + required Float32List speaker, + required String language, + required Qwen3Tables tables, + }) { + if (speaker.length != _hidden) { + throw ArgumentError.value( + speaker.length, + 'speaker.length', + 'Qwen3Prompt.build: speaker must be [$_hidden]', + ); + } + if (ids.length < 4) { + throw ArgumentError.value( + ids.length, + 'ids.length', + 'Qwen3Prompt.build: ids must have at least 4 entries ' + '(role = ids[0:3], first text token = ids[3:4])', + ); + } + + // qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final List control; + if (language == 'auto') { + control = [_codecNothink, _codecThinkBos, _codecThinkEos]; + } else { + final langId = languageIds[language.toLowerCase()]; + if (langId == null) { + final sortedNames = (languageIds.keys.toList()..sort()).join(', '); + throw ArgumentError.value( + language, + 'language', + 'Qwen3Prompt.build: language must be "auto" or one of: ' + '$sortedNames', + ); + } + control = [_codecThink, _codecThinkBos, langId, _codecThinkEos]; + } + + // tts_bos, tts_eos, tts_pad = embedText([_TTS_BOS, _TTS_EOS, _TTS_PAD]). + // qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final ttsSpecial = tables.embedText([_ttsBos, _ttsEos, _ttsPad]); + final ttsBosRow = Float32List.sublistView(ttsSpecial, 0, _hidden); + final ttsEosRow = Float32List.sublistView(ttsSpecial, _hidden, 2 * _hidden); + final ttsPad = Float32List.sublistView( + ttsSpecial, + 2 * _hidden, + 3 * _hidden, + ); + + // codec_pre = codec_emb[control] ++ [speaker] ++ codec_emb[[PAD, BOS]]. + // qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final codecPreLen = control.length + 1 + 2; + final codecPre = List.generate(codecPreLen, (i) { + if (i < control.length) return tables.codecEmbRow(control[i]); + if (i == control.length) return speaker; + if (i == control.length + 1) return tables.codecEmbRow(_codecPad); + return tables.codecEmbRow(_codecBos); + }); + + // role = embedText(ids[0:3]). qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final role = tables.embedText(ids.sublist(0, 3)); + + // pads = repeat(tts_pad, len(codec_pre) - 2); + // body = concat([pads, tts_bos]) + codec_pre[:-1] (element-wise add). + // qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final bodyLen = codecPreLen - 1; + final padsLen = codecPreLen - 2; + final body = Float32List(bodyLen * _hidden); + for (var i = 0; i < bodyLen; i++) { + final base = i * _hidden; + final addend = i < padsLen ? ttsPad : ttsBosRow; + final codecRow = codecPre[i]; + for (var k = 0; k < _hidden; k++) { + body[base + k] = addend[k] + codecRow[k]; + } + } + + // first_text = embedText(ids[3:4]) + codec_pre[-1]. qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final firstTextEmbed = tables.embedText([ids[3]]); + final firstText = Float32List(_hidden); + final lastCodecPre = codecPre[codecPreLen - 1]; + for (var k = 0; k < _hidden; k++) { + firstText[k] = firstTextEmbed[k] + lastCodecPre[k]; + } + + // prefill = concat(role, body, first_text). qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + final promptLen = 3 + bodyLen + 1; + if (promptLen > 32) { + // Defensive sanity check mirroring `Qwen3TtsPipeline._run_prefill`'s + // `p > 32` guard — the talker's prefill signature is a fixed 32-row + // buffer. This is NOT a text-length limit: only role + + // control/x-vector rows enter the prefill (promptLen is fixed at 10 + // for a named language, 9 for 'auto'), so it never trips in practice. + throw StateError( + 'Qwen3Prompt.build: prefill length $promptLen exceeds the talker ' + 'prefill_32 signature (32)', + ); + } + final prefill = Float32List(promptLen * _hidden); + prefill.setRange(0, 3 * _hidden, role); + prefill.setRange(3 * _hidden, (3 + bodyLen) * _hidden, body); + prefill.setRange((3 + bodyLen) * _hidden, promptLen * _hidden, firstText); + + // trailing = concat(embedText(ids[4:-5]), tts_eos). qwen3_tts_pipeline.py Qwen3TtsPipeline._build_prompt. + // Mirrors Python's permissive `ids[4:-5]` slicing (clamped to empty + // rather than throwing) for very short inputs where `len(ids) - 5 < 4`. + final trailingStop = ids.length - 5 < 4 ? 4 : ids.length - 5; + final trailingTextIds = ids.sublist(4, trailingStop); + final trailingText = tables.embedText(trailingTextIds); + final trailing = [ + for (var i = 0; i < trailingTextIds.length; i++) + Float32List.sublistView(trailingText, i * _hidden, (i + 1) * _hidden), + Float32List.fromList(ttsEosRow), + ]; + + return Qwen3PromptResult( + prefill: prefill, + promptLen: promptLen, + trailing: trailing, + ttsPad: Float32List.fromList(ttsPad), + ); + } +} diff --git a/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_sampler.dart b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_sampler.dart new file mode 100644 index 00000000..1b003028 --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_sampler.dart @@ -0,0 +1,185 @@ +// Sampler + cb0 scoring for the Qwen3-TTS talker's per-frame codebook-0 +// (cb0) token pick. +// +// Ports `Qwen3TtsPipeline._pick` (the recipe's `qwen3_tts_pipeline.py` — +// see `qwen3_tts_core.dart`'s file header for where it's vendored and why +// citations below name Python symbols, not line numbers) and the per-step +// scoring edits applied before it in `Qwen3TtsPipeline.synthesize` +// (`suppress`, the min-new-tokens guard, and the HF repetition penalty) +// verbatim. +// +// This file is pure Dart (`dart:math` / `dart:typed_data` only) — no +// Flutter import — so it can be unit-tested without a device and needs no +// model tables (all inputs are hand-constructable vectors). + +import 'dart:math'; +import 'dart:typed_data'; + +/// Talker codec-token vocabulary size. qwen3_tts_pipeline.py _CODEC_VOCAB. +const int _codecVocab = 3072; + +/// Codec end-of-sequence token id. qwen3_tts_pipeline.py _CODEC_EOS. +const int _codecEos = 2150; + +/// Large negative "suppressed" score, added to logits to rule a token out +/// of the argmax/sampling pick without producing `-inf` arithmetic hazards. +/// qwen3_tts_pipeline.py _NEG_INF. +const double _negInf = -1e9; + +/// Builds the per-step control-token suppression vector. +/// +/// Every codec-vocabulary index `>= 2048` is a control/special token (only +/// `< 2048` are audible codebook-0 codes), so those are suppressed by +/// `_negInf` — except [_codecEos], which is always a legal pick (the talker +/// must be able to end the utterance). Ports +/// `qwen3_tts_pipeline.py Qwen3TtsPipeline.synthesize` verbatim: +/// ```python +/// suppress = np.zeros(_CODEC_VOCAB, np.float32) +/// suppress[2048:] = _NEG_INF +/// suppress[_CODEC_EOS] = 0.0 +/// ``` +Float32List buildSuppress() { + final suppress = Float32List(_codecVocab); + for (var i = 2048; i < _codecVocab; i++) { + suppress[i] = _negInf; + } + suppress[_codecEos] = 0.0; + return suppress; +} + +/// Applies the talker's per-step cb0 scoring edits to [scores] **in +/// place** (matches the recipe, which reassigns entries of the `scores` +/// array directly rather than returning a new one — callers must pass an +/// array that is safe to mutate, e.g. a fresh copy of the raw logits, since +/// the original logits are still needed elsewhere in the decode loop). +/// +/// Ports `qwen3_tts_pipeline.py Qwen3TtsPipeline.synthesize` verbatim: +/// ```python +/// scores = logits + suppress +/// if len(frames) < 2: # min_new_tokens=2 +/// scores[_CODEC_EOS] = _NEG_INF +/// for token in history: +/// scores[token] = (scores[token] / repetition_penalty +/// if scores[token] > 0 +/// else scores[token] * repetition_penalty) +/// ``` +/// [minNewTokensGuard] is the caller-evaluated `len(frames) < 2` condition +/// (frame-count bookkeeping lives in the decode loop, `synthesize` — this +/// function only applies the resulting edit). +void applyCb0Scoring( + Float32List scores, { + required Float32List suppress, + required Set history, + required double repetitionPenalty, + required bool minNewTokensGuard, +}) { + for (var i = 0; i < scores.length; i++) { + scores[i] += suppress[i]; + } + if (minNewTokensGuard) { + scores[_codecEos] = _negInf; + } + for (final token in history) { + final s = scores[token]; + scores[token] = s > 0 ? s / repetitionPenalty : s * repetitionPenalty; + } +} + +/// Deterministic, RNG-free pick: the argmax index of [logits]. +/// +/// This is the golden-decode path (`do_sample=False` in the recipe): +/// `return int(np.argmax(logits))` (`qwen3_tts_pipeline.py _pick`). Matches +/// numpy's `argmax` tie-break — the **first** (lowest-index) maximum wins. +int pickGreedy(Float32List logits) { + if (logits.isEmpty) { + throw ArgumentError.value( + logits.length, + 'logits.length', + 'pickGreedy: logits must not be empty', + ); + } + var bestIndex = 0; + var bestValue = logits[0]; + for (var i = 1; i < logits.length; i++) { + if (logits[i] > bestValue) { + bestValue = logits[i]; + bestIndex = i; + } + } + return bestIndex; +} + +/// Top-k / temperature sampling pick (production quality — not the golden +/// gate, which uses [pickGreedy]). +/// +/// Ports `qwen3_tts_pipeline.py _pick`'s sampling branch: +/// ```python +/// scaled = logits.astype(np.float64) / max(temperature, 1e-6) +/// if top_k and top_k < len(scaled): +/// kth = np.partition(scaled, -top_k)[-top_k] +/// scaled = np.where(scaled < kth, -np.inf, scaled) +/// scaled -= scaled.max() +/// probs = np.exp(scaled) +/// probs /= probs.sum() +/// return int(rng.choice(len(probs), p=probs)) +/// ``` +/// Computed in `double` precision throughout (matching the recipe's +/// `.astype(np.float64)`), returning an `int`. The final draw is a plain +/// cumulative-sum inverse-CDF sample from [rng] (a `dart:math.Random`) — +/// this does not attempt to bit-reproduce numpy `Generator.choice`'s +/// internal algorithm; determinism is only required (and tested) against a +/// seeded Dart [Random]. +int pickSampled( + Float32List logits, { + required int topK, + required double temperature, + required Random rng, +}) { + if (logits.isEmpty) { + throw ArgumentError.value( + logits.length, + 'logits.length', + 'pickSampled: logits must not be empty', + ); + } + final n = logits.length; + final denom = max(temperature, 1e-6); + final scaled = List.generate(n, (i) => logits[i] / denom); + + if (topK > 0 && topK < n) { + // np.partition(scaled, -top_k)[-top_k]: the value that would land at + // index `n - top_k` in a fully sorted-ascending array, i.e. the + // top_k-th largest value — the keep/suppress threshold. Ties at the + // threshold are all kept (matches `scaled < kth`, not a rank cutoff), + // so more than top_k entries may survive when the threshold value + // repeats. + final sortedDesc = List.of(scaled)..sort((a, b) => b.compareTo(a)); + final kth = sortedDesc[topK - 1]; + for (var i = 0; i < n; i++) { + if (scaled[i] < kth) scaled[i] = double.negativeInfinity; + } + } + + var maxVal = scaled[0]; + for (var i = 1; i < n; i++) { + if (scaled[i] > maxVal) maxVal = scaled[i]; + } + var sum = 0.0; + final probs = List.generate(n, (i) { + final p = exp(scaled[i] - maxVal); + sum += p; + return p; + }); + for (var i = 0; i < n; i++) { + probs[i] /= sum; + } + + // Cumulative-sum inverse-CDF sample. + final u = rng.nextDouble(); + var cumulative = 0.0; + for (var i = 0; i < n; i++) { + cumulative += probs[i]; + if (u < cumulative) return i; + } + return n - 1; // Guards float round-off so the draw never falls through. +} diff --git a/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_tables.dart b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_tables.dart new file mode 100644 index 00000000..ade51675 --- /dev/null +++ b/packages/flutter_gemma_speech/lib/src/qwen3/qwen3_tables.dart @@ -0,0 +1,289 @@ +// Host embedding tables + text-projection MLP for the Qwen3-TTS pipeline. +// +// `Qwen3Tables` owns four on-disk NumPy tables shipped alongside the +// Qwen3-TTS talker model: +// - `codec_embedding_fp32.npy` (3072, 1024)