Add Parakeet-TDT 0.6B v3 (FastConformer-TDT ASR)

CMakeLists.txt

@@ -419,6 +419,20 @@ audiocpp_add_model(outetts
         qwen3_forced_aligner
 )
 
+audiocpp_add_model(parakeet_tdt
+    SOURCES
+        src/community_models/parakeet_tdt/assets.cpp
+        src/community_models/parakeet_tdt/session.cpp
+        src/community_models/parakeet_tdt/encoder.cpp
+        src/community_models/parakeet_tdt/decoder.cpp
+        src/community_models/parakeet_tdt/frontend.cpp
+        src/community_models/parakeet_tdt/weights.cpp
+    INCLUDES
+        engine/community_models/parakeet_tdt/session.h
+    LOADERS
+        engine::community_models::parakeet_tdt::make_parakeet_tdt_loader
+)
+
 audiocpp_add_model(pocket_tts
     SOURCES
         src/models/pocket_tts/assets.cpp
@@ -1263,6 +1277,7 @@ if (ENGINE_BUILD_WARMBENCH)
     add_engine_warmbench(nemotron_asr_warm_bench tests/nemotron_asr/nemotron_asr_warm_bench.cpp)
     add_engine_warmbench(omnivoice_warm_bench tests/omnivoice/omnivoice_warm_bench.cpp)
     add_engine_warmbench(outetts_warm_bench tests/outetts/outetts_warm_bench.cpp)
+    add_engine_warmbench(parakeet_warm_bench tests/parakeet_tdt/parakeet_warm_bench.cpp)
     add_engine_warmbench(glm_tts_tokenizer_probe tests/glm_tts/glm_tts_tokenizer_probe.cpp)
     add_engine_warmbench(glm_tts_assets_probe tests/glm_tts/glm_tts_assets_probe.cpp)
     add_engine_warmbench(glm_tts_prompt_probe tests/glm_tts/glm_tts_prompt_probe.cpp)
@@ -1528,6 +1543,14 @@ if (ENGINE_BUILD_TESTS)
         COMMAND model_spec_system_test
     )
 
+    add_engine_unittest(tdt_decoder_duration_loop_test tests/unittests/test_tdt_decoder_duration_loop.cpp)
+    target_include_directories(tdt_decoder_duration_loop_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests)
+
+    add_test(
+        NAME tdt_decoder_duration_loop_test
+        COMMAND tdt_decoder_duration_loop_test
+    )
+
     add_engine_unittest(scaled_dot_product_attention_test tests/unittests/test_scaled_dot_product_attention.cpp)
 
     add_test(
@@ -1588,6 +1611,49 @@ if (ENGINE_BUILD_TESTS)
         NAME server_config_test
         COMMAND server_config_test
     )
+
+    add_engine_unittest(parakeet_golden_transcription_test tests/parakeet_tdt/test_golden_transcription.cpp)
+    target_compile_definitions(parakeet_golden_transcription_test PRIVATE
+        ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+
+    add_test(
+        NAME parakeet_golden_transcription_test
+        COMMAND parakeet_golden_transcription_test
+    )
+    # Requires the real (multi-GB) Parakeet-TDT weights, which aren't
+    # downloaded as part of a normal checkout/CI build; the test itself
+    # detects this and exits with kExitSkip (125) rather than failing.
+    set_tests_properties(parakeet_golden_transcription_test PROPERTIES
+        SKIP_RETURN_CODE 125
+        TIMEOUT 120
+    )
+
+    add_engine_unittest(parakeet_streaming_transcription_test tests/parakeet_tdt/test_streaming_transcription.cpp)
+    target_compile_definitions(parakeet_streaming_transcription_test PRIVATE
+        ENGINE_REPO_ROOT="${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+
+    add_test(
+        NAME parakeet_streaming_transcription_test
+        COMMAND parakeet_streaming_transcription_test
+    )
+    set_tests_properties(parakeet_streaming_transcription_test PROPERTIES
+        SKIP_RETURN_CODE 125
+        TIMEOUT 180
+    )
+
+    # Numerical parity dump tool (not registered as an add_test — it's a
+    # manual/pre-release check, not a CI gate). Run it against a NeMo
+    # reference dump via compare_parity.py; see tests/parakeet_tdt/parity/README.md.
+    add_executable(parakeet_parity_dump
+        tests/parakeet_tdt/parity/dump_cpp_reference.cpp
+    )
+    target_include_directories(parakeet_parity_dump PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/parakeet_tdt/parity)
+    target_link_libraries(parakeet_parity_dump PRIVATE engine_runtime ggml)
+    if (ENGINE_ENABLE_OPENMP)
+        target_link_libraries(parakeet_parity_dump PRIVATE OpenMP::OpenMP_CXX)
+    endif()
 endif()
 
 if (ENGINE_BUILD_EXAMPLES)

README.md

@@ -100,6 +100,7 @@ Community model ports live under `community_models` to make the ownership bounda
 | **glm_tts** | TTS, Clone | zh, en | GGUF | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](docs/community_models/glm_tts.md) zero-shot synthesis and voice cloning support |
 | **moss_tts_local** | TTS, Clone, Ctrl | auto, optional language hint | GGUF | [@justinjohn0306](https://github.com/justinjohn0306) | MOSS-TTS-Local Transformer v1.5 support |
 | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support |
+| **parakeet_tdt** | ASR | 25 European languages, auto-detect | safetensors | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) FastConformer-TDT ASR, offline only |
 | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support |
 
 PocketTTS language selection is a model-load option. When the model path points at the PocketTTS root, the loader uses `english` unless you pass `--load-option language=<name>`. Kyutai's normal non-English PocketTTS releases are smaller distilled language models intended for the fast PocketTTS path. The `_24l` variants are larger 24-layer, undistilled preview models that can sound better but are slower. Kyutai currently publishes French only as `french_24l`, not as a normal distilled `french` language directory, so French is not listed as a normal PocketTTS language here.

docs/community_models/models.md

@@ -19,4 +19,5 @@ Practical expectations:
 | **glm_tts** | TTS, voice cloning | zh, en | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](glm_tts.md) zero-shot synthesis and voice cloning support |
 | **moss_tts_local** | TTS, voice cloning | auto, optional language hint | [@justinjohn0306](https://github.com/justinjohn0306) | [MOSS-TTS-Local Transformer v1.5](../models/moss_tts.md) support in the core model tree |
 | **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Mirek [@mirek190](https://github.com/mirek190) | [Llama-OuteTTS-1.0-1B](outetts.md) TTS and voice cloning support |
+| **parakeet_tdt** | ASR | 25 European languages, auto-detect | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](parakeet_tdt.md) FastConformer-TDT ASR, offline only |
 | **vietneu_tts** | TTS, voice cloning | vi, en | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](vietneu_tts.md) TTS and voice cloning support |

docs/community_models/parakeet_tdt.md

@@ -0,0 +1,244 @@
+# Parakeet-TDT 0.6B v3
+
+FastConformer-TDT ASR port of NVIDIA's [`nvidia/parakeet-tdt-0.6b-v3`](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3)
+(0.6B params, 25 European languages with auto language detection), loaded directly
+from the repo's Transformers-compatible `model.safetensors` checkpoint (not the
+`.nemo` archive).
+
+## Status
+
+Offline full-context, bounded-window long-form, and buffered-streaming sessions
+are implemented. The full-context path is verified end to end on CPU and CUDA
+against the real NeMo reference model, both by final transcription and by
+numerical (per-layer activation) comparison; see [Validation](#validation).
+Buffered modes deliberately re-encode fixed bidirectional windows and are not
+native cache-aware streaming.
+
+| Mode | Context and memory | Result timing |
+|---|---|---|
+| Offline `full_context` | Whole utterance; quadratic attention grows with duration | One final result |
+| Offline `long_form` | Bounded overlapping windows; center regions decoded once | One final result |
+| Streaming | Same bounded windows with retained predictor state | Partial snapshots after right-context lookahead, then a final result |
+
+## Architecture
+
+```
+Frontend: 16kHz -> 128 mel bins, preemphasis=0.97, NeMo per-feature normalization
+Encoder:  3-stage Conv2D subsampling (8x) -> 24-layer FastConformer -> 1024-dim
+Decoder:  2-layer LSTM predictor + joint network (TDT: 5 duration classes [0..4])
+```
+
+## Build and run
+
+```bash
+cmake --build build/<preset> --target parakeet_warm_bench
+
+build/<preset>/bin/parakeet_warm_bench \
+    --model models/parakeet-tdt-0.6b-v3 \
+    --audio tests/parakeet_tdt/assets/2086-149220-0033.wav \
+    --backend cpu   # or: --backend cuda
+```
+
+Equivalent CLI commands:
+
+```bash
+# Offline full-context
+build/<preset>/bin/audiocpp_cli \
+    --task asr --family parakeet_tdt \
+    --model models/parakeet-tdt-0.6b-v3 --backend cpu \
+    --audio tests/parakeet_tdt/assets/2086-149220-0033.wav
+
+# Offline bounded-window long-form
+build/<preset>/bin/audiocpp_cli \
+    --task asr --family parakeet_tdt \
+    --model models/parakeet-tdt-0.6b-v3 --backend cpu \
+    --session-option parakeet_tdt.offline_mode=long_form \
+    --audio tests/parakeet_tdt/assets/2086-149220-0033.wav
+
+# Buffered streaming from raw mono 16 kHz PCM
+ffmpeg -loglevel error -i input.wav -f f32le -ac 1 -ar 16000 - \
+  | build/<preset>/bin/audiocpp_cli \
+      --task asr --family parakeet_tdt \
+      --model models/parakeet-tdt-0.6b-v3 --backend cpu \
+      --mode streaming --audio - --input-format f32le \
+      --input-rate 16000 --input-channels 1
+```
+
+Install the model with:
+
+```bash
+python3 tools/model_manager_v2.py install parakeet_tdt --models-root models
+```
+
+Word timestamps are built from the decoder's actual nonblank token-emission
+frames. SentencePiece fragments and following punctuation are merged into
+human-readable words. Each completed word ends at the next word's emission
+boundary; the final word ends at the final token's predicted duration boundary,
+clamped to the encoded audio.
+
+## Buffered streaming
+
+`RunMode::Streaming` provides bounded-window buffered streaming for callers
+that need incremental results. The default window uses a 2-second center
+region, 10 seconds of left context, and 2 seconds of right context. Each fixed
+window is re-encoded with full bidirectional attention, then only its center
+frames are decoded while the TDT predictor state is retained. This is not
+cache-aware streaming: right context adds lookahead latency, partial text can
+differ from offline full-context output, and recent text remains provisional
+until the next center boundary. Use `nemotron_asr` when native cache-aware,
+lower-latency ASR is required.
+
+The window is controlled with `parakeet_tdt.audio_chunk_duration_sec`,
+`parakeet_tdt.left_context_sec`, and `parakeet_tdt.right_context_sec`.
+Streaming currently requires contiguous mono 16 kHz chunks. `finalize()`
+flushes a short tail; `reset()` retains the loaded weights and reusable graphs.
+
+## Long-form audio
+
+Offline mode remains full-context by default. Set
+`parakeet_tdt.offline_mode=long_form` to process arbitrarily long mono 16 kHz
+audio with the same bounded overlapping-window scheduler, or use `auto` to
+switch after `parakeet_tdt.audio_chunk_threshold_sec` (30 seconds by
+default). Long-form mode preserves global timestamps and predictor state while
+decoding every center region exactly once. Because each region sees bounded
+rather than utterance-wide context, its transcript can differ from the default
+full-context result.
+
+## Performance
+
+Reference clip (`2086-149220-0033.wav`, 7.435s) on an Intel i7-9750H (6C/12T)
+and a GTX 1650 Max-Q, against the real NeMo model on the same machine:
+
+| implementation | settings | CPU | CUDA |
+|---|---|---|---|
+| **NeMo / PyTorch** (the Python reference) | as shipped | 857.7 ms (8.7x real-time) | **OOM** — won't load on this 4GB card |
+| **audio.cpp** (this port) | untuned: 8 threads, f32 | ~1245 ms (6.0x real-time) | ~169 ms (44.0x real-time) |
+| **audio.cpp** (this port) | tuned: 12 threads, `q8_0` | **~715 ms** (**10.4x** real-time) | **~131 ms** (56.8x real-time) |
+
+Read down the CPU column: **untuned, this port loses to PyTorch** (~1245 ms vs
+857.7 ms). Tuned, it wins by ~1.2x — and on this GPU the comparison doesn't
+exist at all, because the PyTorch reference cannot load the model in 4 GB
+while the native path runs it in 2642 MiB. "Untuned" here means the two
+defaults, not a deliberately handicapped configuration: `--threads` defaults
+to 8, and `parakeet_tdt.matmul_weight_type` defaults to `native`, which for
+this checkpoint means f32 (`config.json` declares `dtype: float32`).
+
+Re-measured on the current tree with the drift-cancelling harness (ABBA, 4
+passes, median of 5 iterations per run): tuned is **1.72x** default on CPU and
+**1.31x** on CUDA. Both defaults reproduce earlier figures closely; the CPU
+tuned row previously read ~750 ms, which was measured before the optimization
+pass and is now slightly conservative.
+
+```bash
+build/<preset>/bin/parakeet_warm_bench \
+    --model models/parakeet-tdt-0.6b-v3 --audio tests/parakeet_tdt/assets/2086-149220-0033.wav \
+    --backend cpu --threads 12 \
+    --session-option parakeet_tdt.matmul_weight_type=q8_0
+```
+
+Two settings account for essentially all of the tunable gap:
+
+- **Thread count.** 12 threads is fastest on this 6C/12T CPU — 3.1% over 6,
+  1.1% over 8. Sweep it on your own hardware, and sweep it *interleaved*
+  rather than as sequential before/after runs (this machine drifts 20-30%
+  between cold and thermally saturated, which is enough to invert the answer).
+- **Weight precision.** `parakeet_tdt.matmul_weight_type` accepts
+  `native|f32|f16|bf16|q8_0` (default `native`; `parakeet_tdt.conv_weight_type`
+  separately accepts `native|f32|f16`). Encoder graph compute is ~93-96% of
+  wall time, so this is the setting that matters.
+
+Measured over 120 FLEURS clips across 24 languages
+([harness](../../tests/parakeet_tdt/multilingual/README.md),
+[raw transcripts](../../tests/parakeet_tdt/multilingual/results/README.md)):
+
+| weight type | encode speed | transcript identical to f32 | absolute WER |
+|---|---|---|---|
+| `native` (f32) | 1.00x | 100% | 0.1338 |
+| `f16` | ~1.00x | 99.2% | 0.1336 |
+| `bf16` | **1.14x** | 99.2% | 0.1334 |
+| `q8_0` | **1.79x** | 91.7% | 0.1331 |
+
+- **`bf16`** is close to free: 1.14x, 99.2% of transcripts byte-identical.
+- **`q8_0`** is the real speedup at 1.79x. It changes ~8% of transcripts, but
+  absolute WER does not move — the churn is mostly formatting (`T Rex` →
+  `T-Rex`), and where words change it goes both directions. It costs
+  reproducibility against an f32 baseline, not quality.
+- **`f16`** is not a speedup here at all; don't assume fewer bits means faster.
+
+`native` stays the default because it is the only setting that reproduces
+exactly. Full methodology, profiling, and the optimizations that were tried
+and rejected are in
+[docs/reports/parakeet_tdt_performance.md](../reports/parakeet_tdt_performance.md).
+
+## Validation
+
+- **Golden-transcription regression test** (`ctest -R parakeet_golden_transcription_test`,
+  wired into the normal `ENGINE_BUILD_TESTS` suite, skips cleanly when the model
+  isn't downloaded): runs the offline pipeline against a checked-in LibriSpeech
+  test-clean clip (`tests/parakeet_tdt/assets/2086-149220-0033.wav`) and asserts the
+  decoded text matches the real NeMo model's transcription for that clip exactly:
+  *"Well, I don't wish to see it any more, observed Phoebe, turning away her eyes.
+  It is certainly very like the old portrait."*
+- **Numerical parity harness** (`tests/parakeet_tdt/parity/`, manual/pre-release
+  check, not a CI gate — needs a NeMo install): compares mel features, a single
+  isolated encoder layer, and the full encoder output directly against real NeMo
+  activations via forward hooks. See `tests/parakeet_tdt/parity/README.md` for
+  exact setup and run commands. This is what actually caught the encoder
+  conv-module bug below — the golden-transcription test alone did not, since
+  greedy decoding happened to be robust enough to it on that one clip.
+  Currently passing on both backends: `mel_features` and `layer_0` at cosine
+  1.000000, `enc_out` at 0.972258 on CPU and CUDA alike.
+- Backends tested: CPU and CUDA (GTX 1650 Max-Q), both producing the exact same
+  transcription.
+
+Two real bugs were found and fixed getting to this state, worth knowing about if
+you're touching this code:
+
+1. The frontend was missing NeMo's per-feature mel normalization (subtract the
+   per-mel-bin mean, divide by the per-mel-bin unbiased std over valid frames).
+   Without it the encoder saw wildly out-of-distribution input and produced
+   garbage (~500x too large in magnitude by the end of the conv subsampling
+   stack).
+2. The FastConformer encoder layer's conv module used causal (left-only)
+   padding, and separately built the depthwise conv with `use_bias=false`,
+   silently dropping the folded batch-norm bias term. The padding mode is
+   wrong for this model's full-context (non-streaming) configuration — NeMo's
+   `CausalConv1D` is only actually causal when constructed with `padding=None`;
+   this model constructs it with an explicit symmetric `padding=(kernel-1)//2`.
+   The dropped bias corrupted every layer's output by a per-channel constant
+   offset; because greedy/argmax decoding is somewhat robust to numerical
+   drift, this second bug did not visibly break transcription on the one test
+   clip and was only caught by the numerical parity harness comparing actual
+   activations against NeMo, not just final decoded text.
+
+## Known limitations
+
+- **Buffered streaming is not native cache-aware streaming.**
+  `nvidia/parakeet-tdt-0.6b-v3` was trained and exported with
+  `att_context_style="regular"` and `att_context_size=[-1, -1]` — unlimited,
+  fully bidirectional attention context, not NeMo's `"chunked_limited"` style
+  that cache-aware streaming depends on. Calling NeMo's own
+  `encoder.setup_streaming_params()` on this checkpoint does not error, but
+  produces a degenerate configuration (~5.8 second chunks, a ~10000-frame
+  attention cache) that provides essentially none of the latency or bounded-
+  memory benefit real streaming is for. NVIDIA's own maintainers, asked
+  directly about streaming with this model on its Hugging Face discussion
+  page, pointed users at a different, dedicated cache-aware streaming
+  architecture rather than confirming this checkpoint for the purpose.
+  The implemented `RunMode::Streaming` therefore uses bounded, overlapping
+  full-attention windows and retains only the TDT predictor state between
+  center regions. It bounds graph and live-buffer size and produces partial
+  results, but re-encodes context, incurs configured right-lookahead latency,
+  and can differ from utterance-wide offline output. Genuine causal streaming
+  would need a purpose-trained `att_context_style="chunked_limited"`
+  checkpoint.
+  **If you need streaming ASR, this framework already has it**: the
+  `nemotron_asr` family (`nvidia/nemotron-3.5-asr-streaming-0.6b`) is a
+  same-size-class NVIDIA checkpoint actually trained cache-aware, with
+  configurable chunk sizes down to 80ms, and already implements
+  `IStreamingVoiceTaskSession` in this codebase.
+- Validated against a single test clip end to end; the numerical parity
+  harness has not been run across a broader validation set.
+- The frontend does not implement NeMo's preprocessor `dither` (small random
+  waveform noise, standard to disable at inference time in most ASR
+  pipelines) — no observed effect on transcription correctness.