From 9a41ca6c93f2c7325928cbd804270c39e04289d8 Mon Sep 17 00:00:00 2001 From: Ivan Date: Sun, 5 Jul 2026 08:06:26 +0200 Subject: [PATCH 1/2] Support Chatterbox multilingual frontends --- README.md | 8 +- .../ChatterboxTTS/ChatterboxTTSModel.swift | 17 +- Sources/ChatterboxTTS/MTLTokenizer.swift | 217 +++++++++++++++++- Sources/OmniVoiceTTS/OmniVoiceTTSModel.swift | 6 +- .../MTLTokenizerTests.swift | 96 +++++++- .../RoundtripE2ETests.swift | 8 +- .../OmniVoiceModelFacadeTests.swift | 9 +- docs/models/chatterbox-tts.md | 21 +- 8 files changed, 344 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 5240f478..b88a33e2 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,8 @@ On-device speech recognition, synthesis, and understanding for Mac and iOS. Runs - **[VibeVoice TTS](https://soniqo.audio/guides/vibevoice)** — Long-form / multi-speaker TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, up to 90-min podcast/audiobook synthesis, EN/ZH) - **[Magpie TTS](https://soniqo.audio/guides/magpie)** — Multilingual TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB or CoreML INT8 342 MB, 9 languages, 5 baked speakers, streaming on MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — On-device flow-matching TTS (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 languages, 10 voices, G2P-free, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Multilingual TTS with zero-shot voice cloning (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23 languages, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — Non-autoregressive diffusion TTS with zero-shot voice cloning (k2-fsa OmniVoice, Qwen3 backbone, MLX int8 ~1 GB / fp16, 600+ languages, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Multilingual TTS with zero-shot voice cloning (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 22 runtime languages, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — Non-autoregressive diffusion TTS with zero-shot voice cloning (k2-fsa OmniVoice, Qwen3 backbone, MLX fp16 default / int8 available, 600+ languages, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLMs & Translation** @@ -168,8 +168,8 @@ Compact view below. **[Full model catalogue with sizes, quantisations, download | [VibeVoice Realtime-0.5B](https://soniqo.audio/guides/vibevoice) | Text → Speech (long-form, multi-speaker) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/guides/vibevoice) | Text → Speech (up to 90-min podcast) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/guides/magpie) | Text → Speech (5 baked speakers, streaming) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML excludes JA) | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Text → Speech (zero-shot cloning) | MLX | 0.8B (fp16) | 23 | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Text → Speech (NAR diffusion, zero-shot cloning) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Text → Speech (zero-shot cloning) | MLX | 0.8B (fp16) | 22 runtime | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Text → Speech (NAR diffusion, zero-shot cloning) | MLX | 0.8B (fp16 default / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Text → Speech (zero-shot cloning, explicit style markers) | MLX | 0.5B-class (fp16) | Multilingual | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/Sources/ChatterboxTTS/ChatterboxTTSModel.swift b/Sources/ChatterboxTTS/ChatterboxTTSModel.swift index 9164e092..66508d6c 100644 --- a/Sources/ChatterboxTTS/ChatterboxTTSModel.swift +++ b/Sources/ChatterboxTTS/ChatterboxTTSModel.swift @@ -204,7 +204,16 @@ public final class ChatterboxTTSModel { for: s3TokenizerModelId, cacheDirName: "chatterbox-s3-tokenizer") let fm = FileManager.default - if !fm.fileExists(atPath: bundleDir.appendingPathComponent("model.safetensors").path) { + let requiredBundleFiles = [ + "model.safetensors", + "config.json", + "tokenizer.json", + "Cangjie5_TC.json", + ] + let bundleNeedsRepair = requiredBundleFiles.contains { + !fm.fileExists(atPath: bundleDir.appendingPathComponent($0).path) + } + if bundleNeedsRepair { progressHandler?(0.0, "Downloading \(modelId)...") // No `.safetensors` in additionalFiles: that keeps downloadWeights' // automatic `*.safetensors` glob enabled, which fetches the bundle's @@ -212,7 +221,7 @@ public final class ChatterboxTTSModel { // `.safetensors` here would disable that glob and drop model.safetensors. try await HuggingFaceDownloader.downloadWeights( modelId: modelId, to: bundleDir, - additionalFiles: ["config.json", "tokenizer.json"], + additionalFiles: ["config.json", "tokenizer.json", "Cangjie5_TC.json"], offlineMode: offlineMode ) { progressHandler?($0 * 0.7, "Downloading model...") } } @@ -278,7 +287,7 @@ public final class ChatterboxTTSModel { /// - referenceSamples: reference clip, mono. /// - sampleRate: sample rate of `referenceSamples`. /// - text: text to speak. - /// - languageId: e.g. "en", "ar", "hi", "de", "es", "fr", "it", "pt". + /// - languageId: one of ``MTLTokenizer/supportedLanguages``. /// - exaggeration: emotion-advance scalar (T3 `emotion_adv`). /// - cfgWeight: T3 classifier-free-guidance weight. /// - temperature: T3 sampling temperature (0 = greedy). @@ -297,7 +306,7 @@ public final class ChatterboxTTSModel { cfgWeight: Float = 0.5 ) throws -> [Float] { let lang = languageId.lowercased() - guard MTLTokenizer.frontendFreeLanguages.contains(lang) else { + guard MTLTokenizer.supportedLanguages.contains(lang) else { throw ChatterboxModelError.unsupportedLanguage(lang) } diff --git a/Sources/ChatterboxTTS/MTLTokenizer.swift b/Sources/ChatterboxTTS/MTLTokenizer.swift index 35713e5f..5162a3ab 100644 --- a/Sources/ChatterboxTTS/MTLTokenizer.swift +++ b/Sources/ChatterboxTTS/MTLTokenizer.swift @@ -1,4 +1,6 @@ import Foundation +import CoreFoundation +import NaturalLanguage /// Special tokens of the Chatterbox multilingual grapheme tokenizer. public enum ChatterboxToken { @@ -10,12 +12,13 @@ public enum ChatterboxToken { public enum ChatterboxTokenizerError: Error { case malformedTokenizerJSON + case missingCangjieMap } /// Multilingual grapheme tokenizer for Chatterbox — a self-contained Swift port -/// of the reference `MTLTokenizer`. The encode pipeline for Latin-script languages -/// plus Arabic and Hindi is: lowercase → NFKD → prepend the `[lang]` token → -/// replace spaces with `[SPACE]` → BPE. +/// of the reference `MTLTokenizer`. The common encode pipeline is: +/// lowercase → NFKD → prepend the `[lang]` token → replace spaces with +/// `[SPACE]` → BPE. /// /// The BPE faithfully reproduces the HF `tokenizer.json` (model `BPE`, /// pre-tokenizer `Whitespace`, no normalizer, `[UNK]` fallback): added/control @@ -25,20 +28,39 @@ public enum ChatterboxTokenizerError: Error { /// reference implementation exactly. /// /// Some writing systems additionally need a language-specific text frontend -/// before this grapheme path — see `frontendFreeLanguages` for the set this -/// tokenizer encodes directly. +/// before this grapheme path. Runtime language support is intentionally limited +/// to languages whose frontend is implemented here; unsupported upstream tags +/// stay visible as metadata but are not accepted by synthesis. public final class MTLTokenizer { private let vocab: [String: Int] private let idToToken: [Int: String] private let mergeRank: [String: Int] + private let cangjie: [String: String] + private let cangjieDuplicateIndex: [String: Int] /// (content, id) of added/control tokens, sorted longest content first so the /// scan is greedy-longest-match like the HF tokenizer. private let addedTokens: [(token: [Character], id: Int)] private let unkId: Int - /// Languages this tokenizer encodes directly (no language-specific frontend). - public static let frontendFreeLanguages: Set = - ["en", "ar", "hi", "de", "es", "fr", "it", "pt"] + /// Languages published by Resemble Chatterbox Multilingual V3. + public static let upstreamLanguages: Set = [ + "ar", "da", "de", "el", "en", "es", "fi", "fr", "he", "hi", + "it", "ja", "ko", "ms", "nl", "no", "pl", "pt", "ru", "sv", + "sw", "tr", "zh", + ] + + /// Languages whose text frontend is implemented locally without falling + /// back to raw text. Keep languages with missing external frontends out of + /// this runtime gate: `he` needs Dicta-style diacritization. + public static let supportedLanguages: Set = [ + "ar", "da", "de", "el", "en", "es", "fi", "fr", "hi", "it", + "ja", "ko", "ms", "nl", "no", "pl", "pt", "ru", "sv", "sw", + "tr", "zh", + ] + + /// Backwards-compatible alias retained for callers that used this as a + /// runtime language gate. + public static let frontendFreeLanguages: Set = supportedLanguages private struct TokenizerJSON: Decodable { struct Model: Decodable { @@ -69,6 +91,9 @@ public final class MTLTokenizer { var ranks = [String: Int](minimumCapacity: tj.model.merges.count) for (rank, merge) in tj.model.merges.enumerated() { ranks[merge] = rank } self.mergeRank = ranks + let cangjieMap = try Self.loadCangjieMap(from: modelFolder) + self.cangjie = cangjieMap.codes + self.cangjieDuplicateIndex = cangjieMap.duplicateIndex let unk = tj.model.unk_token ?? ChatterboxToken.unk self.unkId = tj.model.vocab[unk] ?? 1 @@ -82,11 +107,23 @@ public final class MTLTokenizer { /// Encode text to token ids. `languageId` (e.g. "en", "ar", "hi") is prepended /// as a `[lang]` control token, matching the multilingual model's training. public func encode(_ text: String, languageId: String? = nil) -> [Int] { + let lang = languageId?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() var s = text.lowercased() - // NFKD — Swift's compatibility decomposition matches Python `normalize("NFKD", …)`. - s = s.decomposedStringWithCompatibilityMapping - if let lang = languageId, !lang.isEmpty { - s = "[\(lang.lowercased())]" + s + if lang == "ja" { + s = Self.normalizeJapaneseToHiragana(s) + // Japanese reference normalization applies NFKD after hiragana conversion. + s = s.decomposedStringWithCompatibilityMapping + } else { + // NFKD — Swift's compatibility decomposition matches Python `normalize("NFKD", …)`. + s = s.decomposedStringWithCompatibilityMapping + } + if lang == "zh", !cangjie.isEmpty { + s = applyCangjieFrontend(Self.segmentChinese(s)) + } else if lang == "ko" { + s = Self.decomposeHangulToJamo(s) + } + if let lang, !lang.isEmpty { + s = "[\(lang)]" + s } s = s.replacingOccurrences(of: " ", with: ChatterboxToken.space) return tokenize(s) @@ -104,6 +141,162 @@ public final class MTLTokenizer { // MARK: - Internals + private static func loadCangjieMap(from modelFolder: URL) throws -> ( + codes: [String: String], + duplicateIndex: [String: Int] + ) { + let url = modelFolder.appendingPathComponent("Cangjie5_TC.json") + guard let data = try? Data(contentsOf: url), + let rows = try? JSONDecoder().decode([String].self, from: data) else { + throw ChatterboxTokenizerError.missingCangjieMap + } + var codes: [String: String] = [:] + var glyphsByCode: [String: [String]] = [:] + codes.reserveCapacity(rows.count) + for row in rows { + let parts = row.split(separator: "\t", omittingEmptySubsequences: false) + guard parts.count >= 2 else { continue } + let glyph = String(parts[0]) + let code = String(parts[1]) + if codes[glyph] == nil { + codes[glyph] = code + glyphsByCode[code, default: []].append(glyph) + } + } + var duplicateIndex: [String: Int] = [:] + for glyphs in glyphsByCode.values { + for (index, glyph) in glyphs.enumerated() where index > 0 { + duplicateIndex[glyph] = index + } + } + return (codes, duplicateIndex) + } + + private func applyCangjieFrontend(_ text: String) -> String { + var out = "" + out.reserveCapacity(text.count * 4) + for character in text.map(String.init) { + guard let code = cangjie[character] else { + out += character + continue + } + let indexedCode = if let index = cangjieDuplicateIndex[character] { + "\(code)\(index)" + } else { + code + } + for scalar in indexedCode.unicodeScalars { + out += "[cj_\(String(scalar))]" + } + out += "[cj_.]" + } + return out + } + + private static func segmentChinese(_ text: String) -> String { + guard !text.isEmpty else { return text } + let tokenizer = NLTokenizer(unit: .word) + tokenizer.string = text + tokenizer.setLanguage(.simplifiedChinese) + + var tokens: [String] = [] + var cursor = text.startIndex + + func appendGap(_ gap: Substring) { + for scalar in gap.unicodeScalars where !CharacterSet.whitespacesAndNewlines.contains(scalar) { + tokens.append(String(scalar)) + } + } + + tokenizer.enumerateTokens(in: text.startIndex.. String { + guard !text.isEmpty else { return text } + let cfText = text as CFString + let tokenizer = CFStringTokenizerCreate( + nil, + cfText, + CFRangeMake(0, CFStringGetLength(cfText)), + kCFStringTokenizerUnitWord, + Locale(identifier: "ja_JP") as CFLocale + ) + + var out = "" + var cursor = text.startIndex + var tokenType = CFStringTokenizerAdvanceToNextToken(tokenizer) + while tokenType != [] { + let cfRange = CFStringTokenizerGetCurrentTokenRange(tokenizer) + let nsRange = NSRange(location: cfRange.location, length: cfRange.length) + guard let swiftRange = Range(nsRange, in: text) else { + tokenType = CFStringTokenizerAdvanceToNextToken(tokenizer) + continue + } + if cursor < swiftRange.lowerBound { + out += text[cursor.. Bool { + text.unicodeScalars.contains { scalar in + (0x4E00...0x9FFF).contains(scalar.value) + } + } + + private static func decomposeHangulToJamo(_ text: String) -> String { + var out = "" + for scalar in text.unicodeScalars { + let value = scalar.value + guard (0xAC00...0xD7A3).contains(value) else { + out.unicodeScalars.append(scalar) + continue + } + let base = value - 0xAC00 + let initial = 0x1100 + base / (21 * 28) + let medial = 0x1161 + (base % (21 * 28)) / 28 + let final = base % 28 + out.unicodeScalars.append(UnicodeScalar(initial)!) + out.unicodeScalars.append(UnicodeScalar(medial)!) + if final > 0 { + out.unicodeScalars.append(UnicodeScalar(0x11A7 + final)!) + } + } + return out + } + /// Split the preprocessed string by added/control tokens (longest match), then /// pre-tokenize each gap with the Whitespace rule and BPE each word. private func tokenize(_ s: String) -> [Int] { diff --git a/Sources/OmniVoiceTTS/OmniVoiceTTSModel.swift b/Sources/OmniVoiceTTS/OmniVoiceTTSModel.swift index 43b01a5c..a2e4eb49 100644 --- a/Sources/OmniVoiceTTS/OmniVoiceTTSModel.swift +++ b/Sources/OmniVoiceTTS/OmniVoiceTTSModel.swift @@ -8,9 +8,9 @@ import MLXNN /// Qwen3 diffusion backbone, the Higgs-audio v2 codec (encode + decode), and the /// text front-end. This is the entry point the Studio sidecar drives. public final class OmniVoiceTTSModel { - /// Default published bundle (int8 backbone + fp16 codec). The fp16 backbone - /// bundle is `aufklarer/OmniVoice-MLX-fp16`. - public static let defaultModelId = "aufklarer/OmniVoice-MLX-int8" + /// Default published bundle (fp16 backbone + fp16 codec). The int8 backbone + /// remains loadable by passing `aufklarer/OmniVoice-MLX-int8` explicitly. + public static let defaultModelId = OmniVoiceConfig.defaultModelId let cfg: OmniVoiceConfig let model: OmniVoiceModel diff --git a/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift b/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift index 4900803e..0aafaa02 100644 --- a/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift +++ b/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift @@ -4,13 +4,13 @@ import XCTest @testable import ChatterboxTTS /// Numeric gate for the Swift `MTLTokenizer`: token ids must match the -/// reference golden exactly for the frontend-free languages we support first -/// (en / ar / hi). +/// reference golden exactly for core languages, and every runtime-supported +/// language frontend must encode without unknown-token fallback. final class MTLTokenizerTests: XCTestCase { private func loadTokenizer() async throws -> MTLTokenizer { let folder = try await HubApi().snapshot( from: "aufklarer/Chatterbox-Multilingual-MLX-fp16", - matching: ["tokenizer.json"] + matching: ["tokenizer.json", "Cangjie5_TC.json"] ) return try MTLTokenizer(modelFolder: folder) } @@ -37,4 +37,94 @@ final class MTLTokenizerTests: XCTestCase { XCTAssertEqual(tok.encode("a", languageId: "ar").first, 721) XCTAssertEqual(tok.encode("a", languageId: "hi").first, 722) } + + func testLanguageSetsSeparateUpstreamFromRuntimeReadyFrontends() async throws { + let expected: Set = [ + "ar", "da", "de", "el", "en", "es", "fi", "fr", "he", "hi", + "it", "ja", "ko", "ms", "nl", "no", "pl", "pt", "ru", "sv", + "sw", "tr", "zh", + ] + XCTAssertEqual(MTLTokenizer.upstreamLanguages, expected) + XCTAssertEqual(MTLTokenizer.upstreamLanguages.count, 23) + XCTAssertFalse(MTLTokenizer.supportedLanguages.contains("he")) + XCTAssertTrue(MTLTokenizer.supportedLanguages.contains("ja")) + XCTAssertTrue(MTLTokenizer.supportedLanguages.contains("zh")) + + let tok = try await loadTokenizer() + for language in MTLTokenizer.supportedLanguages.sorted() { + let encoded = tok.encode("hello", languageId: language) + XCTAssertFalse(encoded.isEmpty, "language \(language) should encode") + XCTAssertEqual( + tok.decode([encoded[0]]), + "[\(language)]", + "language \(language) should prepend its control token") + } + } + + func testChineseUsesCangjieFrontendWhenMapIsAvailable() async throws { + let tok = try await loadTokenizer() + let encoded = tok.encode("今天", languageId: "zh") + + XCTAssertEqual(encoded.first, 725) + XCTAssertEqual( + tok.decode(Array(encoded.dropFirst())), + "[cj_o][cj_i][cj_n][cj_.][cj_m][cj_k][cj_.]") + XCTAssertFalse(encoded.contains(1), "Cangjie-mapped Chinese should not fall back to [UNK]") + } + + func testChineseFrontendSegmentsWordsBeforeCangjie() async throws { + let tok = try await loadTokenizer() + let encoded = tok.encode("今天我们测试中文语音", languageId: "zh") + let decoded = tok.decode(Array(encoded.dropFirst())) + + XCTAssertTrue(decoded.contains(" "), "Chinese frontend should preserve word boundaries") + XCTAssertTrue(decoded.contains("[cj_"), "Chinese text should be converted to Cangjie tokens") + XCTAssertFalse(encoded.contains(1), "Segmented Chinese should not fall back to [UNK]") + } + + func testJapaneseUsesHiraganaFrontendForKanji() async throws { + let tok = try await loadTokenizer() + let encoded = tok.encode("今日はカタカナです。", languageId: "ja") + + XCTAssertEqual(encoded.first, 723) + XCTAssertTrue( + tok.decode(Array(encoded.dropFirst())).contains("きょうはカタカナです"), + "Kanji should be normalized to hiragana while katakana stays unchanged" + ) + XCTAssertFalse(encoded.contains(1), "Japanese frontend should not fall back to [UNK]") + } + + func testBenchmarkSentencesDoNotEmitUnknownTokens() async throws { + let tok = try await loadTokenizer() + let samples = [ + "ar": "اليوم نختبر ما إذا كان الكلام واضحا وسهل الفهم.", + "da": "I dag tester vi, om talen er klar og let at forstå.", + "de": "Heute testen wir, ob die Sprache klar und leicht zu verstehen ist.", + "el": "Σήμερα ελέγχουμε αν η ομιλία είναι καθαρή και εύκολη στην κατανόηση.", + "en": "Today we test whether the speech is clear and easy to understand.", + "es": "Hoy probamos si el habla es clara y fácil de entender.", + "fi": "Tänään testaamme, onko puhe selkeää ja helppoa ymmärtää.", + "fr": "Aujourd'hui, nous testons si la parole est claire et facile à comprendre.", + "hi": "आज हम जाँचते हैं कि आवाज़ साफ़ और समझने में आसान है या नहीं।", + "it": "Oggi testiamo se il parlato è chiaro e facile da capire.", + "ja": "今日は音声が明瞭で理解しやすいかを確認します。", + "ko": "오늘 우리는 음성이 명확하고 이해하기 쉬운지 테스트합니다.", + "ms": "Hari ini kami menguji sama ada pertuturan jelas dan mudah difahami.", + "nl": "Vandaag testen we of de spraak duidelijk en gemakkelijk te begrijpen is.", + "no": "I dag tester vi om talen er klar og lett å forstå.", + "pl": "Dzisiaj sprawdzamy, czy mowa jest wyraźna i łatwa do zrozumienia.", + "pt": "Hoje testamos se a fala é clara e fácil de entender.", + "ru": "Сегодня мы проверяем, является ли речь четкой и легкой для понимания.", + "sv": "I dag testar vi om talet är tydligt och lätt att förstå.", + "sw": "Leo tunajaribu kama hotuba iko wazi na ni rahisi kuelewa.", + "tr": "Bugün konuşmanın net ve kolay anlaşılır olup olmadığını test ediyoruz.", + "zh": "今天我们测试中文语音是否清楚且容易理解。", + ] + + XCTAssertEqual(Set(samples.keys), MTLTokenizer.supportedLanguages) + for (language, text) in samples { + let encoded = tok.encode(text, languageId: language) + XCTAssertFalse(encoded.contains(1), "\(language) should not emit [UNK]") + } + } } diff --git a/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift b/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift index 3498b99f..4a0fd6d2 100644 --- a/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift +++ b/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift @@ -11,7 +11,7 @@ import XCTest /// and the reference speaker embedding — is buildable today and gated here. /// The full text→audio→verify roundtrip is scaffolded and activates once the T3 /// (text→speech-token) and S3Gen (speech-token→waveform) stages land. -final class RoundtripE2ETests: XCTestCase { +final class E2ERoundtripTests: XCTestCase { private let veWeights = "/tmp/cbx_ve.safetensors" private let veInput = "/tmp/cbx_ve_input16k.f32" @@ -79,8 +79,8 @@ final class RoundtripE2ETests: XCTestCase { to: "/tmp/cbx_swift_synth.wav") } - /// Synthesize across languages (en/ar/hi — the multilingual point of Chatterbox) - /// and write each to /tmp/cbx_swift_.wav for ASR verification. + /// Synthesize across languages, including frontend-heavy scripts, and write + /// each to /tmp/cbx_swift_.wav for ASR verification. func testMultilingualSynthesis() throws { let fm = FileManager.default let bundleDir = "/tmp/cbx-fp16" @@ -101,6 +101,8 @@ final class RoundtripE2ETests: XCTestCase { ("en", "Hello there, this is a cloned voice."), ("ar", "مرحبا، هذا صوت مستنسخ."), ("hi", "नमस्ते, यह एक क्लोन की गई आवाज़ है।"), + ("ja", "今日は音声が明瞭で理解しやすいかを確認します。"), + ("zh", "今天我们测试中文语音是否清楚且容易理解。"), ] for (lang, text) in cases { let audio = try model.clone( diff --git a/Tests/OmniVoiceTTSTests/OmniVoiceModelFacadeTests.swift b/Tests/OmniVoiceTTSTests/OmniVoiceModelFacadeTests.swift index 064d107a..58d0452b 100644 --- a/Tests/OmniVoiceTTSTests/OmniVoiceModelFacadeTests.swift +++ b/Tests/OmniVoiceTTSTests/OmniVoiceModelFacadeTests.swift @@ -6,9 +6,14 @@ import XCTest /// Exercises the high-level `OmniVoiceTTSModel` facade the Studio sidecar drives: /// load a local bundle, then synthesize cloned speech from a reference WAV + text. -/// Writes the waveform for an external ASR check. Uses the int8 bundle so the -/// quantization-from-config load path is covered too. +/// Writes the waveform for an external ASR check. The optional bundle test below +/// still uses int8 so the quantization-from-config load path remains covered. final class OmniVoiceModelFacadeTests: XCTestCase { + func testDefaultModelIdUsesFp16Bundle() { + XCTAssertEqual(OmniVoiceTTSModel.defaultModelId, OmniVoiceConfig.defaultModelId) + XCTAssertEqual(OmniVoiceTTSModel.defaultModelId, "aufklarer/OmniVoice-MLX-fp16") + } + func testFromBundleAndGenerate() async throws { let bundleDir = "/tmp/omnivoice-int8" let refWav = "/tmp/cbx_swift_en.wav" diff --git a/docs/models/chatterbox-tts.md b/docs/models/chatterbox-tts.md index 8a6519fc..dcac3e69 100644 --- a/docs/models/chatterbox-tts.md +++ b/docs/models/chatterbox-tts.md @@ -3,9 +3,10 @@ ## Overview Chatterbox Multilingual (Resemble AI, MIT) is a zero-shot voice-cloning TTS that -synthesizes a target text in the timbre of a short reference clip across 23 -languages — the broadest language coverage of any TTS in this package. The MLX -bundle is a genuine fp16 +synthesizes a target text in the timbre of a short reference clip. The upstream +model publishes 23 language tags; the Swift runtime enables the 22 languages +whose required text frontends are implemented locally. The MLX bundle is a +genuine fp16 conversion (~1.3 GB) of the three upstream checkpoints, published at [`aufklarer/Chatterbox-Multilingual-MLX-fp16`](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16). @@ -14,12 +15,18 @@ speaker embeddings checked against a known-good reference at each stage. ## Languages -23 languages: Arabic, Chinese, Danish, Dutch, English, Finnish, French, German, -Greek, Hebrew, Hindi, Italian, Japanese, Korean, Malay, Norwegian, Polish, -Portuguese, Russian, Spanish, Swahili, Swedish, Turkish. +Runtime-enabled languages: Arabic, Chinese, Danish, Dutch, English, Finnish, +French, German, Greek, Hindi, Italian, Japanese, Korean, Malay, Norwegian, +Polish, Portuguese, Russian, Spanish, Swahili, Swedish, Turkish. Text encodes through a grapheme path: NFKD normalization → a per-language -`[lang]` token → BPE. +`[lang]` token → BPE. Frontend-heavy scripts are handled before that path: + +- Chinese uses word segmentation plus the published Cangjie-5 token map. +- Japanese converts kanji readings to hiragana through `CFStringTokenizer`. +- Korean decomposes Hangul syllables to Jamo. +- Hebrew remains disabled until the runtime includes the diacritization frontend + needed by the upstream model. ## Pipeline Stages From 1c31dd27f993a36f66827964993da6e7ee522a96 Mon Sep 17 00:00:00 2001 From: Ivan Date: Sun, 5 Jul 2026 08:39:31 +0200 Subject: [PATCH 2/2] Add safe Hebrew support for Chatterbox --- README.md | 4 +- README_ar.md | 8 +-- README_de.md | 8 +-- README_es.md | 8 +-- README_fr.md | 8 +-- README_hi.md | 8 +-- README_ja.md | 7 ++- README_ko.md | 7 ++- README_pt.md | 7 ++- README_ru.md | 7 ++- README_th.md | 7 ++- README_tr.md | 7 ++- README_vi.md | 7 ++- README_zh.md | 7 ++- .../ChatterboxTTS/ChatterboxTTSModel.swift | 11 +++- Sources/ChatterboxTTS/MTLTokenizer.swift | 56 +++++++++++++++++-- .../MTLTokenizerTests.swift | 25 ++++++++- .../RoundtripE2ETests.swift | 1 + docs/models/chatterbox-tts.md | 14 ++--- 19 files changed, 146 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index b88a33e2..59139b35 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ On-device speech recognition, synthesis, and understanding for Mac and iOS. Runs - **[VibeVoice TTS](https://soniqo.audio/guides/vibevoice)** — Long-form / multi-speaker TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, up to 90-min podcast/audiobook synthesis, EN/ZH) - **[Magpie TTS](https://soniqo.audio/guides/magpie)** — Multilingual TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB or CoreML INT8 342 MB, 9 languages, 5 baked speakers, streaming on MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — On-device flow-matching TTS (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 languages, 10 voices, G2P-free, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Multilingual TTS with zero-shot voice cloning (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 22 runtime languages, MIT) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Multilingual TTS with zero-shot voice cloning (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23 runtime languages; Hebrew requires niqqud, MIT) - **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — Non-autoregressive diffusion TTS with zero-shot voice cloning (k2-fsa OmniVoice, Qwen3 backbone, MLX fp16 default / int8 available, 600+ languages, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) @@ -168,7 +168,7 @@ Compact view below. **[Full model catalogue with sizes, quantisations, download | [VibeVoice Realtime-0.5B](https://soniqo.audio/guides/vibevoice) | Text → Speech (long-form, multi-speaker) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/guides/vibevoice) | Text → Speech (up to 90-min podcast) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/guides/magpie) | Text → Speech (5 baked speakers, streaming) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML excludes JA) | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Text → Speech (zero-shot cloning) | MLX | 0.8B (fp16) | 22 runtime | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Text → Speech (zero-shot cloning) | MLX | 0.8B (fp16) | 23 (HE requires niqqud) | | [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Text → Speech (NAR diffusion, zero-shot cloning) | MLX | 0.8B (fp16 default / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Text → Speech (zero-shot cloning, explicit style markers) | MLX | 0.5B-class (fp16) | Multilingual | diff --git a/README_ar.md b/README_ar.md index e3dc4883..4f4c1527 100644 --- a/README_ar.md +++ b/README_ar.md @@ -56,8 +56,8 @@ - **[VibeVoice TTS](https://soniqo.audio/ar/guides/vibevoice)** — تحويل النص إلى كلام للنصوص الطويلة / متعدد المتحدثين (Microsoft VibeVoice Realtime-0.5B + 1.5B، MLX، توليد بودكاست/كتب صوتية حتى 90 دقيقة، EN/ZH) - **[Magpie TTS](https://soniqo.audio/ar/guides/magpie)** — تحويل النص إلى كلام متعدد اللغات (NVIDIA Magpie-TTS Multilingual 357M، MLX INT8 411 ميغابايت أو CoreML INT8 342 ميغابايت، 9 لغات، 5 متحدثين جاهزين، تدفق على MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — تحويل النص إلى كلام على الجهاز بمطابقة التدفق (Supertone Supertonic-3 99M، CoreML/Neural Engine، 31 لغة، 10 أصوات، بدون G2P، 44.1 كيلوهرتز) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — تحويل النص إلى كلام متعدد اللغات مع استنساخ الصوت بدون تدريب مسبق (Resemble AI Chatterbox Multilingual، MLX fp16 ~1.3 غيغابايت، 23 لغة، MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — تحويل النص إلى كلام بالانتشار غير الانحداري التلقائي مع استنساخ الصوت بدون تدريب مسبق (k2-fsa OmniVoice، عمود Qwen3 الفقري، MLX int8 ~1 GB / fp16، أكثر من 600 لغة، Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — تحويل النص إلى كلام متعدد اللغات مع استنساخ الصوت بدون تدريب مسبق (Resemble AI Chatterbox Multilingual، MLX fp16 ~1.3 غيغابايت، 23 لغة وقت تشغيل؛ العبرية تتطلب النِّقُود، MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — تحويل النص إلى كلام بالانتشار غير الانحداري التلقائي مع استنساخ الصوت بدون تدريب مسبق (k2-fsa OmniVoice، عمود Qwen3 الفقري، MLX fp16 افتراضي / int8 متاح، أكثر من 600 لغة، Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **نماذج LLM والترجمة** @@ -207,8 +207,8 @@ struct DictateView: View { | [Qwen3-TTS](https://soniqo.audio/ar/guides/speak) | نص → كلام | MLX, CoreML | 0.6B, 1.7B | 10 | | [CosyVoice3](https://soniqo.audio/ar/guides/cosyvoice) | نص → كلام | MLX | 0.5B | 9 | | [VoxCPM2](https://soniqo.audio/ar/speech-generation) | نص → كلام (48 كيلوهرتز، تصميم + استنساخ صوت) | MLX | 2B (bf16/int8) | 30 | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | نص → كلام (استنساخ صوت بدون تدريب) | MLX | 0.8B (fp16) | 23 | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | نص → كلام (انتشار NAR، استنساخ بدون تدريب) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | نص → كلام (استنساخ صوت بدون تدريب) | MLX | 0.8B (fp16) | 23 (HE تتطلب النِّقُود) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | نص → كلام (انتشار NAR، استنساخ بدون تدريب) | MLX | 0.8B (fp16 افتراضي / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | نص → كلام (استنساخ صوت بدون تدريب، علامات نمط صريحة) | MLX | 0.5B-class (fp16) | متعدد اللغات | | [Kokoro-82M](https://soniqo.audio/ar/guides/kokoro) | نص → كلام | CoreML (ANE) | 82M | 10 | diff --git a/README_de.md b/README_de.md index 0a3c126f..856b77b7 100644 --- a/README_de.md +++ b/README_de.md @@ -46,8 +46,8 @@ Spracherkennung, -synthese und -verständnis auf dem Gerät für Mac und iOS. L - **[VibeVoice TTS](https://soniqo.audio/de/guides/vibevoice)** — Langform-/Multi-Speaker-TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, bis zu 90 Min. Podcast-/Hörbuch-Synthese, EN/ZH) - **[Magpie TTS](https://soniqo.audio/de/guides/magpie)** — Mehrsprachiges TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB oder CoreML INT8 342 MB, 9 Sprachen, 5 vordefinierte Sprecher, Streaming auf MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — Flow-Matching-TTS auf dem Gerät (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 Sprachen, 10 Stimmen, G2P-frei, 44,1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Mehrsprachiges TTS mit Zero-Shot-Stimmklonen (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 Sprachen, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — Nicht-autoregressives Diffusions-TTS mit Zero-Shot-Stimmklonen (k2-fsa OmniVoice, Qwen3-Backbone, MLX int8 ~1 GB / fp16, 600+ Sprachen, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Mehrsprachiges TTS mit Zero-Shot-Stimmklonen (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 Laufzeitsprachen; Hebräisch erfordert Nikkud, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — Nicht-autoregressives Diffusions-TTS mit Zero-Shot-Stimmklonen (k2-fsa OmniVoice, Qwen3-Backbone, MLX fp16 Standard / int8 verfügbar, 600+ Sprachen, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLMs und Übersetzung** @@ -163,8 +163,8 @@ Kompakte Übersicht unten. **[Vollständiger Modellkatalog mit Größen, Quantis | [Qwen3-TTS](https://soniqo.audio/de/guides/speak) | Text → Sprache | MLX, CoreML | 0.6B, 1.7B | 10 | | [CosyVoice3](https://soniqo.audio/de/guides/cosyvoice) | Text → Sprache | MLX | 0.5B | 9 | | [VoxCPM2](https://soniqo.audio/de/speech-generation) | Text → Sprache (48 kHz, Voice Design + Klonen) | MLX | 2B (bf16/int8) | 30 | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Text → Sprache (Zero-Shot-Klonen) | MLX | 0.8B (fp16) | 23 | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Text → Sprache (NAR-Diffusion, Zero-Shot-Klonen) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Text → Sprache (Zero-Shot-Klonen) | MLX | 0.8B (fp16) | 23 (HE erfordert Nikkud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Text → Sprache (NAR-Diffusion, Zero-Shot-Klonen) | MLX | 0.8B (fp16 Standard / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Text → Sprache (Zero-Shot-Klonen, explizite Stilmarker) | MLX | 0.5B-class (fp16) | Mehrsprachig | | [Kokoro-82M](https://soniqo.audio/de/guides/kokoro) | Text → Sprache | CoreML (ANE) | 82M | 10 | diff --git a/README_es.md b/README_es.md index 805322b4..1ab39769 100644 --- a/README_es.md +++ b/README_es.md @@ -46,8 +46,8 @@ Reconocimiento, síntesis y comprensión de voz en el dispositivo para Mac e iOS - **[VibeVoice TTS](https://soniqo.audio/es/guides/vibevoice)** — TTS de formato largo / múltiples hablantes (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, síntesis de podcast/audiolibro de hasta 90 min, EN/ZH) - **[Magpie TTS](https://soniqo.audio/es/guides/magpie)** — TTS multilingüe (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB o CoreML INT8 342 MB, 9 idiomas, 5 hablantes preconfigurados, streaming en MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — TTS de flow-matching en el dispositivo (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 idiomas, 10 voces, sin G2P, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS multilingüe con clonación de voz zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 idiomas, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — TTS de difusión no autorregresivo con clonación de voz zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX int8 ~1 GB / fp16, 600+ idiomas, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS multilingüe con clonación de voz zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 idiomas en runtime; hebreo requiere niqqud, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — TTS de difusión no autorregresivo con clonación de voz zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX fp16 por defecto / int8 disponible, 600+ idiomas, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM y traducción** @@ -163,8 +163,8 @@ Vista compacta a continuación. **[Catálogo completo de modelos con tamaños, c | [Qwen3-TTS](https://soniqo.audio/es/guides/speak) | Texto → Voz | MLX, CoreML | 0.6B, 1.7B | 10 | | [CosyVoice3](https://soniqo.audio/es/guides/cosyvoice) | Texto → Voz | MLX | 0.5B | 9 | | [VoxCPM2](https://soniqo.audio/es/speech-generation) | Texto → Voz (48 kHz, diseño de voz + clonación) | MLX | 2B (bf16/int8) | 30 | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Texto → Voz (clonación zero-shot) | MLX | 0.8B (fp16) | 23 | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Texto → Voz (difusión NAR, clonación zero-shot) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Texto → Voz (clonación zero-shot) | MLX | 0.8B (fp16) | 23 (HE requiere niqqud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Texto → Voz (difusión NAR, clonación zero-shot) | MLX | 0.8B (fp16 por defecto / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Texto → Voz (clonación zero-shot, marcadores de estilo explícitos) | MLX | 0.5B-class (fp16) | Multilingüe | | [Kokoro-82M](https://soniqo.audio/es/guides/kokoro) | Texto → Voz | CoreML (ANE) | 82M | 10 | diff --git a/README_fr.md b/README_fr.md index 92dab454..84bdafad 100644 --- a/README_fr.md +++ b/README_fr.md @@ -46,8 +46,8 @@ Reconnaissance, synthese et comprehension vocale embarquees pour Mac et iOS. S'e - **[VibeVoice TTS](https://soniqo.audio/fr/guides/vibevoice)** -- TTS long format / multi-locuteurs (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, synthese de podcast/livre audio jusqu'a 90 min, EN/ZH) - **[Magpie TTS](https://soniqo.audio/fr/guides/magpie)** — TTS multilingue (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 Mo ou CoreML INT8 342 Mo, 9 langues, 5 voix prédéfinies, streaming sur MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — TTS embarque par appariement de flux (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 langues, 10 voix, sans G2P, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS multilingue avec clonage vocal zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 Go, 23 langues, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — TTS à diffusion non autorégressive avec clonage vocal zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX int8 ~1 Go / fp16, 600+ langues, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS multilingue avec clonage vocal zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 Go, 23 langues runtime; l'hébreu exige le niqqud, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — TTS à diffusion non autorégressive avec clonage vocal zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX fp16 par défaut / int8 disponible, 600+ langues, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM et traduction** @@ -163,8 +163,8 @@ Vue compacte ci-dessous. **[Catalogue complet des modeles avec tailles, quantifi | [Qwen3-TTS](https://soniqo.audio/fr/guides/speak) | Texte → Parole | MLX, CoreML | 0.6B, 1.7B | 10 | | [CosyVoice3](https://soniqo.audio/fr/guides/cosyvoice) | Texte → Parole | MLX | 0.5B | 9 | | [VoxCPM2](https://soniqo.audio/fr/speech-generation) | Texte → Parole (48 kHz, conception vocale + clonage) | MLX | 2B (bf16/int8) | 30 | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Texte → Voix (clonage zero-shot) | MLX | 0.8B (fp16) | 23 | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Texte → Voix (diffusion NAR, clonage zero-shot) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Texte → Voix (clonage zero-shot) | MLX | 0.8B (fp16) | 23 (HE exige le niqqud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Texte → Voix (diffusion NAR, clonage zero-shot) | MLX | 0.8B (fp16 par défaut / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Texte → Voix (clonage zero-shot, marqueurs de style explicites) | MLX | 0.5B-class (fp16) | Multilingue | | [Kokoro-82M](https://soniqo.audio/fr/guides/kokoro) | Texte → Parole | CoreML (ANE) | 82M | 10 | diff --git a/README_hi.md b/README_hi.md index d3beedc8..30828bd0 100644 --- a/README_hi.md +++ b/README_hi.md @@ -46,8 +46,8 @@ Mac और iOS के लिए ऑन-डिवाइस स्पीच रि - **[VibeVoice TTS](https://soniqo.audio/hi/guides/vibevoice)** — लंबे-रूप / बहु-वक्ता TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, 90 मिनट तक के पॉडकास्ट / ऑडियोबुक संश्लेषण, EN/ZH) - **[Magpie TTS](https://soniqo.audio/hi/guides/magpie)** — बहुभाषी TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB या CoreML INT8 342 MB, 9 भाषाएँ, 5 पूर्व-निर्धारित वक्ता, MLX पर स्ट्रीमिंग) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — ऑन-डिवाइस फ़्लो-मैचिंग TTS (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 भाषाएँ, 10 वॉयस, G2P-free, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — ज़ीरो-शॉट वॉयस क्लोनिंग के साथ बहुभाषी TTS (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23 भाषाएँ, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — ज़ीरो-शॉट वॉयस क्लोनिंग के साथ नॉन-ऑटोरिग्रेसिव डिफ्यूज़न TTS (k2-fsa OmniVoice, Qwen3 बैकबोन, MLX int8 ~1 GB / fp16, 600+ भाषाएँ, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — ज़ीरो-शॉट वॉयस क्लोनिंग के साथ बहुभाषी TTS (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23 रनटाइम भाषाएँ; हिब्रू के लिए niqqud आवश्यक, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — ज़ीरो-शॉट वॉयस क्लोनिंग के साथ नॉन-ऑटोरिग्रेसिव डिफ्यूज़न TTS (k2-fsa OmniVoice, Qwen3 बैकबोन, MLX fp16 डिफ़ॉल्ट / int8 उपलब्ध, 600+ भाषाएँ, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM और अनुवाद** @@ -163,8 +163,8 @@ struct DictateView: View { | [Qwen3-TTS](https://soniqo.audio/hi/guides/speak) | टेक्स्ट → स्पीच | MLX, CoreML | 0.6B, 1.7B | 10 | | [CosyVoice3](https://soniqo.audio/hi/guides/cosyvoice) | टेक्स्ट → स्पीच | MLX | 0.5B | 9 | | [VoxCPM2](https://soniqo.audio/hi/speech-generation) | टेक्स्ट → स्पीच (48 kHz, वॉयस डिज़ाइन + क्लोनिंग) | MLX | 2B (bf16/int8) | 30 | -| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | टेक्स्ट → स्पीच (ज़ीरो-शॉट क्लोनिंग) | MLX | 0.8B (fp16) | 23 | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | टेक्स्ट → स्पीच (NAR डिफ्यूज़न, ज़ीरो-शॉट क्लोनिंग) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | टेक्स्ट → स्पीच (ज़ीरो-शॉट क्लोनिंग) | MLX | 0.8B (fp16) | 23 (HE के लिए niqqud आवश्यक) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | टेक्स्ट → स्पीच (NAR डिफ्यूज़न, ज़ीरो-शॉट क्लोनिंग) | MLX | 0.8B (fp16 डिफ़ॉल्ट / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | टेक्स्ट → स्पीच (ज़ीरो-शॉट क्लोनिंग, स्पष्ट स्टाइल मार्कर) | MLX | 0.5B-class (fp16) | बहुभाषी | | [Kokoro-82M](https://soniqo.audio/hi/guides/kokoro) | टेक्स्ट → स्पीच | CoreML (ANE) | 82M | 10 | diff --git a/README_ja.md b/README_ja.md index 9022cee4..5d19a360 100644 --- a/README_ja.md +++ b/README_ja.md @@ -46,8 +46,8 @@ Mac・iOS向けのオンデバイス音声認識・合成・理解。Apple Silic - **[VibeVoice TTS](https://soniqo.audio/ja/guides/vibevoice)** — 長尺・マルチスピーカーTTS(Microsoft VibeVoice Realtime-0.5B + 1.5B、MLX、最長90分のポッドキャスト/オーディオブック生成、EN/ZH) - **[Magpie TTS](https://soniqo.audio/ja/guides/magpie)** — 多言語 TTS(NVIDIA Magpie-TTS Multilingual 357M、MLX INT8 411 MB または CoreML INT8 342 MB、9 言語、5 つの組み込みスピーカー、MLX でストリーミング) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — オンデバイスのフローマッチングTTS(Supertone Supertonic-3 99M、CoreML/Neural Engine、31言語、10ボイス、G2P-free、44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — ゼロショット音声クローン対応の多言語TTS(Resemble AI Chatterbox Multilingual、MLX fp16 ~1.3 GB、23言語、MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — ゼロショット音声クローンに対応した非自己回帰拡散TTS(k2-fsa OmniVoice、Qwen3バックボーン、MLX int8 ~1 GB / fp16、600+言語、Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — ゼロショット音声クローン対応の多言語TTS(Resemble AI Chatterbox Multilingual、MLX fp16 ~1.3 GB、23ランタイム言語、ヘブライ語はニクダーが必要、MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — ゼロショット音声クローンに対応した非自己回帰拡散TTS(k2-fsa OmniVoice、Qwen3バックボーン、MLX fp16デフォルト / int8利用可、600+言語、Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM と翻訳** @@ -168,7 +168,8 @@ struct DictateView: View { | [VibeVoice Realtime-0.5B](https://soniqo.audio/ja/guides/vibevoice) | テキスト → 音声(長尺・マルチスピーカー) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/ja/guides/vibevoice) | テキスト → 音声(最長90分のポッドキャスト) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/ja/guides/magpie) | テキスト → 音声(5 つの組み込みスピーカー、ストリーミング) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9(CoreML は日本語を除く) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | テキスト → 音声(NAR 拡散、ゼロショットクローン) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | テキスト → 音声(ゼロショットクローン) | MLX | 0.8B (fp16) | 23(HEはニクダー必須) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | テキスト → 音声(NAR 拡散、ゼロショットクローン) | MLX | 0.8B (fp16デフォルト / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | テキスト → 音声(ゼロショットクローン、明示的スタイルマーカー) | MLX | 0.5B-class (fp16) | 多言語 | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_ko.md b/README_ko.md index 125551a0..7a80838a 100644 --- a/README_ko.md +++ b/README_ko.md @@ -46,8 +46,8 @@ Mac과 iOS를 위한 온디바이스 음성 인식, 합성 및 이해. Apple Sil - **[VibeVoice TTS](https://soniqo.audio/ko/guides/vibevoice)** — 장문 / 멀티 스피커 TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, 최대 90분 팟캐스트 / 오디오북 합성, EN/ZH) - **[Magpie TTS](https://soniqo.audio/ko/guides/magpie)** — 다국어 TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB 또는 CoreML INT8 342 MB, 9개 언어, 5개 내장 스피커, MLX 스트리밍) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — 온디바이스 flow-matching TTS (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31개 언어, 10개 음색, G2P-free, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — 제로샷 음성 복제를 지원하는 다국어 TTS (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23개 언어, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — 제로샷 음성 복제를 지원하는 비자기회귀 확산 TTS (k2-fsa OmniVoice, Qwen3 백본, MLX int8 ~1 GB / fp16, 600+개 언어, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — 제로샷 음성 복제를 지원하는 다국어 TTS (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 런타임 23개 언어, 히브리어는 niqqud 필요, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — 제로샷 음성 복제를 지원하는 비자기회귀 확산 TTS (k2-fsa OmniVoice, Qwen3 백본, MLX fp16 기본 / int8 사용 가능, 600+개 언어, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM 및 번역** @@ -168,7 +168,8 @@ struct DictateView: View { | [VibeVoice Realtime-0.5B](https://soniqo.audio/ko/guides/vibevoice) | 텍스트 → 음성 (장문, 멀티 스피커) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/ko/guides/vibevoice) | 텍스트 → 음성 (최대 90분 팟캐스트) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/ko/guides/magpie) | 텍스트 → 음성 (5개 내장 스피커, 스트리밍) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML은 일본어 제외) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | 텍스트 → 음성 (NAR 확산, 제로샷 복제) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | 텍스트 → 음성 (제로샷 복제) | MLX | 0.8B (fp16) | 23 (HE는 niqqud 필요) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | 텍스트 → 음성 (NAR 확산, 제로샷 복제) | MLX | 0.8B (fp16 기본 / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | 텍스트 → 음성 (제로샷 복제, 명시적 스타일 마커) | MLX | 0.5B-class (fp16) | 다국어 | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_pt.md b/README_pt.md index 2b0b184c..2bc76a7b 100644 --- a/README_pt.md +++ b/README_pt.md @@ -46,8 +46,8 @@ Reconhecimento, sintese e compreensao de fala no dispositivo para Mac e iOS. Exe - **[VibeVoice TTS](https://soniqo.audio/pt/guides/vibevoice)** — TTS de formato longo / multi-alto-falante (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, sintese de podcast/audiolivro de ate 90 min, EN/ZH) - **[Magpie TTS](https://soniqo.audio/pt/guides/magpie)** — TTS multilíngue (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB ou CoreML INT8 342 MB, 9 idiomas, 5 oradores predefinidos, streaming em MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — TTS de flow-matching no dispositivo (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 idiomas, 10 vozes, sem G2P, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS multilíngue com clonagem de voz zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 idiomas, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — TTS de difusao nao-autorregressiva com clonagem de voz zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX int8 ~1 GB / fp16, 600+ idiomas, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS multilíngue com clonagem de voz zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 idiomas em runtime; hebraico requer niqqud, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — TTS de difusao nao-autorregressiva com clonagem de voz zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX fp16 por padrão / int8 disponível, 600+ idiomas, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLMs e tradução** @@ -168,7 +168,8 @@ Vista compacta abaixo. **[Catalogo completo de modelos com tamanhos, quantizacoe | [VibeVoice Realtime-0.5B](https://soniqo.audio/pt/guides/vibevoice) | Texto → Fala (formato longo, multi-alto-falante) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/pt/guides/vibevoice) | Texto → Fala (podcast de ate 90 min) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/pt/guides/magpie) | Texto → Fala (5 oradores predefinidos, streaming) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML exclui JA) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Texto → Fala (difusao NAR, clonagem zero-shot) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Texto → Fala (clonagem zero-shot) | MLX | 0.8B (fp16) | 23 (HE requer niqqud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Texto → Fala (difusao NAR, clonagem zero-shot) | MLX | 0.8B (fp16 por padrão / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Texto → Fala (clonagem zero-shot, marcadores de estilo explicitos) | MLX | 0.5B-class (fp16) | Multilingue | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_ru.md b/README_ru.md index bfe0083e..4582a68a 100644 --- a/README_ru.md +++ b/README_ru.md @@ -46,8 +46,8 @@ - **[VibeVoice TTS](https://soniqo.audio/ru/guides/vibevoice)** — Длинный формат / многоголосый TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, синтез подкастов/аудиокниг до 90 минут, EN/ZH) - **[Magpie TTS](https://soniqo.audio/ru/guides/magpie)** — Многоязычный TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 МБ или CoreML INT8 342 МБ, 9 языков, 5 встроенных голосов, стриминг на MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — Синтез речи на устройстве на основе flow-matching (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 язык, 10 голосов, G2P-free, 44.1 кГц) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Многоязычный TTS с zero-shot клонированием голоса (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 ГБ, 23 языка, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — Неавторегрессионный диффузионный TTS с zero-shot клонированием голоса (k2-fsa OmniVoice, бэкбон Qwen3, MLX int8 ~1 ГБ / fp16, 600+ языков, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Многоязычный TTS с zero-shot клонированием голоса (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 ГБ, 23 языка в рантайме; для иврита нужен никкуд, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — Неавторегрессионный диффузионный TTS с zero-shot клонированием голоса (k2-fsa OmniVoice, бэкбон Qwen3, MLX fp16 по умолчанию / доступен int8, 600+ языков, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM и перевод** @@ -168,7 +168,8 @@ struct DictateView: View { | [VibeVoice Realtime-0.5B](https://soniqo.audio/ru/guides/vibevoice) | Текст → Речь (длинный формат, многоголосый) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/ru/guides/vibevoice) | Текст → Речь (подкаст до 90 минут) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/ru/guides/magpie) | Текст → Речь (5 встроенных голосов, стриминг) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML без JA) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Текст → Речь (NAR-диффузия, zero-shot клонирование) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Текст → Речь (zero-shot клонирование) | MLX | 0.8B (fp16) | 23 (HE нужен никкуд) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Текст → Речь (NAR-диффузия, zero-shot клонирование) | MLX | 0.8B (fp16 по умолчанию / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Текст → Речь (zero-shot клонирование, явные маркеры стиля) | MLX | 0.5B-class (fp16) | Многоязычный | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_th.md b/README_th.md index 2735185a..1a335ed9 100644 --- a/README_th.md +++ b/README_th.md @@ -46,8 +46,8 @@ - **[VibeVoice TTS](https://soniqo.audio/guides/vibevoice)** — TTS แบบยาว / หลายผู้พูด (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX สังเคราะห์พอดแคสต์/หนังสือเสียงได้นานสูงสุด 90 นาที, EN/ZH) - **[Magpie TTS](https://soniqo.audio/guides/magpie)** — TTS หลายภาษา (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB หรือ CoreML INT8 342 MB, 9 ภาษา, 5 ผู้พูดสำเร็จรูป สตรีมมิ่งบน MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — TTS แบบ flow-matching บนอุปกรณ์ (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 ภาษา, 10 เสียง, G2P-free, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS หลายภาษาพร้อมการโคลนเสียงแบบ zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23 ภาษา, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — TTS แบบ diffusion non-autoregressive พร้อมการโคลนเสียงแบบ zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX int8 ~1 GB / fp16, 600+ ภาษา, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS หลายภาษาพร้อมการโคลนเสียงแบบ zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1.3 GB, 23 ภาษาที่รันไทม์; ภาษาฮีบรูต้องมี niqqud, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — TTS แบบ diffusion non-autoregressive พร้อมการโคลนเสียงแบบ zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX fp16 เป็นค่าเริ่มต้น / มี int8, 600+ ภาษา, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM และการแปล** @@ -168,7 +168,8 @@ struct DictateView: View { | [VibeVoice Realtime-0.5B](https://soniqo.audio/guides/vibevoice) | ข้อความ → เสียงพูด (รูปแบบยาว หลายผู้พูด) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/guides/vibevoice) | ข้อความ → เสียงพูด (พอดแคสต์ยาวสุด 90 นาที) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/guides/magpie) | ข้อความ → เสียงพูด (5 ผู้พูดสำเร็จรูป สตรีมมิ่ง) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML ไม่รวม JA) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | ข้อความ → เสียงพูด (diffusion NAR, การโคลนแบบ zero-shot) | MLX | 0.8B (int8/fp16) | 600+ | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | ข้อความ → เสียงพูด (โคลนแบบ zero-shot) | MLX | 0.8B (fp16) | 23 (HE ต้องมี niqqud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | ข้อความ → เสียงพูด (diffusion NAR, การโคลนแบบ zero-shot) | MLX | 0.8B (fp16 ค่าเริ่มต้น / int8) | 600+ | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | ข้อความ → เสียงพูด (การโคลนแบบ zero-shot, marker สไตล์แบบชัดเจน) | MLX | 0.5B-class (fp16) | หลายภาษา | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_tr.md b/README_tr.md index 7f1d88ef..49942861 100644 --- a/README_tr.md +++ b/README_tr.md @@ -46,8 +46,8 @@ Mac ve iOS için cihaz üzerinde konuşma tanıma, sentezleme ve anlama. Apple S - **[VibeVoice TTS](https://soniqo.audio/guides/vibevoice)** — Uzun biçimli / çok konuşmacılı TTS (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, 90 dakikaya kadar podcast/sesli kitap sentezi, EN/ZH) - **[Magpie TTS](https://soniqo.audio/guides/magpie)** — Çok dilli TTS (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB veya CoreML INT8 342 MB, 9 dil, 5 hazır konuşmacı, MLX'te akış) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — Cihaz üzerinde flow-matching TTS (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 dil, 10 ses, G2P-free, 44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Zero-shot ses klonlama özellikli çok dilli TTS (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 dil, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — Zero-shot ses klonlama ile otoregresif olmayan difüzyon TTS (k2-fsa OmniVoice, Qwen3 omurgası, MLX int8 ~1 GB / fp16, 600+ dil, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — Zero-shot ses klonlama özellikli çok dilli TTS (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 çalışma zamanı dili; İbranice için niqqud gerekir, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — Zero-shot ses klonlama ile otoregresif olmayan difüzyon TTS (k2-fsa OmniVoice, Qwen3 omurgası, MLX fp16 varsayılan / int8 kullanılabilir, 600+ dil, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM ve çeviri** @@ -168,7 +168,8 @@ Aşağıda kompakt bir görünüm. **[Boyutlar, kuantizasyonlar, indirme URL'ler | [VibeVoice Realtime-0.5B](https://soniqo.audio/guides/vibevoice) | Metin → Konuşma (uzun biçimli, çok konuşmacılı) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/guides/vibevoice) | Metin → Konuşma (90 dakikaya kadar podcast) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/guides/magpie) | Metin → Konuşma (5 hazır konuşmacı, akış) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML, JA hariç) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Metin → Konuşma (NAR difüzyon, zero-shot klonlama) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Metin → Konuşma (zero-shot klonlama) | MLX | 0.8B (fp16) | 23 (HE için niqqud gerekir) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Metin → Konuşma (NAR difüzyon, zero-shot klonlama) | MLX | 0.8B (fp16 varsayılan / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Metin → Konuşma (zero-shot klonlama, açık stil işaretleri) | MLX | 0.5B-class (fp16) | Çok dilli | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_vi.md b/README_vi.md index 8a08f3e3..8e2555c6 100644 --- a/README_vi.md +++ b/README_vi.md @@ -46,8 +46,8 @@ Nhận dạng, tổng hợp và hiểu giọng nói trên thiết bị cho Mac v - **[VibeVoice TTS](https://soniqo.audio/guides/vibevoice)** — TTS định dạng dài / nhiều người nói (Microsoft VibeVoice Realtime-0.5B + 1.5B, MLX, tổng hợp podcast/sách nói lên đến 90 phút, EN/ZH) - **[Magpie TTS](https://soniqo.audio/guides/magpie)** — TTS đa ngôn ngữ (NVIDIA Magpie-TTS Multilingual 357M, MLX INT8 411 MB hoặc CoreML INT8 342 MB, 9 ngôn ngữ, 5 giọng nói có sẵn, streaming trên MLX) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — TTS flow-matching trên thiết bị (Supertone Supertonic-3 99M, CoreML/Neural Engine, 31 ngôn ngữ, 10 giọng, không cần G2P, 44,1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS đa ngôn ngữ với nhân bản giọng nói zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 ngôn ngữ, MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — TTS khuếch tán phi tự hồi quy với nhân bản giọng nói zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX int8 ~1 GB / fp16, 600+ ngôn ngữ, Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — TTS đa ngôn ngữ với nhân bản giọng nói zero-shot (Resemble AI Chatterbox Multilingual, MLX fp16 ~1,3 GB, 23 ngôn ngữ runtime; tiếng Hebrew cần niqqud, MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — TTS khuếch tán phi tự hồi quy với nhân bản giọng nói zero-shot (k2-fsa OmniVoice, backbone Qwen3, MLX fp16 mặc định / có int8, 600+ ngôn ngữ, Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM và dịch thuật** @@ -168,7 +168,8 @@ Xem tổng quan gọn bên dưới. **[Danh mục mô hình đầy đủ với k | [VibeVoice Realtime-0.5B](https://soniqo.audio/guides/vibevoice) | Văn bản → Giọng nói (định dạng dài, nhiều người nói) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/guides/vibevoice) | Văn bản → Giọng nói (podcast đến 90 phút) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/guides/magpie) | Văn bản → Giọng nói (5 giọng có sẵn, streaming) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9 (CoreML loại trừ JA) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | Văn bản → Giọng nói (khuếch tán NAR, nhân bản zero-shot) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | Văn bản → Giọng nói (nhân bản zero-shot) | MLX | 0.8B (fp16) | 23 (HE cần niqqud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | Văn bản → Giọng nói (khuếch tán NAR, nhân bản zero-shot) | MLX | 0.8B (fp16 mặc định / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | Văn bản → Giọng nói (nhân bản zero-shot, marker phong cách tường minh) | MLX | 0.5B-class (fp16) | Đa ngôn ngữ | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/README_zh.md b/README_zh.md index 02053b5d..3f2ef3a8 100644 --- a/README_zh.md +++ b/README_zh.md @@ -46,8 +46,8 @@ - **[VibeVoice TTS](https://soniqo.audio/zh/guides/vibevoice)** — 长篇 / 多说话人 TTS(Microsoft VibeVoice Realtime-0.5B + 1.5B,MLX,可合成最长 90 分钟的播客 / 有声书,英语 / 中文) - **[Magpie TTS](https://soniqo.audio/zh/guides/magpie)** — 多语言 TTS(NVIDIA Magpie-TTS Multilingual 357M,MLX INT8 411 MB 或 CoreML INT8 342 MB,9 种语言,5 位预设说话人,MLX 端流式) - **[Supertonic TTS](https://soniqo.audio/guides/supertonic)** — 端侧流匹配 TTS(Supertone Supertonic-3 99M,CoreML/神经引擎,31 种语言,10 种音色,G2P-free,44.1 kHz) -- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — 支持零样本声音克隆的多语言 TTS(Resemble AI Chatterbox Multilingual,MLX fp16 ~1.3 GB,23 种语言,MIT) -- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-int8)** — 支持零样本语音克隆的非自回归扩散 TTS(k2-fsa OmniVoice,Qwen3 骨干,MLX int8 ~1 GB / fp16,600+ 种语言,Apache-2.0) +- **[Chatterbox TTS](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16)** — 支持零样本声音克隆的多语言 TTS(Resemble AI Chatterbox Multilingual,MLX fp16 ~1.3 GB,23 种运行时语言;希伯来语需要 niqqud,MIT) +- **[OmniVoice TTS](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16)** — 支持零样本语音克隆的非自回归扩散 TTS(k2-fsa OmniVoice,Qwen3 骨干,MLX fp16 默认 / 可用 int8,600+ 种语言,Apache-2.0) - **[Indic-Mio](docs/models/indic-mio-tts.md)** — Hindi/Indic TTS with inline emotion markers and optional reference-voice cloning (MLX, 24 kHz) **LLM 与翻译** @@ -168,7 +168,8 @@ struct DictateView: View { | [VibeVoice Realtime-0.5B](https://soniqo.audio/zh/guides/vibevoice) | 文本 → 语音(长篇、多说话人) | MLX | 0.5B | EN/ZH | | [VibeVoice 1.5B](https://soniqo.audio/zh/guides/vibevoice) | 文本 → 语音(最长 90 分钟播客) | MLX | 1.5B | EN/ZH | | [Magpie-TTS Multilingual](https://soniqo.audio/zh/guides/magpie) | 文本 → 语音(5 位预设说话人,流式) | MLX / CoreML | 357M (MLX INT8, CoreML INT8) | 9(CoreML 不含日语) | -| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-int8) | 文本 → 语音(NAR 扩散,零样本克隆) | MLX | 0.8B (int8/fp16) | **600+** | +| [Chatterbox Multilingual](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16) | 文本 → 语音(零样本克隆) | MLX | 0.8B (fp16) | 23(HE 需要 niqqud) | +| [OmniVoice](https://huggingface.co/aufklarer/OmniVoice-MLX-fp16) | 文本 → 语音(NAR 扩散,零样本克隆) | MLX | 0.8B (fp16 默认 / int8) | **600+** | | [Indic-Mio](docs/models/indic-mio-tts.md) | Text → Speech (Hindi/Indic, emotion tags, voice cloning) | MLX | fp16 | Hindi / Indic | | [Fish Audio S2 Pro](docs/models/fish-audio-s2-pro.md) | 文本 → 语音(零样本克隆,显式风格标记) | MLX | 0.5B-class (fp16) | 多语言 | | [Qwen3.5 Chat](docs/models/qwen35-chat.md) | Text → Text (LLM) | MLX, CoreML | 0.8B | Multi | diff --git a/Sources/ChatterboxTTS/ChatterboxTTSModel.swift b/Sources/ChatterboxTTS/ChatterboxTTSModel.swift index 66508d6c..e82768e0 100644 --- a/Sources/ChatterboxTTS/ChatterboxTTSModel.swift +++ b/Sources/ChatterboxTTS/ChatterboxTTSModel.swift @@ -19,12 +19,15 @@ import MLXNN public enum ChatterboxModelError: Error, LocalizedError { case missingFile(String) case unsupportedLanguage(String) + case invalidText(String) public var errorDescription: String? { switch self { case let .missingFile(p): return "Chatterbox: required file not found: \(p)" case let .unsupportedLanguage(l): return "Chatterbox: language '\(l)' is not supported" + case let .invalidText(message): + return "Chatterbox: \(message)" } } } @@ -333,7 +336,13 @@ public final class ChatterboxTTSModel { let t3PromptTokens = Array(s3gen.tokenizer.encode(ref16kEnc).prefix(Self.speechCondPromptLen)) // --- text tokenization: [sot] + ids + [eot] --- - let ids = tokenizer.encode(text, languageId: lang) + let ids: [Int] + do { + ids = try tokenizer.encodeStrict(text, languageId: lang) + } catch ChatterboxTokenizerError.hebrewRequiresDiacritics { + throw ChatterboxModelError.invalidText( + "Hebrew input must include niqqud/diacritics; automatic Dicta diacritization is not bundled yet") + } let textTokens = [Self.startTextToken] + ids + [Self.stopTextToken] // --- T3: text -> speech tokens --- diff --git a/Sources/ChatterboxTTS/MTLTokenizer.swift b/Sources/ChatterboxTTS/MTLTokenizer.swift index 5162a3ab..40d3b22a 100644 --- a/Sources/ChatterboxTTS/MTLTokenizer.swift +++ b/Sources/ChatterboxTTS/MTLTokenizer.swift @@ -10,9 +10,21 @@ public enum ChatterboxToken { public static let space = "[SPACE]" } -public enum ChatterboxTokenizerError: Error { +public enum ChatterboxTokenizerError: Error, LocalizedError { case malformedTokenizerJSON case missingCangjieMap + case hebrewRequiresDiacritics + + public var errorDescription: String? { + switch self { + case .malformedTokenizerJSON: + return "Malformed Chatterbox tokenizer.json" + case .missingCangjieMap: + return "Chatterbox tokenizer is missing Cangjie5_TC.json" + case .hebrewRequiresDiacritics: + return "Chatterbox Hebrew input must include niqqud/diacritics; automatic diacritization is not bundled yet" + } + } } /// Multilingual grapheme tokenizer for Chatterbox — a self-contained Swift port @@ -50,11 +62,11 @@ public final class MTLTokenizer { ] /// Languages whose text frontend is implemented locally without falling - /// back to raw text. Keep languages with missing external frontends out of - /// this runtime gate: `he` needs Dicta-style diacritization. + /// back to raw text. Hebrew is accepted only when the caller supplies + /// pre-diacritized text; automatic Dicta ONNX diacritization is not ported. public static let supportedLanguages: Set = [ - "ar", "da", "de", "el", "en", "es", "fi", "fr", "hi", "it", - "ja", "ko", "ms", "nl", "no", "pl", "pt", "ru", "sv", "sw", + "ar", "da", "de", "el", "en", "es", "fi", "fr", "he", "hi", + "it", "ja", "ko", "ms", "nl", "no", "pl", "pt", "ru", "sv", "sw", "tr", "zh", ] @@ -129,6 +141,19 @@ public final class MTLTokenizer { return tokenize(s) } + /// Strict encode path used by synthesis. It preserves `encode`'s legacy + /// behaviour for tests/debugging, but rejects Hebrew text that has not been + /// pre-diacritized; upstream relies on Dicta before tokenization for this. + public func encodeStrict(_ text: String, languageId: String? = nil) throws -> [Int] { + let lang = languageId?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + if lang == "he", + Self.containsHebrewLetters(text), + !Self.containsHebrewDiacritics(text) { + throw ChatterboxTokenizerError.hebrewRequiresDiacritics + } + return encode(text, languageId: languageId) + } + /// Inverse of `encode` for debugging. public func decode(_ ids: [Int]) -> String { var s = ids.map { idToToken[$0] ?? "" }.joined() @@ -276,6 +301,27 @@ public final class MTLTokenizer { } } + public static func containsHebrewLetters(_ text: String) -> Bool { + text.unicodeScalars.contains { scalar in + (0x05D0...0x05EA).contains(scalar.value) + } + } + + public static func containsHebrewDiacritics(_ text: String) -> Bool { + text.unicodeScalars.contains { scalar in + switch scalar.value { + case 0x0591...0x05BD, 0x05BF, 0x05C1...0x05C2, 0x05C4...0x05C5, 0x05C7: + return true + default: + return false + } + } + } + + public static func isHebrewTextDiacritized(_ text: String) -> Bool { + !containsHebrewLetters(text) || containsHebrewDiacritics(text) + } + private static func decomposeHangulToJamo(_ text: String) -> String { var out = "" for scalar in text.unicodeScalars { diff --git a/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift b/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift index 0aafaa02..6c3a797c 100644 --- a/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift +++ b/Tests/ChatterboxTTSTests/MTLTokenizerTests.swift @@ -46,7 +46,9 @@ final class MTLTokenizerTests: XCTestCase { ] XCTAssertEqual(MTLTokenizer.upstreamLanguages, expected) XCTAssertEqual(MTLTokenizer.upstreamLanguages.count, 23) - XCTAssertFalse(MTLTokenizer.supportedLanguages.contains("he")) + XCTAssertEqual(MTLTokenizer.supportedLanguages, expected) + XCTAssertEqual(MTLTokenizer.supportedLanguages.count, 23) + XCTAssertTrue(MTLTokenizer.supportedLanguages.contains("he")) XCTAssertTrue(MTLTokenizer.supportedLanguages.contains("ja")) XCTAssertTrue(MTLTokenizer.supportedLanguages.contains("zh")) @@ -105,6 +107,7 @@ final class MTLTokenizerTests: XCTestCase { "es": "Hoy probamos si el habla es clara y fácil de entender.", "fi": "Tänään testaamme, onko puhe selkeää ja helppoa ymmärtää.", "fr": "Aujourd'hui, nous testons si la parole est claire et facile à comprendre.", + "he": "הַיּוֹם אֲנַחְנוּ בּוֹדְקִים אִם הַדִּבּוּר בָּרוּר וְקַל לַהֲבָנָה.", "hi": "आज हम जाँचते हैं कि आवाज़ साफ़ और समझने में आसान है या नहीं।", "it": "Oggi testiamo se il parlato è chiaro e facile da capire.", "ja": "今日は音声が明瞭で理解しやすいかを確認します。", @@ -123,8 +126,26 @@ final class MTLTokenizerTests: XCTestCase { XCTAssertEqual(Set(samples.keys), MTLTokenizer.supportedLanguages) for (language, text) in samples { - let encoded = tok.encode(text, languageId: language) + let encoded = try tok.encodeStrict(text, languageId: language) XCTAssertFalse(encoded.contains(1), "\(language) should not emit [UNK]") } } + + func testHebrewRequiresDiacriticsForStrictEncoding() async throws { + let tok = try await loadTokenizer() + + XCTAssertThrowsError(try tok.encodeStrict("שלום עולם", languageId: "he")) { error in + guard case ChatterboxTokenizerError.hebrewRequiresDiacritics = error else { + return XCTFail("expected Hebrew diacritics error, got \(error)") + } + } + } + + func testHebrewDiacriticsEncodeWithoutUnknownTokens() async throws { + let tok = try await loadTokenizer() + let encoded = try tok.encodeStrict("שָׁלוֹם עוֹלָם", languageId: "he") + + XCTAssertEqual(encoded.first, 2110) + XCTAssertFalse(encoded.contains(1), "diacritized Hebrew should not emit [UNK]") + } } diff --git a/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift b/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift index 4a0fd6d2..6b89df91 100644 --- a/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift +++ b/Tests/ChatterboxTTSTests/RoundtripE2ETests.swift @@ -100,6 +100,7 @@ final class E2ERoundtripTests: XCTestCase { let cases: [(String, String)] = [ ("en", "Hello there, this is a cloned voice."), ("ar", "مرحبا، هذا صوت مستنسخ."), + ("he", "שָׁלוֹם, זֶה קוֹל מְשֻׁבָּט."), ("hi", "नमस्ते, यह एक क्लोन की गई आवाज़ है।"), ("ja", "今日は音声が明瞭で理解しやすいかを確認します。"), ("zh", "今天我们测试中文语音是否清楚且容易理解。"), diff --git a/docs/models/chatterbox-tts.md b/docs/models/chatterbox-tts.md index dcac3e69..d0aa06bd 100644 --- a/docs/models/chatterbox-tts.md +++ b/docs/models/chatterbox-tts.md @@ -4,10 +4,10 @@ Chatterbox Multilingual (Resemble AI, MIT) is a zero-shot voice-cloning TTS that synthesizes a target text in the timbre of a short reference clip. The upstream -model publishes 23 language tags; the Swift runtime enables the 22 languages -whose required text frontends are implemented locally. The MLX bundle is a -genuine fp16 -conversion (~1.3 GB) of the three upstream checkpoints, published at +model publishes 23 language tags; the Swift runtime enables all 23, with Hebrew +requiring pre-diacritized text (niqqud) until automatic Dicta ONNX +diacritization is bundled. The MLX bundle is a genuine fp16 conversion +(~1.3 GB) of the three upstream checkpoints, published at [`aufklarer/Chatterbox-Multilingual-MLX-fp16`](https://huggingface.co/aufklarer/Chatterbox-Multilingual-MLX-fp16). The Swift port is built component-by-component, with token ids, mel features, and @@ -16,7 +16,7 @@ speaker embeddings checked against a known-good reference at each stage. ## Languages Runtime-enabled languages: Arabic, Chinese, Danish, Dutch, English, Finnish, -French, German, Greek, Hindi, Italian, Japanese, Korean, Malay, Norwegian, +French, German, Greek, Hebrew, Hindi, Italian, Japanese, Korean, Malay, Norwegian, Polish, Portuguese, Russian, Spanish, Swahili, Swedish, Turkish. Text encodes through a grapheme path: NFKD normalization → a per-language @@ -25,8 +25,8 @@ Text encodes through a grapheme path: NFKD normalization → a per-language - Chinese uses word segmentation plus the published Cangjie-5 token map. - Japanese converts kanji readings to hiragana through `CFStringTokenizer`. - Korean decomposes Hangul syllables to Jamo. -- Hebrew remains disabled until the runtime includes the diacritization frontend - needed by the upstream model. +- Hebrew accepts pre-diacritized input with niqqud. Raw Hebrew is rejected with a + clear error until the runtime includes automatic Dicta ONNX diacritization. ## Pipeline Stages