diff --git a/cloud/.env.example b/cloud/.env.example index b352d442e3..80e432daf5 100644 --- a/cloud/.env.example +++ b/cloud/.env.example @@ -50,6 +50,7 @@ AZURE_SPEECH_KEY= # Soniox Speech SONIOX_API_KEY= +SONIOX_FALLBACK_API_KEYS= # ============================================================================= # LLM Configuration diff --git a/cloud/issues/108-soniox-fallback-api-keys/spec.md b/cloud/issues/108-soniox-fallback-api-keys/spec.md new file mode 100644 index 0000000000..3adb631417 --- /dev/null +++ b/cloud/issues/108-soniox-fallback-api-keys/spec.md @@ -0,0 +1,42 @@ +# Spec: Soniox Fallback API Keys + +## Environment + +Existing: + +```bash +SONIOX_API_KEY=primary-key +``` + +New: + +```bash +SONIOX_FALLBACK_API_KEYS=fallback-key-a,fallback-key-b,fallback-key-c +``` + +`SONIOX_API_KEY` remains the preferred primary credential. Fallback keys are +comma-separated, trimmed, and deduplicated. Empty entries are ignored. + +## Runtime Behavior + +1. New transcription stream creation first tries the primary key when it is not + cooling down. +2. If the primary key is unavailable or stream creation fails with a + credential/limit/provider error, stream creation tries fallback keys. +3. Fallback keys are chosen round-robin among keys that are not cooling down. +4. No local max-concurrent accounting is used. +5. Errors are classified into cooldown classes: + - concurrent stream limit: very short cooldown + - request/rate limit: short cooldown + - spend/account quota: long cooldown + - invalid key/authentication: disabled for this process + - transient/network/server: short cooldown +6. Logs include credential fingerprints only. Raw API keys must never be logged. +7. Existing transcription and translation retry behavior remains in place. A + retry should create a new stream, which reselects a Soniox key from the pool. + +## Non-Goals + +- Per-key concurrency env vars. +- Cross-pod key usage coordination. +- New external state such as Redis for quota tracking. diff --git a/cloud/issues/108-soniox-fallback-api-keys/spike.md b/cloud/issues/108-soniox-fallback-api-keys/spike.md new file mode 100644 index 0000000000..0304d21c21 --- /dev/null +++ b/cloud/issues/108-soniox-fallback-api-keys/spike.md @@ -0,0 +1,45 @@ +# Spike: Soniox Fallback API Keys + +## Problem + +Legacy Cloud v1 currently constructs Soniox transcription and translation +providers from `SONIOX_API_KEY`. When that Soniox org/key hits an account-level +limit, every new Soniox transcription or translation stream in production fails +through the same exhausted credential. The existing provider fallback machinery +only switches between provider types. It does not support multiple Soniox +credentials. + +## Observed Code Path + +- `cloud/packages/cloud/src/services/session/transcription/types.ts` + reads `SONIOX_API_KEY` into `DEFAULT_TRANSCRIPTION_CONFIG.soniox.apiKey`. +- `SonioxTranscriptionProvider` initializes one Soniox SDK client with that key. +- `TranscriptionManager` creates one Soniox provider for non-China deployments. +- Stream retry logic retries the same Soniox provider/key after 429, 408, and + server errors. +- `cloud/packages/cloud/src/services/session/translation/types.ts` also reads + `SONIOX_API_KEY` into `DEFAULT_TRANSLATION_CONFIG.soniox.apiKey`. +- `TranslationManager` retries translation streams, but without multiple Soniox + credentials it retries the same exhausted key. + +## Important Constraint + +Do not configure local max-concurrent limits per key. Soniox keys may be shared +across pods or environments, so a local counter is incomplete and can make a key +look available when another process already consumed its concurrency quota. The +source of truth is Soniox accepting or rejecting a stream. + +## Failure Classes + +- Spend or account quota exhausted: long cooldown. This may not recover until + billing quota resets or the org is changed. +- Request rate limited: short cooldown. +- Concurrent stream limit: very short cooldown. Capacity may return as soon as + another stream closes, possibly in another process. +- Invalid/auth key: disable for this process. +- Network/server/transient errors: short cooldown. + +## Scope + +This hotfix targets Cloud v1 transcription and translation streams that use +Soniox. diff --git a/cloud/packages/cloud/src/services/session/soniox/SonioxKeyPool.ts b/cloud/packages/cloud/src/services/session/soniox/SonioxKeyPool.ts new file mode 100644 index 0000000000..1afce58269 --- /dev/null +++ b/cloud/packages/cloud/src/services/session/soniox/SonioxKeyPool.ts @@ -0,0 +1,251 @@ +import crypto from "crypto"; + +export type SonioxCredentialRole = "primary" | "fallback"; + +export interface SonioxCredential { + id: string; + apiKey: string; + role: SonioxCredentialRole; +} + +type SonioxCredentialFailureKind = + | "auth" + | "concurrency" + | "quota" + | "rate_limit" + | "transient"; + +interface SonioxCredentialState extends SonioxCredential { + cooldownUntil: number; + disabled: boolean; + failureKind?: SonioxCredentialFailureKind; + lastFailureMessage?: string; +} + +export interface SonioxCredentialFailureClassification { + kind: SonioxCredentialFailureKind; + cooldownMs: number; + disabled?: boolean; +} + +const CONCURRENCY_COOLDOWN_MS = 5_000; +const RATE_LIMIT_COOLDOWN_MS = 60_000; +const QUOTA_COOLDOWN_MS = 30 * 60_000; +const TRANSIENT_COOLDOWN_MS = 10_000; +const sharedPools = new Map(); + +export function parseSonioxFallbackApiKeys(value: string | undefined): string[] { + if (!value) return []; + return value + .split(",") + .map((key) => key.trim()) + .filter(Boolean); +} + +export function fingerprintSonioxKey(apiKey: string): string { + return crypto.createHash("sha256").update(apiKey).digest("hex").slice(0, 12); +} + +export function classifySonioxCredentialFailure(error: Error): SonioxCredentialFailureClassification { + const message = error.message || ""; + const lower = message.toLowerCase(); + const code = extractSonioxErrorCode(message); + + if ( + code === 401 || + lower.includes("invalid api key") || + lower.includes("invalid_api_key") || + lower.includes("bad api key") || + lower.includes("unauthorized") + ) { + return { kind: "auth", cooldownMs: Number.POSITIVE_INFINITY, disabled: true }; + } + + if ( + lower.includes("concurrent") || + lower.includes("concurrency") || + lower.includes("connection limit") || + lower.includes("stream limit") || + lower.includes("too many streams") || + lower.includes("maximum streams") || + lower.includes("max streams") + ) { + return { kind: "concurrency", cooldownMs: CONCURRENCY_COOLDOWN_MS }; + } + + if ( + code === 429 || + lower.includes("rate limit") || + lower.includes("rate_limit") || + lower.includes("too many requests") + ) { + return { kind: "rate_limit", cooldownMs: RATE_LIMIT_COOLDOWN_MS }; + } + + if ( + code === 402 || + /\bquota\b/.test(lower) || + /\bbudget\b/.test(lower) || + /\bcredit(?:s)?\b/.test(lower) || + /\bbilling\b/.test(lower) || + /\bspend(?:ing)?\b/.test(lower) || + /\bbalance\b/.test(lower) || + lower.includes("usage limit") || + lower.includes("monthly limit") + ) { + return { kind: "quota", cooldownMs: QUOTA_COOLDOWN_MS }; + } + + return { kind: "transient", cooldownMs: TRANSIENT_COOLDOWN_MS }; +} + +export function getSharedSonioxKeyPool(primaryApiKey: string, fallbackApiKeys: string[] = []): SonioxKeyPool { + const poolKey = [ + fingerprintSonioxKey(primaryApiKey.trim()), + ...fallbackApiKeys.map((key) => key.trim()).filter(Boolean).map(fingerprintSonioxKey), + ].join(":"); + + const existing = sharedPools.get(poolKey); + if (existing) return existing; + + const pool = new SonioxKeyPool(primaryApiKey, fallbackApiKeys); + sharedPools.set(poolKey, pool); + return pool; +} + +export function resetSharedSonioxKeyPoolsForTests(): void { + sharedPools.clear(); +} + +export class SonioxKeyPool { + private credentials: SonioxCredentialState[]; + private nextFallbackIndex = 0; + + constructor(primaryApiKey: string, fallbackApiKeys: string[] = []) { + const seen = new Set(); + const credentials: SonioxCredentialState[] = []; + + const addCredential = (apiKey: string, role: SonioxCredentialRole): void => { + const trimmed = apiKey.trim(); + if (!trimmed || seen.has(trimmed)) return; + seen.add(trimmed); + credentials.push({ + id: fingerprintSonioxKey(trimmed), + apiKey: trimmed, + role, + cooldownUntil: 0, + disabled: false, + }); + }; + + addCredential(primaryApiKey, "primary"); + for (const key of fallbackApiKeys) { + addCredential(key, "fallback"); + } + + this.credentials = credentials; + } + + get size(): number { + return this.credentials.length; + } + + get hasFallbacks(): boolean { + return this.credentials.some((credential) => credential.role === "fallback"); + } + + selectCredential(attempted = new Set(), now = Date.now()): SonioxCredential | null { + const primary = this.credentials.find((credential) => credential.role === "primary"); + if (primary && !attempted.has(primary.id) && this.isAvailable(primary, now)) { + return this.toPublicCredential(primary); + } + + const fallbackCredentials = this.credentials.filter((credential) => credential.role === "fallback"); + if (fallbackCredentials.length === 0) return null; + + for (let offset = 0; offset < fallbackCredentials.length; offset++) { + const index = (this.nextFallbackIndex + offset) % fallbackCredentials.length; + const credential = fallbackCredentials[index]; + if (attempted.has(credential.id) || !this.isAvailable(credential, now)) continue; + + this.nextFallbackIndex = (index + 1) % fallbackCredentials.length; + return this.toPublicCredential(credential); + } + + return null; + } + + recordSuccess(credentialId: string, now = Date.now()): void { + const credential = this.findCredential(credentialId); + if (!credential || credential.disabled) return; + if (credential.cooldownUntil > now) return; + credential.cooldownUntil = 0; + credential.failureKind = undefined; + credential.lastFailureMessage = undefined; + } + + recordFailure(credentialId: string, error: Error, now = Date.now()): SonioxCredentialFailureClassification | null { + const credential = this.findCredential(credentialId); + if (!credential) return null; + + const classification = classifySonioxCredentialFailure(error); + credential.failureKind = classification.kind; + credential.lastFailureMessage = error.message; + + if (classification.disabled) { + credential.disabled = true; + credential.cooldownUntil = Number.POSITIVE_INFINITY; + } else { + credential.cooldownUntil = Math.max( + credential.cooldownUntil, + now + classification.cooldownMs, + ); + } + + return classification; + } + + describeAvailability(now = Date.now()): Array<{ + id: string; + role: SonioxCredentialRole; + available: boolean; + disabled: boolean; + cooldownRemainingMs: number; + failureKind?: SonioxCredentialFailureKind; + }> { + return this.credentials.map((credential) => ({ + id: credential.id, + role: credential.role, + available: this.isAvailable(credential, now), + disabled: credential.disabled, + cooldownRemainingMs: + credential.cooldownUntil === Number.POSITIVE_INFINITY + ? Number.POSITIVE_INFINITY + : Math.max(0, credential.cooldownUntil - now), + failureKind: credential.failureKind, + })); + } + + private findCredential(credentialId: string): SonioxCredentialState | undefined { + return this.credentials.find((credential) => credential.id === credentialId); + } + + private isAvailable(credential: SonioxCredentialState, now: number): boolean { + return !credential.disabled && credential.cooldownUntil <= now; + } + + private toPublicCredential(credential: SonioxCredentialState): SonioxCredential { + return { + id: credential.id, + apiKey: credential.apiKey, + role: credential.role, + }; + } +} + +function extractSonioxErrorCode(message: string): number | null { + const match = message.match(/Soniox error (\d+):/i); + if (!match) return null; + const parsed = Number.parseInt(match[1], 10); + return Number.isFinite(parsed) ? parsed : null; +} diff --git a/cloud/packages/cloud/src/services/session/soniox/__tests__/SonioxKeyPool.test.ts b/cloud/packages/cloud/src/services/session/soniox/__tests__/SonioxKeyPool.test.ts new file mode 100644 index 0000000000..e733d50c29 --- /dev/null +++ b/cloud/packages/cloud/src/services/session/soniox/__tests__/SonioxKeyPool.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "bun:test"; + +import { + SonioxKeyPool, + classifySonioxCredentialFailure, + getSharedSonioxKeyPool, + parseSonioxFallbackApiKeys, + resetSharedSonioxKeyPoolsForTests, +} from "../SonioxKeyPool"; +import { SONIOX_MODEL as TRANSCRIPTION_SONIOX_MODEL } from "../../transcription/types"; +import { SONIOX_MODEL as TRANSLATION_SONIOX_MODEL } from "../../translation/types"; + +describe("SonioxKeyPool", () => { + it("defaults legacy Soniox transcription and translation to the current real-time model", () => { + expect(TRANSCRIPTION_SONIOX_MODEL).toBe("stt-rt-v5"); + expect(TRANSLATION_SONIOX_MODEL).toBe("stt-rt-v5"); + }); + + it("parses comma-separated fallback keys", () => { + expect(parseSonioxFallbackApiKeys(" a, b ,, c ")).toEqual(["a", "b", "c"]); + expect(parseSonioxFallbackApiKeys(undefined)).toEqual([]); + }); + + it("prefers the primary key while available", () => { + const pool = new SonioxKeyPool("primary", ["fallback-a", "fallback-b"]); + + const credential = pool.selectCredential(new Set(), 1000); + + expect(credential?.role).toBe("primary"); + }); + + it("round-robins fallback keys when primary is cooling down", () => { + const pool = new SonioxKeyPool("primary", ["fallback-a", "fallback-b"]); + const primary = pool.selectCredential(new Set(), 1000)!; + pool.recordFailure(primary.id, new Error("Soniox error 429: rate limit"), 1000); + + const first = pool.selectCredential(new Set(), 1000); + const second = pool.selectCredential(new Set(), 1000); + const third = pool.selectCredential(new Set(), 1000); + + expect(first?.role).toBe("fallback"); + expect(second?.role).toBe("fallback"); + expect(third?.role).toBe("fallback"); + expect(first?.id).not.toBe(second?.id); + expect(third?.id).toBe(first?.id); + }); + + it("deduplicates fallback keys that match the primary", () => { + const pool = new SonioxKeyPool("primary", ["primary", "fallback"]); + + expect(pool.size).toBe(2); + }); + + it("makes concurrency failures available again after a short cooldown", () => { + const pool = new SonioxKeyPool("primary", ["fallback"]); + const primary = pool.selectCredential(new Set(), 1000)!; + pool.recordFailure(primary.id, new Error("Soniox error 429: maximum concurrent streams reached"), 1000); + + expect(pool.selectCredential(new Set(), 1000)?.role).toBe("fallback"); + expect(pool.selectCredential(new Set(), 6_001)?.role).toBe("primary"); + }); + + it("disables invalid keys for the process", () => { + const pool = new SonioxKeyPool("primary", ["fallback"]); + const primary = pool.selectCredential(new Set(), 1000)!; + pool.recordFailure(primary.id, new Error("Soniox error 401: invalid api key"), 1000); + + const availability = pool.describeAvailability(10_000).find((item) => item.id === primary.id); + + expect(availability?.disabled).toBe(true); + expect(availability?.available).toBe(false); + expect(pool.selectCredential(new Set(), 10_000)?.role).toBe("fallback"); + }); + + it("shares cooldown state for the same configured credentials", () => { + resetSharedSonioxKeyPoolsForTests(); + const first = getSharedSonioxKeyPool("primary", ["fallback"]); + const second = getSharedSonioxKeyPool("primary", ["fallback"]); + + const primary = first.selectCredential(new Set(), 1000)!; + first.recordFailure(primary.id, new Error("Soniox error 402: Organization monthly budget exhausted"), 1000); + + expect(second.selectCredential(new Set(), 1000)?.role).toBe("fallback"); + }); + + it("does not let overlapping success clear active cooldown", () => { + const pool = new SonioxKeyPool("primary", ["fallback"]); + const primary = pool.selectCredential(new Set(), 1000)!; + pool.recordFailure(primary.id, new Error("Soniox error 402: Organization monthly budget exhausted"), 1000); + + pool.recordSuccess(primary.id, 2000); + + const availability = pool.describeAvailability(2000).find((item) => item.id === primary.id); + expect(availability?.failureKind).toBe("quota"); + expect(availability?.available).toBe(false); + expect(pool.selectCredential(new Set(), 2000)?.role).toBe("fallback"); + }); +}); + +describe("classifySonioxCredentialFailure", () => { + it("classifies quota exhaustion separately from request rate limits", () => { + expect(classifySonioxCredentialFailure(new Error("Monthly quota exceeded")).kind).toBe("quota"); + expect(classifySonioxCredentialFailure(new Error("Soniox error 402: Organization monthly budget exhausted")).kind).toBe( + "quota", + ); + expect(classifySonioxCredentialFailure(new Error("Soniox error 429: rate limit exhausted")).kind).toBe( + "rate_limit", + ); + }); + + it("treats concurrent stream errors as temporary capacity errors", () => { + expect(classifySonioxCredentialFailure(new Error("Too many concurrent streams")).kind).toBe("concurrency"); + }); +}); diff --git a/cloud/packages/cloud/src/services/session/transcription/TranscriptionManager.ts b/cloud/packages/cloud/src/services/session/transcription/TranscriptionManager.ts index 28e116affb..9140786d08 100644 --- a/cloud/packages/cloud/src/services/session/transcription/TranscriptionManager.ts +++ b/cloud/packages/cloud/src/services/session/transcription/TranscriptionManager.ts @@ -21,6 +21,7 @@ import { estimateArrayBufferBytes, sumEstimatedBytes } from "../../metrics/memor import UserSession from "../UserSession"; import { AlibabaTranscriptionProvider } from "./providers/AlibabaTranscriptionProvider"; +import { classifySonioxCredentialFailure } from "../soniox/SonioxKeyPool"; import { SonioxTranscriptionProvider } from "./providers/SonioxTranscriptionProvider"; import { ProviderSelector } from "./ProviderSelector"; import { @@ -1615,8 +1616,26 @@ export class TranscriptionManager { return false; } + if (error.message.includes("No available Soniox credentials")) { + this.logger.warn({ message: error.message }, "Soniox credentials cooling down - retrying"); + return true; + } + // Soniox-specific error handling if (error.message.includes("Soniox error")) { + const credentialFailure = classifySonioxCredentialFailure(error); + if ( + credentialFailure.kind === "quota" || + credentialFailure.kind === "concurrency" || + credentialFailure.kind === "rate_limit" + ) { + this.logger.warn( + { failureKind: credentialFailure.kind, message: error.message }, + "Soniox credential capacity error - retrying so fallback credentials can be tried", + ); + return true; + } + // Extract error code if available const errorCodeMatch = error.message.match(/Soniox error (\d+):/); if (errorCodeMatch) { diff --git a/cloud/packages/cloud/src/services/session/transcription/providers/SonioxSdkStream.ts b/cloud/packages/cloud/src/services/session/transcription/providers/SonioxSdkStream.ts index 3706cab455..72f4c090a6 100644 --- a/cloud/packages/cloud/src/services/session/transcription/providers/SonioxSdkStream.ts +++ b/cloud/packages/cloud/src/services/session/transcription/providers/SonioxSdkStream.ts @@ -189,6 +189,7 @@ export class SonioxSdkStream implements StreamInstance { public readonly logger: Logger, private readonly config: SonioxProviderConfig, client: SonioxNodeClient, + private readonly credentialId?: string, ) { this.endpointDebounceMs = config.endpointDebounceMs ?? SonioxSdkStream.DEFAULT_ENDPOINT_DEBOUNCE_MS; @@ -854,6 +855,9 @@ export class SonioxSdkStream implements StreamInstance { this.metrics.consecutiveFailures++; this.provider.recordFailure(error); + if (this.credentialId) { + this.provider.recordCredentialFailure(this.credentialId, error); + } this.logger.error({ error, streamId: this.id, sessionState: this.session.state }, "Soniox SDK stream error"); @@ -960,7 +964,7 @@ export class SonioxSdkStream implements StreamInstance { const enableLanguageIdentification = !(disableLangIdParam === true || disableLangIdParam === "true"); const sessionConfig: SttSessionConfig = { - model: this.config.model || "stt-rt-v4", + model: this.config.model || "stt-rt-v5", audio_format: "pcm_s16le", sample_rate: 16000, num_channels: 1, diff --git a/cloud/packages/cloud/src/services/session/transcription/providers/SonioxTranscriptionProvider.ts b/cloud/packages/cloud/src/services/session/transcription/providers/SonioxTranscriptionProvider.ts index 82afbedabf..911f71887f 100644 --- a/cloud/packages/cloud/src/services/session/transcription/providers/SonioxTranscriptionProvider.ts +++ b/cloud/packages/cloud/src/services/session/transcription/providers/SonioxTranscriptionProvider.ts @@ -15,6 +15,7 @@ import { SonioxNodeClient } from "@soniox/node"; import { StreamType, getLanguageInfo, parseLanguageStream, TranscriptionData, SonioxToken } from "@mentra/sdk"; import { SonioxSdkStream } from "./SonioxSdkStream"; +import { SonioxCredential, SonioxKeyPool, getSharedSonioxKeyPool } from "../../soniox/SonioxKeyPool"; import { TranscriptionProvider, @@ -70,6 +71,10 @@ interface SonioxResponse { finished?: boolean; // Indicates end of transcription } +type InitializableStreamInstance = StreamInstance & { + initialize(): Promise; +}; + export class SonioxTranscriptionProvider implements TranscriptionProvider { readonly name = ProviderType.SONIOX; readonly logger: Logger; @@ -78,8 +83,9 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { private failureCount = 0; private lastFailureTime = 0; - /** Soniox Node SDK client — initialized when SONIOX_USE_SDK=true */ - private sdkClient: SonioxNodeClient | null = null; + /** Soniox credential pool: primary key plus optional fallback keys. */ + private readonly keyPool: SonioxKeyPool; + private readonly sdkClients = new Map(); private readonly useSdk: boolean; constructor( @@ -88,6 +94,7 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { ) { this.logger = parentLogger.child({ provider: this.name }); this.useSdk = process.env.SONIOX_USE_SDK !== "false"; + this.keyPool = getSharedSonioxKeyPool(config.apiKey, config.fallbackApiKeys ?? []); this.healthStatus = { isHealthy: true, @@ -100,6 +107,8 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { supportedLanguages: SONIOX_SUPPORTED_LANGUAGES.length, languages: SONIOX_SUPPORTED_LANGUAGES, useSdk: this.useSdk, + credentialCount: this.keyPool.size, + hasFallbackCredentials: this.keyPool.hasFallbacks, }, `Soniox provider initialized with ${SONIOX_SUPPORTED_LANGUAGES.length} supported languages (SDK mode: ${this.useSdk})`, ); @@ -108,23 +117,16 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { async initialize(): Promise { this.logger.info({ useSdk: this.useSdk }, "Initializing Soniox provider"); - if (!this.config.apiKey) { + if (this.keyPool.size === 0) { throw new Error("Soniox API key is required"); } - // Initialize SDK client if enabled - if (this.useSdk) { - this.sdkClient = new SonioxNodeClient({ - api_key: this.config.apiKey, - }); - this.logger.info("✅ Soniox SDK client initialized"); - } - this.logger.info( { endpoint: this.config.endpoint, - keyLength: this.config.apiKey.length, useSdk: this.useSdk, + credentialCount: this.keyPool.size, + hasFallbackCredentials: this.keyPool.hasFallbacks, }, "Soniox provider initialized", ); @@ -132,7 +134,7 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { async dispose(): Promise { this.logger.info("Disposing Soniox provider"); - this.sdkClient = null; + this.sdkClients.clear(); } async createTranscriptionStream(language: string, options: StreamOptions): Promise { @@ -149,40 +151,59 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { throw new SonioxProviderError(`Language ${language} not supported by Soniox`, 400); } - // Use SDK-based stream when enabled - if (this.useSdk && this.sdkClient) { - const stream = new SonioxSdkStream( - options.streamId, - options.subscription, - this, - language, - undefined, - options.callbacks, - this.logger.child({ streamId: options.streamId, provider: "soniox-sdk" }), - this.config, - this.sdkClient, - ); + const attempted = new Set(); + let lastError: Error | null = null; + + while (attempted.size < this.keyPool.size) { + const credential = this.keyPool.selectCredential(attempted); + if (!credential) break; + attempted.add(credential.id); - await stream.initialize(); - return stream; + let stream: InitializableStreamInstance | null = null; + try { + stream = this.createStreamForCredential(credential, language, options); + await stream.initialize(); + this.keyPool.recordSuccess(credential.id); + return stream; + } catch (error) { + lastError = error as Error; + const classification = this.keyPool.recordFailure(credential.id, lastError); + this.logger.warn( + { + streamId: options.streamId, + credentialId: credential.id, + credentialRole: credential.role, + failureKind: classification?.kind, + error: lastError.message, + availableCredentials: this.keyPool.describeAvailability(), + }, + "Soniox credential failed while creating stream; trying another credential if available", + ); + if (stream) { + await stream.close().catch((closeError) => { + this.logger.debug({ closeError, streamId: options.streamId }, "Ignoring failed Soniox stream cleanup error"); + }); + } + } } - // Fall back to legacy raw-WebSocket stream - const stream = new SonioxTranscriptionStream( - options.streamId, - options.subscription, - this, - language, - undefined, - options.callbacks, - this.logger, - this.config, + throw ( + lastError ?? + new Error(`No available Soniox credentials: ${JSON.stringify(this.keyPool.describeAvailability())}`) ); + } - // Initialize WebSocket connection - await stream.initialize(); - - return stream; + recordCredentialFailure(credentialId: string, error: Error): void { + const classification = this.keyPool.recordFailure(credentialId, error); + this.logger.warn( + { + credentialId, + failureKind: classification?.kind, + error: error.message, + availableCredentials: this.keyPool.describeAvailability(), + }, + "Recorded Soniox credential failure", + ); } // Translation is now handled by a separate TranslationManager @@ -287,6 +308,71 @@ export class SonioxTranscriptionProvider implements TranscriptionProvider { this.logger.debug("Recorded provider success"); } + private createStreamForCredential( + credential: SonioxCredential, + language: string, + options: StreamOptions, + ): InitializableStreamInstance { + const credentialConfig: SonioxProviderConfig = { + ...this.config, + apiKey: credential.apiKey, + }; + + const credentialLogger = this.logger.child({ + streamId: options.streamId, + credentialId: credential.id, + credentialRole: credential.role, + }); + + // Use SDK-based stream when enabled + if (this.useSdk) { + const sdkClient = this.getSdkClient(credential); + return new SonioxSdkStream( + options.streamId, + options.subscription, + this, + language, + undefined, + options.callbacks, + credentialLogger.child({ provider: "soniox-sdk" }), + credentialConfig, + sdkClient, + credential.id, + ); + } + + // Fall back to legacy raw-WebSocket stream + return new SonioxTranscriptionStream( + options.streamId, + options.subscription, + this, + language, + undefined, + options.callbacks, + credentialLogger, + credentialConfig, + credential.id, + ); + } + + private getSdkClient(credential: SonioxCredential): SonioxNodeClient { + const existing = this.sdkClients.get(credential.id); + if (existing) return existing; + + const client = new SonioxNodeClient({ + api_key: credential.apiKey, + }); + this.sdkClients.set(credential.id, client); + this.logger.info( + { + credentialId: credential.id, + credentialRole: credential.role, + }, + "✅ Soniox SDK client initialized", + ); + return client; + } + private getRecentFailureCount(timeWindowMs: number): number { const now = Date.now(); return this.lastFailureTime && now - this.lastFailureTime < timeWindowMs ? this.failureCount : 0; @@ -381,6 +467,7 @@ class SonioxTranscriptionStream implements StreamInstance { public readonly callbacks: StreamCallbacks, public readonly logger: Logger, private readonly config: SonioxProviderConfig, + private readonly credentialId?: string, ) { this.metrics = { totalDuration: 0, @@ -536,7 +623,7 @@ class SonioxTranscriptionStream implements StreamInstance { const enableLanguageIdentification = !(disableLangIdParam === true || disableLangIdParam === "true"); const config: any = { api_key: this.config.apiKey, - model: this.config.model || "stt-rt-v4", + model: this.config.model || "stt-rt-v5", audio_format: "pcm_s16le", sample_rate: 16000, num_channels: 1, @@ -943,6 +1030,9 @@ class SonioxTranscriptionStream implements StreamInstance { this.lastSentInterim = ""; this.provider.recordFailure(error); + if (this.credentialId) { + this.provider.recordCredentialFailure(this.credentialId, error); + } if (this.callbacks.onError) { this.callbacks.onError(error); diff --git a/cloud/packages/cloud/src/services/session/transcription/types.ts b/cloud/packages/cloud/src/services/session/transcription/types.ts index 2419155229..67b4a8f267 100644 --- a/cloud/packages/cloud/src/services/session/transcription/types.ts +++ b/cloud/packages/cloud/src/services/session/transcription/types.ts @@ -12,8 +12,9 @@ dotenv.config(); const isChinaDeployment = process.env.DEPLOYMENT_REGION === "china"; // Environment variables for provider configuration export const SONIOX_API_KEY = process.env.SONIOX_API_KEY || ""; +export const SONIOX_FALLBACK_API_KEYS = process.env.SONIOX_FALLBACK_API_KEYS || ""; export const SONIOX_ENDPOINT = process.env.SONIOX_ENDPOINT || "wss://stt-rt.soniox.com/transcribe-websocket"; -export const SONIOX_MODEL = process.env.SONIOX_MODEL || "stt-rt-v4"; +export const SONIOX_MODEL = process.env.SONIOX_MODEL || "stt-rt-v5"; export const ALIBABA_ENDPOINT = process.env.ALIBABA_ENDPOINT || "wss://dashscope.aliyuncs.com/api-ws/v1/inference"; export const ALIBABA_WORKSPACE = process.env.ALIBABA_WORKSPACE || ""; export const ALIBABA_DASHSCOPE_API_KEY = process.env.ALIBABA_DASHSCOPE_API_KEY || ""; @@ -73,8 +74,9 @@ export interface TranscriptionConfig { export interface SonioxProviderConfig { apiKey: string; + fallbackApiKeys?: string[]; endpoint: string; - model?: string; // Default: SONIOX_MODEL env var or 'stt-rt-v4' + model?: string; // Default: SONIOX_MODEL env var or 'stt-rt-v5' maxConnections?: number; /** * Soniox `max_endpoint_delay_ms`. Allowed 500–3000, default 2000. @@ -360,6 +362,9 @@ export const DEFAULT_TRANSCRIPTION_CONFIG: TranscriptionConfig = { soniox: { apiKey: SONIOX_API_KEY, + fallbackApiKeys: SONIOX_FALLBACK_API_KEYS.split(",") + .map((key) => key.trim()) + .filter(Boolean), endpoint: SONIOX_ENDPOINT, model: SONIOX_MODEL, // 3000ms = Soniox's max. Higher = Soniox waits longer before diff --git a/cloud/packages/cloud/src/services/session/translation/TranslationManager.ts b/cloud/packages/cloud/src/services/session/translation/TranslationManager.ts index df4ff39b4e..ada956a9a5 100644 --- a/cloud/packages/cloud/src/services/session/translation/TranslationManager.ts +++ b/cloud/packages/cloud/src/services/session/translation/TranslationManager.ts @@ -12,6 +12,7 @@ import { parseLanguageStream, } from "@mentra/sdk"; import UserSession from "../UserSession"; +import { classifySonioxCredentialFailure } from "../soniox/SonioxKeyPool"; import { PosthogService } from "../../logging/posthog.service"; import { MemoryOwnerStat } from "../../metrics/memory-census"; import { estimateArrayBufferBytes, sumEstimatedBytes } from "../../metrics/memory-estimate"; @@ -987,6 +988,22 @@ export class TranslationManager { return; } + if (error.message.includes("No available Soniox translation credentials")) { + this.logger.warn({ subscription, message: error.message }, "Soniox translation credentials cooling down"); + } else if (error.message.includes("Soniox error")) { + const credentialFailure = classifySonioxCredentialFailure(error); + if ( + credentialFailure.kind === "quota" || + credentialFailure.kind === "concurrency" || + credentialFailure.kind === "rate_limit" + ) { + this.logger.warn( + { subscription, failureKind: credentialFailure.kind, message: error.message }, + "Soniox translation credential capacity error - retrying so fallback credentials can be tried", + ); + } + } + // Schedule retry this.scheduleStreamRetry(subscription, attempts + 1); } diff --git a/cloud/packages/cloud/src/services/session/translation/providers/SonioxTranslationProvider.ts b/cloud/packages/cloud/src/services/session/translation/providers/SonioxTranslationProvider.ts index 7f5df1959d..f7c0b43599 100644 --- a/cloud/packages/cloud/src/services/session/translation/providers/SonioxTranslationProvider.ts +++ b/cloud/packages/cloud/src/services/session/translation/providers/SonioxTranslationProvider.ts @@ -11,6 +11,7 @@ import { TranslationData, StreamType } from "@mentra/sdk"; import { MemoryOwnerStat } from "../../../metrics/memory-census"; import { estimateArrayBufferBytes, estimateStringBytes, sumEstimatedBytes } from "../../../metrics/memory-estimate"; import { ResourceTracker } from "../../../../utils/resource-tracker"; +import { SonioxCredential, SonioxKeyPool, getSharedSonioxKeyPool } from "../../soniox/SonioxKeyPool"; import { TranslationProvider, TranslationProviderType, @@ -54,7 +55,7 @@ interface SonioxToken { class SonioxTranslationStream implements TranslationStreamInstance { readonly id: string; readonly subscription: string; - readonly provider: TranslationProvider; + readonly provider: SonioxTranslationProvider; readonly logger: Logger; readonly sourceLanguage: string; readonly targetLanguage: string; @@ -119,8 +120,9 @@ class SonioxTranslationStream implements TranslationStreamInstance { constructor( options: TranslationStreamOptions, - provider: TranslationProvider, + provider: SonioxTranslationProvider, private config: SonioxTranslationConfig, + private readonly credentialId?: string, ) { this.id = options.streamId; this.subscription = options.subscription; @@ -329,7 +331,7 @@ class SonioxTranslationStream implements TranslationStreamInstance { const config = { api_key: this.config.apiKey, - model: this.config.model || "stt-rt-v4", + model: this.config.model || "stt-rt-v5", audio_format: "pcm_s16le", sample_rate: 16000, num_channels: 1, @@ -406,9 +408,11 @@ class SonioxTranslationStream implements TranslationStreamInstance { break; case "error": + const errorCode = message.error_code ?? message.code; + const errorMessage = message.message || message.error_message || "Unknown error"; this.handleError( new TranslationProviderError( - `Soniox error: ${message.message || "Unknown error"}`, + errorCode ? `Soniox error ${errorCode}: ${errorMessage}` : `Soniox error: ${errorMessage}`, TranslationProviderType.SONIOX, ), ); @@ -759,6 +763,9 @@ class SonioxTranslationStream implements TranslationStreamInstance { this.metrics.errorCount++; this.metrics.lastError = error; this.state = TranslationStreamState.ERROR; + if (this.credentialId) { + this.provider.recordCredentialFailure(this.credentialId, error); + } this.callbacks.onError?.(error); } @@ -987,18 +994,20 @@ export class SonioxTranslationProvider implements TranslationProvider { // Get supported languages from the actual Soniox mappings private supportedLanguages = new Set(SonioxTranslationUtils.getSupportedLanguages()); + private readonly keyPool: SonioxKeyPool; constructor( private config: SonioxTranslationConfig, parentLogger: Logger, ) { this.logger = parentLogger.child({ provider: "soniox-translation" }); + this.keyPool = getSharedSonioxKeyPool(config.apiKey, config.fallbackApiKeys ?? []); } async initialize(): Promise { try { // Validate configuration - if (!this.config.apiKey) { + if (this.keyPool.size === 0) { throw new Error("Soniox translation provider requires API key"); } @@ -1007,7 +1016,13 @@ export class SonioxTranslationProvider implements TranslationProvider { } this.isInitialized = true; - this.logger.info("Soniox translation provider initialized"); + this.logger.info( + { + credentialCount: this.keyPool.size, + hasFallbackCredentials: this.keyPool.hasFallbacks, + }, + "Soniox translation provider initialized", + ); } catch (error) { this.logger.error({ error }, "Failed to initialize Soniox translation provider"); throw error; @@ -1033,11 +1048,69 @@ export class SonioxTranslationProvider implements TranslationProvider { ); } - const stream = new SonioxTranslationStream(options, this, this.config); - await stream.initialize(); + const attempted = new Set(); + let lastError: Error | null = null; + + while (attempted.size < this.keyPool.size) { + const credential = this.keyPool.selectCredential(attempted); + if (!credential) break; + attempted.add(credential.id); + + const stream = this.createStreamForCredential(options, credential); + try { + await stream.initialize(); + this.keyPool.recordSuccess(credential.id); + this.recordSuccess(); + return stream; + } catch (error) { + lastError = error as Error; + const classification = this.keyPool.recordFailure(credential.id, lastError); + this.logger.warn( + { + streamId: options.streamId, + credentialId: credential.id, + credentialRole: credential.role, + failureKind: classification?.kind, + error: lastError.message, + availableCredentials: this.keyPool.describeAvailability(), + }, + "Soniox translation credential failed while creating stream; trying another credential if available", + ); + await stream.close().catch((closeError) => { + this.logger.debug({ closeError, streamId: options.streamId }, "Ignoring failed Soniox translation stream cleanup error"); + }); + } + } + + throw ( + lastError ?? + new Error(`No available Soniox translation credentials: ${JSON.stringify(this.keyPool.describeAvailability())}`) + ); + } + + recordCredentialFailure(credentialId: string, error: Error): void { + const classification = this.keyPool.recordFailure(credentialId, error); + this.logger.warn( + { + credentialId, + failureKind: classification?.kind, + error: error.message, + availableCredentials: this.keyPool.describeAvailability(), + }, + "Recorded Soniox translation credential failure", + ); + } + + private createStreamForCredential( + options: TranslationStreamOptions, + credential: SonioxCredential, + ): SonioxTranslationStream { + const credentialConfig: SonioxTranslationConfig = { + ...this.config, + apiKey: credential.apiKey, + }; - this.recordSuccess(); - return stream; + return new SonioxTranslationStream(options, this, credentialConfig, credential.id); } supportsLanguagePair(source: string, target: string): boolean { diff --git a/cloud/packages/cloud/src/services/session/translation/types.ts b/cloud/packages/cloud/src/services/session/translation/types.ts index f6e3a9f4d7..533d8c65ed 100644 --- a/cloud/packages/cloud/src/services/session/translation/types.ts +++ b/cloud/packages/cloud/src/services/session/translation/types.ts @@ -10,8 +10,9 @@ dotenv.config(); // Environment variables for provider configuration export const SONIOX_API_KEY = process.env.SONIOX_API_KEY || ""; +export const SONIOX_FALLBACK_API_KEYS = process.env.SONIOX_FALLBACK_API_KEYS || ""; export const SONIOX_ENDPOINT = process.env.SONIOX_ENDPOINT || "wss://stt-rt.soniox.com/transcribe-websocket"; -export const SONIOX_MODEL = process.env.SONIOX_MODEL || "stt-rt-v4"; +export const SONIOX_MODEL = process.env.SONIOX_MODEL || "stt-rt-v5"; export const ALIBABA_ENDPOINT = process.env.ALIBABA_ENDPOINT || "wss://dashscope.aliyuncs.com/api-ws/v1/inference"; export const ALIBABA_WORKSPACE = process.env.ALIBABA_WORKSPACE || ""; @@ -68,8 +69,9 @@ export interface TranslationConfig { export interface SonioxTranslationConfig { apiKey: string; + fallbackApiKeys?: string[]; endpoint: string; - model?: string; // Default: SONIOX_MODEL env var or 'stt-rt-v4' + model?: string; // Default: SONIOX_MODEL env var or 'stt-rt-v5' maxConnections?: number; } @@ -289,6 +291,9 @@ export const DEFAULT_TRANSLATION_CONFIG: TranslationConfig = { soniox: { apiKey: SONIOX_API_KEY, + fallbackApiKeys: SONIOX_FALLBACK_API_KEYS.split(",") + .map((key) => key.trim()) + .filter(Boolean), endpoint: SONIOX_ENDPOINT, model: SONIOX_MODEL, },