Add Soniox fallback keys to cloud v2

cloud-v2/docs/runbooks/doppler/porter-integration.md

@@ -157,7 +157,7 @@ MONGO_URL, REDIS_URL
 
 NODE_ENV, LOG_LEVEL, LOG_STDOUT_JSON, REGION
 
-SONIOX_API_KEY
+SONIOX_API_KEY, SONIOX_FALLBACK_API_KEYS, SONIOX_MODEL
 
 BETTERSTACK_PASSWORD, BETTERSTACK_USERNAME, BETTERSTACK_SOURCE_TOKEN
 

cloud-v2/packages/runtime/src/services/audio/providers/soniox-key-pool.test.ts

@@ -0,0 +1,121 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  SonioxKeyPool,
+  classifySonioxCredentialFailure,
+  parseSonioxFallbackApiKeys,
+} from "./soniox-key-pool";
+
+describe("SonioxKeyPool", () => {
+  test("parses comma-separated fallback keys", () => {
+    expect(parseSonioxFallbackApiKeys(" a, b ,, c ")).toEqual(["a", "b", "c"]);
+    expect(parseSonioxFallbackApiKeys(undefined)).toEqual([]);
+  });
+
+  test("prefers the primary key while available", () => {
+    const pool = new SonioxKeyPool("primary", ["fallback-a", "fallback-b"]);
+
+    const credential = pool.selectCredential(new Set(), 1_000);
+
+    expect(credential?.role).toBe("primary");
+  });
+
+  test("round-robins fallback keys when primary is cooling down", () => {
+    const pool = new SonioxKeyPool("primary", ["fallback-a", "fallback-b"]);
+    const primary = pool.selectCredential(new Set(), 1_000)!;
+    pool.recordFailure(primary.id, new Error("Soniox error 429: rate limit"), 1_000);
+
+    const first = pool.selectCredential(new Set(), 1_000);
+    const second = pool.selectCredential(new Set(), 1_000);
+    const third = pool.selectCredential(new Set(), 1_000);
+
+    expect(first?.role).toBe("fallback");
+    expect(second?.role).toBe("fallback");
+    expect(third?.role).toBe("fallback");
+    expect(first?.id).not.toBe(second?.id);
+    expect(third?.id).toBe(first?.id);
+  });
+
+  test("deduplicates fallback keys that match the primary", () => {
+    const pool = new SonioxKeyPool("primary", ["primary", "fallback"]);
+
+    expect(pool.size).toBe(2);
+  });
+
+  test("makes concurrency failures available again after a short cooldown", () => {
+    const pool = new SonioxKeyPool("primary", ["fallback"]);
+    const primary = pool.selectCredential(new Set(), 1_000)!;
+    pool.recordFailure(
+      primary.id,
+      new Error("Soniox error 429: maximum concurrent streams reached"),
+      1_000,
+    );
+
+    expect(pool.selectCredential(new Set(), 1_000)?.role).toBe("fallback");
+    expect(pool.selectCredential(new Set(), 6_001)?.role).toBe("primary");
+  });
+
+  test("disables invalid keys for the process", () => {
+    const pool = new SonioxKeyPool("primary", ["fallback"]);
+    const primary = pool.selectCredential(new Set(), 1_000)!;
+    pool.recordFailure(primary.id, new Error("Soniox error 401: invalid api key"), 1_000);
+
+    const availability = pool
+      .describeAvailability(10_000)
+      .find((item) => item.id === primary.id);
+
+    expect(availability?.disabled).toBe(true);
+    expect(availability?.available).toBe(false);
+    expect(pool.selectCredential(new Set(), 10_000)?.role).toBe("fallback");
+  });
+
+  test("does not let overlapping success clear active cooldown", () => {
+    const pool = new SonioxKeyPool("primary", ["fallback"]);
+    const primary = pool.selectCredential(new Set(), 1_000)!;
+    pool.recordFailure(
+      primary.id,
+      new Error("Soniox error 402: organization_monthly_budget_exhausted"),
+      1_000,
+    );
+
+    pool.recordSuccess(primary.id, 2_000);
+
+    const availability = pool
+      .describeAvailability(2_000)
+      .find((item) => item.id === primary.id);
+    expect(availability?.failureKind).toBe("quota");
+    expect(availability?.available).toBe(false);
+    expect(pool.selectCredential(new Set(), 2_000)?.role).toBe("fallback");
+  });
+});
+
+describe("classifySonioxCredentialFailure", () => {
+  test("classifies quota exhaustion separately from request rate limits", () => {
+    expect(classifySonioxCredentialFailure(new Error("Monthly quota exceeded")).kind).toBe(
+      "quota",
+    );
+    expect(
+      classifySonioxCredentialFailure(
+        new Error("Soniox error 402: Organization monthly budget exhausted"),
+      ).kind,
+    ).toBe("quota");
+    expect(
+      classifySonioxCredentialFailure(new Error("Soniox error 429: rate limit exhausted")).kind,
+    ).toBe("rate_limit");
+  });
+
+  test("treats concurrent stream errors as temporary capacity errors", () => {
+    expect(classifySonioxCredentialFailure(new Error("Too many concurrent streams")).kind).toBe(
+      "concurrency",
+    );
+  });
+
+  test("does not treat generic exhaustion as quota", () => {
+    expect(classifySonioxCredentialFailure(new Error("retries exhausted")).kind).toBe(
+      "transient",
+    );
+    expect(classifySonioxCredentialFailure(new Error("connection suspended")).kind).toBe(
+      "transient",
+    );
+  });
+});

cloud-v2/packages/runtime/src/services/audio/providers/soniox-key-pool.ts

@@ -0,0 +1,249 @@
+import crypto from "node:crypto";
+
+export type SonioxCredentialRole = "primary" | "fallback";
+
+export interface SonioxCredential {
+  id: string;
+  apiKey: string;
+  role: SonioxCredentialRole;
+}
+
+type SonioxCredentialFailureKind =
+  | "auth"
+  | "concurrency"
+  | "quota"
+  | "rate_limit"
+  | "transient";
+
+interface SonioxCredentialState extends SonioxCredential {
+  cooldownUntil: number;
+  disabled: boolean;
+  failureKind?: SonioxCredentialFailureKind;
+  lastFailureMessage?: string;
+}
+
+export interface SonioxCredentialFailureClassification {
+  kind: SonioxCredentialFailureKind;
+  cooldownMs: number;
+  disabled?: boolean;
+}
+
+const CONCURRENCY_COOLDOWN_MS = 5_000;
+const RATE_LIMIT_COOLDOWN_MS = 60_000;
+const QUOTA_COOLDOWN_MS = 30 * 60_000;
+const TRANSIENT_COOLDOWN_MS = 10_000;
+
+export function parseSonioxFallbackApiKeys(value: string | undefined): string[] {
+  if (!value) return [];
+  return value
+    .split(",")
+    .map((key) => key.trim())
+    .filter(Boolean);
+}
+
+export function fingerprintSonioxKey(apiKey: string): string {
+  return crypto.createHash("sha256").update(apiKey).digest("hex").slice(0, 12);
+}
+
+export function classifySonioxCredentialFailure(
+  error: Error,
+): SonioxCredentialFailureClassification {
+  const message = error.message || "";
+  const lower = message.toLowerCase();
+  const code = extractSonioxErrorCode(message);
+
+  if (
+    code === 401 ||
+    lower.includes("invalid api key") ||
+    lower.includes("invalid_api_key") ||
+    lower.includes("bad api key") ||
+    lower.includes("unauthorized")
+  ) {
+    return { kind: "auth", cooldownMs: Number.POSITIVE_INFINITY, disabled: true };
+  }
+
+  if (
+    lower.includes("concurrent") ||
+    lower.includes("concurrency") ||
+    lower.includes("connection limit") ||
+    lower.includes("stream limit") ||
+    lower.includes("too many streams") ||
+    lower.includes("maximum streams") ||
+    lower.includes("max streams")
+  ) {
+    return { kind: "concurrency", cooldownMs: CONCURRENCY_COOLDOWN_MS };
+  }
+
+  if (
+    code === 429 ||
+    lower.includes("rate limit") ||
+    lower.includes("rate_limit") ||
+    lower.includes("too many requests")
+  ) {
+    return { kind: "rate_limit", cooldownMs: RATE_LIMIT_COOLDOWN_MS };
+  }
+
+  if (
+    code === 402 ||
+    /\bquota\b/.test(lower) ||
+    /\bbudget\b/.test(lower) ||
+    /\bcredit(?:s)?\b/.test(lower) ||
+    /\bbilling\b/.test(lower) ||
+    /\bspend(?:ing)?\b/.test(lower) ||
+    /\bbalance\b/.test(lower) ||
+    lower.includes("usage limit") ||
+    lower.includes("monthly limit")
+  ) {
+    return { kind: "quota", cooldownMs: QUOTA_COOLDOWN_MS };
+  }
+
+  return { kind: "transient", cooldownMs: TRANSIENT_COOLDOWN_MS };
+}
+
+export class SonioxKeyPool {
+  private credentials: SonioxCredentialState[];
+  private nextFallbackIndex = 0;
+
+  constructor(primaryApiKey: string, fallbackApiKeys: string[] = []) {
+    const seen = new Set<string>();
+    const credentials: SonioxCredentialState[] = [];
+
+    const addCredential = (apiKey: string, role: SonioxCredentialRole): void => {
+      const trimmed = apiKey.trim();
+      if (!trimmed || seen.has(trimmed)) return;
+      seen.add(trimmed);
+      credentials.push({
+        id: fingerprintSonioxKey(trimmed),
+        apiKey: trimmed,
+        role,
+        cooldownUntil: 0,
+        disabled: false,
+      });
+    };
+
+    addCredential(primaryApiKey, "primary");
+    for (const key of fallbackApiKeys) {
+      addCredential(key, "fallback");
+    }
+
+    this.credentials = credentials;
+  }
+
+  get size(): number {
+    return this.credentials.length;
+  }
+
+  get hasFallbacks(): boolean {
+    return this.credentials.some((credential) => credential.role === "fallback");
+  }
+
+  selectCredential(
+    attempted = new Set<string>(),
+    now = Date.now(),
+  ): SonioxCredential | null {
+    const primary = this.credentials.find(
+      (credential) => credential.role === "primary",
+    );
+    if (primary && !attempted.has(primary.id) && this.isAvailable(primary, now)) {
+      return this.toPublicCredential(primary);
+    }
+
+    const fallbackCredentials = this.credentials.filter(
+      (credential) => credential.role === "fallback",
+    );
+    if (fallbackCredentials.length === 0) return null;
+
+    for (let offset = 0; offset < fallbackCredentials.length; offset += 1) {
+      const index = (this.nextFallbackIndex + offset) % fallbackCredentials.length;
+      const credential = fallbackCredentials[index];
+      if (attempted.has(credential.id) || !this.isAvailable(credential, now)) {
+        continue;
+      }
+
+      this.nextFallbackIndex = (index + 1) % fallbackCredentials.length;
+      return this.toPublicCredential(credential);
+    }
+
+    return null;
+  }
+
+  recordSuccess(credentialId: string, now = Date.now()): void {
+    const credential = this.findCredential(credentialId);
+    if (!credential || credential.disabled) return;
+    if (credential.cooldownUntil > now) return;
+    credential.cooldownUntil = 0;
+    credential.failureKind = undefined;
+    credential.lastFailureMessage = undefined;
+  }
+
+  recordFailure(
+    credentialId: string,
+    error: Error,
+    now = Date.now(),
+  ): SonioxCredentialFailureClassification | null {
+    const credential = this.findCredential(credentialId);
+    if (!credential) return null;
+
+    const classification = classifySonioxCredentialFailure(error);
+    credential.failureKind = classification.kind;
+    credential.lastFailureMessage = error.message;
+
+    if (classification.disabled) {
+      credential.disabled = true;
+      credential.cooldownUntil = Number.POSITIVE_INFINITY;
+    } else {
+      credential.cooldownUntil = Math.max(
+        credential.cooldownUntil,
+        now + classification.cooldownMs,
+      );
+    }
+
+    return classification;
+  }
+
+  describeAvailability(now = Date.now()): Array<{
+    id: string;
+    role: SonioxCredentialRole;
+    available: boolean;
+    disabled: boolean;
+    cooldownRemainingMs: number;
+    failureKind?: SonioxCredentialFailureKind;
+  }> {
+    return this.credentials.map((credential) => ({
+      id: credential.id,
+      role: credential.role,
+      available: this.isAvailable(credential, now),
+      disabled: credential.disabled,
+      cooldownRemainingMs:
+        credential.cooldownUntil === Number.POSITIVE_INFINITY
+          ? Number.POSITIVE_INFINITY
+          : Math.max(0, credential.cooldownUntil - now),
+      failureKind: credential.failureKind,
+    }));
+  }
+
+  private findCredential(
+    credentialId: string,
+  ): SonioxCredentialState | undefined {
+    return this.credentials.find((credential) => credential.id === credentialId);
+  }
+
+  private isAvailable(credential: SonioxCredentialState, now: number): boolean {
+    return !credential.disabled && credential.cooldownUntil <= now;
+  }
+
+  private toPublicCredential(credential: SonioxCredentialState): SonioxCredential {
+    return {
+      id: credential.id,
+      apiKey: credential.apiKey,
+      role: credential.role,
+    };
+  }
+}
+
+function extractSonioxErrorCode(message: string): number | null {
+  const match = message.match(/Soniox error (\d+):/i);
+  if (!match) return null;
+  const parsed = Number.parseInt(match[1], 10);
+  return Number.isFinite(parsed) ? parsed : null;
+}