diff --git a/.gitignore b/.gitignore index 0ec569d7..c2a43131 100644 --- a/.gitignore +++ b/.gitignore @@ -189,4 +189,6 @@ docs # vscode workspace config agents-js.code-workspace -examples/src/test_*.ts \ No newline at end of file +examples/src/test_*.ts + +.claude \ No newline at end of file diff --git a/examples/package.json b/examples/package.json index b5ae131d..4cceba5e 100644 --- a/examples/package.json +++ b/examples/package.json @@ -32,6 +32,7 @@ "@livekit/agents-plugin-cartesia": "workspace:*", "@livekit/agents-plugin-neuphonic": "workspace:*", "@livekit/noise-cancellation-node": "^0.1.9", + "@livekit/agents-plugin-upliftai": "workspace:^", "@livekit/rtc-node": "^0.13.11", "livekit-server-sdk": "^2.13.3", "zod": "^3.23.8" diff --git a/examples/src/upliftai_agent.ts b/examples/src/upliftai_agent.ts new file mode 100644 index 00000000..d87029ab --- /dev/null +++ b/examples/src/upliftai_agent.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type JobContext, + type JobProcess, + WorkerOptions, + cli, + defineAgent, + metrics, + voice, +} from '@livekit/agents'; +import * as livekit from '@livekit/agents-plugin-livekit'; +import * as openai from '@livekit/agents-plugin-openai'; +import * as silero from '@livekit/agents-plugin-silero'; +import * as upliftai from '@livekit/agents-plugin-upliftai'; +import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node'; +import { fileURLToPath } from 'node:url'; + +export default defineAgent({ + prewarm: async (proc: JobProcess) => { + proc.userData.vad = await silero.VAD.load(); + }, + entry: async (ctx: JobContext) => { + const vad = ctx.proc.userData.vad! as silero.VAD; + + const agent = new voice.Agent({ + vad, // openai stt needs this + instructions: + 'You are a helpful voice assistant that shares some jokes. Always respond in Urdu Nastaliq script. Normalize responses for narration.', + }); + + const session = new voice.AgentSession({ + vad, // VAD is required here for OpenAI STT + stt: new openai.STT({ + model: 'gpt-4o-transcribe', + language: 'ur', + }), + tts: new upliftai.TTS(), + llm: new openai.LLM({ + model: 'gpt-4o-mini', + }), + turnDetection: new livekit.turnDetector.MultilingualModel(), + }); + + const usageCollector = new metrics.UsageCollector(); + + session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => { + metrics.logMetrics(ev.metrics); + usageCollector.collect(ev.metrics); + }); + + await session.start({ + agent, + room: ctx.room, + inputOptions: { + noiseCancellation: BackgroundVoiceCancellation(), + }, + }); + + session.generateReply({ + instructions: 'Greet the user', + }); + }, +}); + +cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) })); diff --git a/plugins/upliftai/README.md b/plugins/upliftai/README.md new file mode 100644 index 00000000..7d58f685 --- /dev/null +++ b/plugins/upliftai/README.md @@ -0,0 +1,38 @@ + + +# @livekit/agents-plugin-upliftai + +UpliftAI TTS plugin for LiveKit Node Agents. + +## Installation + +```bash +npm install @livekit/agents-plugin-upliftai +``` + +## Usage + +```typescript +import { TTS } from '@livekit/agents-plugin-upliftai'; + +// Initialize TTS with your API key +const tts = new TTS({ + apiKey: 'your-api-key', // or set UPLIFTAI_API_KEY environment variable + voiceId: 'v_meklc281', // optional, defaults to v_meklc281 +}); + + +## Configuration + +### Environment Variables + +- `UPLIFTAI_API_KEY`: Your UpliftAI API key +- `UPLIFTAI_BASE_URL`: Base URL for the UpliftAI API (defaults to `wss://api.upliftai.org`) + +## License + +Apache-2.0 \ No newline at end of file diff --git a/plugins/upliftai/api-extractor.json b/plugins/upliftai/api-extractor.json new file mode 100644 index 00000000..1c75d907 --- /dev/null +++ b/plugins/upliftai/api-extractor.json @@ -0,0 +1,20 @@ +/** + * Config file for API Extractor. For more info, please visit: https://api-extractor.com + */ +{ + "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", + + /** + * Optionally specifies another JSON config file that this file extends from. This provides a way for + * standard settings to be shared across multiple projects. + * + * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains + * the "extends" field. Otherwise, the first path segment is interpreted as an NPM package name, and will be + * resolved using NodeJS require(). + * + * SUPPORTED TOKENS: none + * DEFAULT VALUE: "" + */ + "extends": "../../api-extractor-shared.json", + "mainEntryPointFilePath": "./dist/index.d.ts" +} \ No newline at end of file diff --git a/plugins/upliftai/package.json b/plugins/upliftai/package.json new file mode 100644 index 00000000..c1012d87 --- /dev/null +++ b/plugins/upliftai/package.json @@ -0,0 +1,51 @@ +{ + "name": "@livekit/agents-plugin-upliftai", + "version": "1.0.4", + "description": "UpliftAI TTS plugin for LiveKit Node Agents", + "main": "dist/index.js", + "require": "dist/index.cjs", + "types": "dist/index.d.ts", + "exports": { + "import": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "require": { + "types": "./dist/index.d.cts", + "default": "./dist/index.cjs" + } + }, + "author": "LiveKit", + "type": "module", + "repository": "git@github.com:livekit/agents-js.git", + "license": "Apache-2.0", + "files": [ + "dist", + "src", + "README.md" + ], + "scripts": { + "build": "tsup --onSuccess \"pnpm build:types\"", + "build:types": "tsc --declaration --emitDeclarationOnly && node ../../scripts/copyDeclarationOutput.js", + "clean": "rm -rf dist", + "clean:build": "pnpm clean && pnpm build", + "lint": "eslint -f unix \"src/**/*.{ts,js}\"", + "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript", + "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose" + }, + "devDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "^0.13.12", + "@microsoft/api-extractor": "^7.35.0", + "@types/node": "^20.0.0", + "tsup": "^8.3.5", + "typescript": "^5.0.0" + }, + "dependencies": { + "socket.io-client": "^4.7.2" + }, + "peerDependencies": { + "@livekit/agents": "workspace:*", + "@livekit/rtc-node": "^0.13.12" + } +} \ No newline at end of file diff --git a/plugins/upliftai/src/index.ts b/plugins/upliftai/src/index.ts new file mode 100644 index 00000000..63a5e2a5 --- /dev/null +++ b/plugins/upliftai/src/index.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { Plugin } from '@livekit/agents'; + +export * from './tts.js'; + +class UpliftAIPlugin extends Plugin { + constructor() { + super({ + title: 'upliftai', + version: '0.1.0', + package: '@livekit/agents-plugin-upliftai', + }); + } +} + +Plugin.registerPlugin(new UpliftAIPlugin()); diff --git a/plugins/upliftai/src/tts.test.ts b/plugins/upliftai/src/tts.test.ts new file mode 100644 index 00000000..6258b31f --- /dev/null +++ b/plugins/upliftai/src/tts.test.ts @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { APIConnectOptions, initializeLogger, tokenize } from '@livekit/agents'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ChunkedStream, type OutputFormat, TTS, type TTSOptions } from './tts.js'; + +// Mock socket.io-client +vi.mock('socket.io-client', () => ({ + Manager: vi.fn().mockImplementation(() => ({ + socket: vi.fn().mockReturnValue({ + on: vi.fn(), + off: vi.fn(), + emit: vi.fn(), + connect: vi.fn(), + disconnect: vi.fn(), + connected: false, + }), + })), +})); + +describe('UpliftAI TTS', () => { + const mockApiKey = 'test-api-key'; + const originalEnv = process.env; + + beforeAll(() => { + // Initialize the logger before running tests + initializeLogger({ pretty: false }); + }); + + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + describe('TTS Constructor', () => { + it('should create a TTS instance with API key from options', () => { + const tts = new TTS({ apiKey: mockApiKey }); + expect(tts).toBeInstanceOf(TTS); + expect(tts.label).toBe('upliftai.TTS'); + expect(tts.sampleRate).toBe(22050); + expect(tts.numChannels).toBe(1); + }); + + it('should create a TTS instance with API key from environment', () => { + process.env.UPLIFTAI_API_KEY = mockApiKey; + const tts = new TTS(); + expect(tts).toBeInstanceOf(TTS); + }); + + it('should throw error when no API key is provided', () => { + delete process.env.UPLIFTAI_API_KEY; + expect(() => new TTS()).toThrow( + 'UpliftAI API key is required, either as argument or set UPLIFTAI_API_KEY environment variable', + ); + }); + + it('should use custom base URL from options', () => { + const customURL = 'wss://custom.upliftai.org'; + const tts = new TTS({ apiKey: mockApiKey, baseURL: customURL }); + expect(tts).toBeInstanceOf(TTS); + }); + + it('should use base URL from environment', () => { + process.env.UPLIFTAI_API_KEY = mockApiKey; + process.env.UPLIFTAI_BASE_URL = 'wss://env.upliftai.org'; + const tts = new TTS(); + expect(tts).toBeInstanceOf(TTS); + }); + + it('should use custom voice ID', () => { + const tts = new TTS({ apiKey: mockApiKey, voiceId: 'custom_voice' }); + expect(tts).toBeInstanceOf(TTS); + }); + + it('should handle different output formats', () => { + const formats: OutputFormat[] = [ + 'PCM_22050_16', + 'WAV_22050_16', + 'WAV_22050_32', + 'MP3_22050_32', + 'MP3_22050_64', + 'MP3_22050_128', + 'OGG_22050_16', + 'ULAW_8000_8', + ]; + + formats.forEach((format) => { + const tts = new TTS({ apiKey: mockApiKey, outputFormat: format }); + expect(tts).toBeInstanceOf(TTS); + if (format === 'ULAW_8000_8') { + expect(tts.sampleRate).toBe(8000); + } else { + expect(tts.sampleRate).toBe(22050); + } + }); + }); + + it('should accept custom tokenizer', () => { + const customTokenizer = new tokenize.basic.WordTokenizer(); + const tts = new TTS({ apiKey: mockApiKey, tokenizer: customTokenizer }); + expect(tts).toBeInstanceOf(TTS); + }); + + it('should accept custom chunk timeout', () => { + const tts = new TTS({ apiKey: mockApiKey, chunkTimeout: 5000 }); + expect(tts).toBeInstanceOf(TTS); + }); + }); + + describe('TTS Methods', () => { + let tts: TTS; + + beforeEach(() => { + tts = new TTS({ apiKey: mockApiKey }); + }); + + afterEach(async () => { + await tts.close(); + }); + + it('should create a ChunkedStream when synthesize is called', () => { + const text = 'Hello world'; + const stream = tts.synthesize(text); + expect(stream).toBeInstanceOf(ChunkedStream); + expect(stream.label).toBe('upliftai.ChunkedStream'); + }); + + it('should create a SynthesizeStream when stream is called', () => { + const stream = tts.stream(); + expect(stream).toBeDefined(); + expect(stream.label).toBe('upliftai.SynthesizeStream'); + }); + + it('should handle close method gracefully', async () => { + await expect(tts.close()).resolves.not.toThrow(); + // Calling close multiple times should not throw + await expect(tts.close()).resolves.not.toThrow(); + }); + + it('should pass connection options to synthesize', () => { + const text = 'Test text'; + const connOptions = new APIConnectOptions({ + timeoutMs: 5000, + maxRetry: 3, + }); + const stream = tts.synthesize(text, connOptions); + expect(stream).toBeInstanceOf(ChunkedStream); + }); + + it('should pass connection options to stream', () => { + const connOptions = new APIConnectOptions({ + timeoutMs: 5000, + maxRetry: 3, + }); + const stream = tts.stream(connOptions); + expect(stream).toBeDefined(); + }); + }); + + describe('TTS Configuration Options', () => { + it('should create instance with all options specified', () => { + const options: TTSOptions = { + apiKey: mockApiKey, + baseURL: 'wss://custom.upliftai.org', + voiceId: 'custom_voice_123', + outputFormat: 'MP3_22050_128', + tokenizer: new tokenize.basic.SentenceTokenizer(), + chunkTimeout: 15000, + }; + + const tts = new TTS(options); + expect(tts).toBeInstanceOf(TTS); + expect(tts.sampleRate).toBe(22050); + expect(tts.numChannels).toBe(1); + }); + + it('should use default values when options are not provided', () => { + process.env.UPLIFTAI_API_KEY = mockApiKey; + const tts = new TTS(); + expect(tts).toBeInstanceOf(TTS); + expect(tts.sampleRate).toBe(22050); // Default for PCM_22050_16 + expect(tts.numChannels).toBe(1); + }); + + it('should prioritize options over environment variables', () => { + process.env.UPLIFTAI_API_KEY = 'env-api-key'; + process.env.UPLIFTAI_BASE_URL = 'wss://env.upliftai.org'; + + const tts = new TTS({ + apiKey: 'options-api-key', + baseURL: 'wss://options.upliftai.org', + }); + + expect(tts).toBeInstanceOf(TTS); + // We can't directly test the private fields, but the instance should be created successfully + }); + }); + + describe('ChunkedStream', () => { + let tts: TTS; + + beforeEach(() => { + tts = new TTS({ apiKey: mockApiKey }); + }); + + afterEach(async () => { + await tts.close(); + }); + + it('should create ChunkedStream with input text', () => { + const inputText = 'This is a test message'; + const stream = tts.synthesize(inputText); + expect(stream).toBeInstanceOf(ChunkedStream); + expect(stream.inputText).toBe(inputText); + }); + + it('should handle empty text input', () => { + const stream = tts.synthesize(''); + expect(stream).toBeInstanceOf(ChunkedStream); + expect(stream.inputText).toBe(''); + }); + + it('should handle long text input', () => { + const longText = 'Lorem ipsum '.repeat(100); + const stream = tts.synthesize(longText); + expect(stream).toBeInstanceOf(ChunkedStream); + expect(stream.inputText).toBe(longText); + }); + + it('should handle special characters in text', () => { + const specialText = "Hello! How are you? I'm fine. #test @user"; + const stream = tts.synthesize(specialText); + expect(stream).toBeInstanceOf(ChunkedStream); + expect(stream.inputText).toBe(specialText); + }); + }); + + describe('Error Handling', () => { + it('should throw error for invalid API key format', () => { + expect(() => new TTS({ apiKey: '' })).toThrow( + 'UpliftAI API key is required, either as argument or set UPLIFTAI_API_KEY environment variable', + ); + }); + + it('should handle undefined environment variables gracefully', () => { + delete process.env.UPLIFTAI_API_KEY; + delete process.env.UPLIFTAI_BASE_URL; + + expect(() => new TTS()).toThrow( + 'UpliftAI API key is required, either as argument or set UPLIFTAI_API_KEY environment variable', + ); + }); + }); + + describe('Streaming capabilities', () => { + let tts: TTS; + + beforeEach(() => { + tts = new TTS({ apiKey: mockApiKey }); + }); + + afterEach(async () => { + await tts.close(); + }); + + it('should indicate streaming support', () => { + expect(tts.capabilities.streaming).toBe(true); + }); + + it('should create stream with default tokenizer', () => { + const stream = tts.stream(); + expect(stream).toBeDefined(); + expect(stream.label).toBe('upliftai.SynthesizeStream'); + }); + + it('should create stream with word tokenizer', () => { + const wordTokenizer = new tokenize.basic.WordTokenizer(); + const ttsWithWordTokenizer = new TTS({ + apiKey: mockApiKey, + tokenizer: wordTokenizer, + }); + const stream = ttsWithWordTokenizer.stream(); + expect(stream).toBeDefined(); + }); + }); +}); diff --git a/plugins/upliftai/src/tts.ts b/plugins/upliftai/src/tts.ts new file mode 100644 index 00000000..ad3c5ec7 --- /dev/null +++ b/plugins/upliftai/src/tts.ts @@ -0,0 +1,640 @@ +// SPDX-FileCopyrightText: 2025 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +import { + type APIConnectOptions, + APIConnectionError, + APITimeoutError, + AsyncIterableQueue, + AudioByteStream, + DEFAULT_API_CONNECT_OPTIONS, + log, + shortuuid, + tokenize, + tts, +} from '@livekit/agents'; +import type { AudioFrame } from '@livekit/rtc-node'; +import { EventEmitter } from 'node:events'; +import type { Socket } from 'socket.io-client'; +import { Manager } from 'socket.io-client'; + +export type OutputFormat = + | 'PCM_22050_16' + | 'WAV_22050_16' + | 'WAV_22050_32' + | 'MP3_22050_32' + | 'MP3_22050_64' + | 'MP3_22050_128' + | 'OGG_22050_16' + | 'ULAW_8000_8'; + +export interface VoiceSettings { + voiceId: string; + outputFormat: OutputFormat; +} + +export interface TTSOptions { + apiKey?: string; + baseURL?: string; + voiceId?: string; + outputFormat?: OutputFormat; + tokenizer?: tokenize.WordTokenizer | tokenize.SentenceTokenizer; + /** + * The timeout for the next audio chunk to be received from the UpliftAI API. + * Default: 10000ms + */ + chunkTimeout?: number; +} + +const DEFAULT_BASE_URL = 'wss://api.upliftai.org'; +const DEFAULT_SAMPLE_RATE = 22050; +const DEFAULT_NUM_CHANNELS = 1; +const DEFAULT_VOICE_ID = 'v_meklc281'; +const DEFAULT_OUTPUT_FORMAT: OutputFormat = 'PCM_22050_16'; +const WEBSOCKET_NAMESPACE = '/text-to-speech/multi-stream'; +const DEFAULT_CHUNK_TIMEOUT = 10000; + +const getSampleRateFromFormat = (outputFormat: OutputFormat): number => { + if (outputFormat === 'ULAW_8000_8') { + return 8000; + } + return DEFAULT_SAMPLE_RATE; +}; + +export class TTS extends tts.TTS { + #apiKey: string; + #baseURL: string; + #voiceSettings: VoiceSettings; + #tokenizer: tokenize.WordTokenizer | tokenize.SentenceTokenizer; + #client: WebSocketClient | null = null; + #logger = log(); + #chunkTimeout: number; + label = 'upliftai.TTS'; + + constructor(opts: TTSOptions = {}) { + const outputFormat = opts.outputFormat || DEFAULT_OUTPUT_FORMAT; + const sampleRate = getSampleRateFromFormat(outputFormat); + + super(sampleRate, DEFAULT_NUM_CHANNELS, { + streaming: true, + }); + + this.#apiKey = opts.apiKey || process.env.UPLIFTAI_API_KEY || ''; + if (!this.#apiKey) { + throw new Error( + 'UpliftAI API key is required, either as argument or set UPLIFTAI_API_KEY environment variable', + ); + } + + this.#baseURL = opts.baseURL || process.env.UPLIFTAI_BASE_URL || DEFAULT_BASE_URL; + this.#voiceSettings = { + voiceId: opts.voiceId || DEFAULT_VOICE_ID, + outputFormat, + }; + this.#tokenizer = opts.tokenizer || new tokenize.basic.SentenceTokenizer(); + this.#chunkTimeout = opts.chunkTimeout || DEFAULT_CHUNK_TIMEOUT; + } + + synthesize( + text: string, + connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, + ): tts.ChunkedStream { + return new ChunkedStream( + this, + this.#apiKey, + this.#baseURL, + this.#voiceSettings, + text, + this.#chunkTimeout, + connOptions, + ); + } + + stream(connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS): tts.SynthesizeStream { + return new SynthesizeStream( + this, + this.#apiKey, + this.#baseURL, + this.#voiceSettings, + this.#tokenizer, + this.#chunkTimeout, + connOptions, + ); + } + + async close() { + if (this.#client) { + await this.#client.disconnect(); + this.#client = null; + } + } +} + +interface AudioMessage { + type: string; + requestId?: string; + audio?: string; + message?: string; + sessionId?: string; +} + +class WebSocketClient extends EventEmitter { + private manager: Manager; + private socket: Socket | null = null; + private connected = false; + private audioCallbacks: Map> = new Map(); + private logger = log(); + + constructor( + private apiKey: string, + private baseURL: string, + ) { + super(); + this.manager = new Manager(this.baseURL, { + transports: ['websocket'], + reconnection: true, + reconnectionAttempts: 3, + reconnectionDelay: 1000, + autoConnect: false, + }); + } + + async connect(): Promise { + if (this.connected) { + return true; + } + + try { + this.socket = this.manager.socket(WEBSOCKET_NAMESPACE, { + auth: { + token: this.apiKey, + }, + }); + + this.socket.on('connect', () => { + this.logger.debug('WebSocket connected to UpliftAI'); + }); + + this.socket.on('disconnect', () => { + this.connected = false; + this.logger.debug('WebSocket disconnected from UpliftAI'); + // Close all active queues + for (const queue of this.audioCallbacks.values()) { + if (!queue.closed) { + queue.put(null); + queue.close(); + } + } + this.audioCallbacks.clear(); + }); + + this.socket.on('message', (data: AudioMessage) => { + this.handleMessage(data); + }); + + this.socket.connect(); + + return new Promise((resolve) => { + const timeout = setTimeout(() => { + resolve(false); + }, 5000); + + const handleReady = (data: AudioMessage) => { + if (data.type === 'ready') { + this.connected = true; + clearTimeout(timeout); + this.socket!.off('message', handleReady); + resolve(true); + } + }; + this.socket!.on('message', handleReady); + }); + } catch (error) { + this.logger.error('Failed to connect to UpliftAI:', error); + return false; + } + } + + async synthesize( + text: string, + voiceSettings: VoiceSettings, + ): Promise> { + if (!this.socket || !this.connected) { + if (!(await this.connect())) { + throw new APIConnectionError({ message: 'Failed to connect to UpliftAI service' }); + } + } + + // Always create a new request ID for each synthesis request + const requestId = shortuuid(); + + // Create a new audio queue for this request + const audioQueue = new AsyncIterableQueue(); + this.audioCallbacks.set(requestId, audioQueue); + + const message = { + type: 'synthesize', + requestId, + text, + voiceId: voiceSettings.voiceId, + outputFormat: voiceSettings.outputFormat, + }; + + this.logger.debug(`Sending synthesis request ${requestId} for text: "${text.slice(0, 50)}..."`); + + try { + this.socket!.emit('synthesize', message); + } catch (error) { + this.logger.error('Failed to emit synthesis:', error); + this.audioCallbacks.delete(requestId); + audioQueue.close(); + throw error; + } + + return audioQueue; + } + + private handleMessage(data: AudioMessage) { + const messageType = data.type; + + if (messageType === 'ready') { + this.connected = true; + this.logger.debug(`Ready with session: ${data.sessionId}`); + } else if (messageType === 'audio') { + const requestId = data.requestId; + const audioB64 = data.audio; + + if (audioB64 && requestId && this.audioCallbacks.has(requestId)) { + const audioBytes = Buffer.from(audioB64, 'base64'); + const queue = this.audioCallbacks.get(requestId); + if (queue && !queue.closed) { + try { + queue.put(audioBytes); + } catch (error) { + this.logger.debug(`Queue closed for ${requestId}, ignoring audio data`); + } + } + } + } else if (messageType === 'audio_end') { + const requestId = data.requestId; + if (requestId && this.audioCallbacks.has(requestId)) { + const queue = this.audioCallbacks.get(requestId); + if (queue && !queue.closed) { + queue.put(null); + queue.close(); + } + this.audioCallbacks.delete(requestId); + } + } else if (messageType === 'error') { + const requestId = data.requestId || 'unknown'; + const errorMsg = data.message || JSON.stringify(data); + this.logger.error(`Error for ${requestId}: ${errorMsg}`); + + if (requestId !== 'unknown' && this.audioCallbacks.has(requestId)) { + const queue = this.audioCallbacks.get(requestId); + if (queue && !queue.closed) { + queue.put(null); + queue.close(); + } + this.audioCallbacks.delete(requestId); + } + } + } + + async disconnect() { + if (this.socket) { + // Clean up all active requests before disconnecting + for (const queue of this.audioCallbacks.values()) { + if (!queue.closed) { + queue.close(); + } + } + this.audioCallbacks.clear(); + + this.socket.disconnect(); + this.socket = null; + } + this.connected = false; + } +} + +export class ChunkedStream extends tts.ChunkedStream { + #apiKey: string; + #baseURL: string; + #voiceSettings: VoiceSettings; + #client: WebSocketClient | null = null; + #logger = log(); + #chunkTimeout: number; + label = 'upliftai.ChunkedStream'; + + constructor( + tts: TTS, + apiKey: string, + baseURL: string, + voiceSettings: VoiceSettings, + inputText: string, + chunkTimeout: number, + connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, + ) { + super(inputText, tts, connOptions); + this.#apiKey = apiKey; + this.#baseURL = baseURL; + this.#voiceSettings = voiceSettings; + this.#chunkTimeout = chunkTimeout; + } + + protected async run() { + const requestId = shortuuid(); + const sampleRate = getSampleRateFromFormat(this.#voiceSettings.outputFormat); + + try { + if (!this.#client) { + this.#client = new WebSocketClient(this.#apiKey, this.#baseURL); + } + + const audioQueue = await this.#client.synthesize(this.inputText, this.#voiceSettings); + + const bstream = new AudioByteStream(sampleRate, DEFAULT_NUM_CHANNELS); + let lastFrame: AudioFrame | undefined; + + const sendLastFrame = (final: boolean) => { + if (lastFrame) { + this.queue.put({ + requestId, + segmentId: requestId, + frame: lastFrame, + final, + }); + lastFrame = undefined; + } + }; + + let timeoutId: NodeJS.Timeout | null = null; + const clearChunkTimeout = () => { + if (timeoutId) { + clearTimeout(timeoutId); + timeoutId = null; + } + }; + + let hasReceivedData = false; + + try { + // Set timeout for first chunk + timeoutId = setTimeout(() => { + if (!hasReceivedData && !audioQueue.closed) { + this.#logger.warn(`No audio received after ${this.#chunkTimeout}ms`); + audioQueue.put(null); + audioQueue.close(); + } + }, this.#chunkTimeout); + + for await (const audioData of audioQueue) { + if (!hasReceivedData) { + clearChunkTimeout(); + hasReceivedData = true; + } + + if (audioData === null) { + this.#logger.debug('Audio stream ended'); + break; + } + + const audioArray = + audioData instanceof Buffer + ? (audioData.buffer.slice( + audioData.byteOffset, + audioData.byteOffset + audioData.byteLength, + ) as ArrayBuffer) + : ((audioData as Uint8Array).buffer as ArrayBuffer); + for (const frame of bstream.write(audioArray)) { + sendLastFrame(false); + lastFrame = frame; + } + } + } finally { + clearChunkTimeout(); + } + + // Flush remaining data + for (const frame of bstream.flush()) { + sendLastFrame(false); + lastFrame = frame; + } + + // Send the last frame as final + sendLastFrame(true); + + // Close the queue + this.queue.close(); + } catch (error) { + if (error instanceof APITimeoutError) { + throw error; + } + throw new APIConnectionError({ + message: `TTS synthesis failed: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } +} + +export class SynthesizeStream extends tts.SynthesizeStream { + #apiKey: string; + #baseURL: string; + #voiceSettings: VoiceSettings; + #tokenizer: tokenize.WordTokenizer | tokenize.SentenceTokenizer; + #client: WebSocketClient | null = null; + #logger = log(); + #chunkTimeout: number; + label = 'upliftai.SynthesizeStream'; + + constructor( + tts: TTS, + apiKey: string, + baseURL: string, + voiceSettings: VoiceSettings, + tokenizer: tokenize.WordTokenizer | tokenize.SentenceTokenizer, + chunkTimeout: number, + connOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, + ) { + super(tts, connOptions); + this.#apiKey = apiKey; + this.#baseURL = baseURL; + this.#voiceSettings = voiceSettings; + this.#tokenizer = tokenizer; + this.#chunkTimeout = chunkTimeout; + } + + protected async run() { + const segments = new AsyncIterableQueue(); + + const tokenizeInput = async () => { + let stream: tokenize.WordStream | tokenize.SentenceStream | null = null; + + try { + for await (const text of this.input) { + if (this.abortController.signal.aborted) { + break; + } + + if (text === SynthesizeStream.FLUSH_SENTINEL) { + if (stream) { + stream.endInput(); + stream = null; + } + } else { + if (!stream) { + stream = this.#tokenizer.stream(); + segments.put(stream); + } + stream.pushText(text); + } + } + + if (stream) { + stream.endInput(); + } + } finally { + segments.close(); + } + }; + + const processSegments = async () => { + for await (const wordStream of segments) { + if (this.abortController.signal.aborted) { + break; + } + await this.runSegment(wordStream); + } + }; + + try { + await Promise.all([tokenizeInput(), processSegments()]); + } catch (error) { + if (!this.abortController.signal.aborted) { + throw error; + } + } + } + + private async runSegment(wordStream: tokenize.WordStream | tokenize.SentenceStream) { + // Each segment gets its own unique IDs + const segmentId = shortuuid(); + const requestId = shortuuid(); + const sampleRate = getSampleRateFromFormat(this.#voiceSettings.outputFormat); + + if (this.abortController.signal.aborted) { + return; + } + + try { + if (!this.#client) { + this.#client = new WebSocketClient(this.#apiKey, this.#baseURL); + } + + // Collect text from tokenizer + const textParts: string[] = []; + for await (const data of wordStream) { + textParts.push(data.token); + } + + if (!textParts.length) { + return; + } + + const fullText = textParts.join(' '); + + // Create a new synthesis request with unique request ID + const audioQueue = await this.#client.synthesize(fullText, this.#voiceSettings); + const bstream = new AudioByteStream(sampleRate, DEFAULT_NUM_CHANNELS); + + let lastFrame: AudioFrame | undefined; + + const sendLastFrame = (segmentId: string, final: boolean) => { + if (lastFrame && !this.queue.closed) { + this.queue.put({ requestId, segmentId, frame: lastFrame, final }); + lastFrame = undefined; + } + }; + + let timeoutId: NodeJS.Timeout | null = null; + const clearChunkTimeout = () => { + if (timeoutId) { + clearTimeout(timeoutId); + timeoutId = null; + } + }; + + let hasReceivedData = false; + + try { + // Set timeout for first chunk + timeoutId = setTimeout(() => { + if (!hasReceivedData && !audioQueue.closed) { + this.#logger.warn( + `No audio received after ${this.#chunkTimeout}ms in segment ${segmentId}`, + ); + audioQueue.put(null); + audioQueue.close(); + } + }, this.#chunkTimeout); + + for await (const audioData of audioQueue) { + if (this.abortController.signal.aborted) { + break; + } + + if (!hasReceivedData) { + clearChunkTimeout(); + hasReceivedData = true; + } + + if (audioData === null) { + break; + } + + const audioArray = + audioData instanceof Buffer + ? (audioData.buffer.slice( + audioData.byteOffset, + audioData.byteOffset + audioData.byteLength, + ) as ArrayBuffer) + : ((audioData as Uint8Array).buffer as ArrayBuffer); + for (const frame of bstream.write(audioArray)) { + sendLastFrame(segmentId, false); + lastFrame = frame; + } + } + } finally { + clearChunkTimeout(); + } + + if (this.abortController.signal.aborted) { + return; + } + + // Flush remaining data + for (const frame of bstream.flush()) { + sendLastFrame(segmentId, false); + lastFrame = frame; + } + + // Send the last frame as final + sendLastFrame(segmentId, true); + + // Signal end of stream for this segment + if (!this.queue.closed) { + this.queue.put(SynthesizeStream.END_OF_STREAM); + } + } catch (error) { + if (this.abortController.signal.aborted) { + return; + } + + this.#logger.error('Segment synthesis error:', error); + throw new APIConnectionError({ + message: `Segment synthesis failed: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } +} diff --git a/plugins/upliftai/tsconfig.json b/plugins/upliftai/tsconfig.json new file mode 100644 index 00000000..dfacaa57 --- /dev/null +++ b/plugins/upliftai/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.json", + "include": ["./src"], + "compilerOptions": { + "rootDir": "./src", + "declarationDir": "./dist", + "outDir": "./dist" + }, + "typedocOptions": { + "name": "plugins/agents-plugin-upliftai", + "entryPointStrategy": "resolve", + "readme": "none", + "entryPoints": ["src/index.ts"] + } +} \ No newline at end of file diff --git a/plugins/upliftai/tsup.config.ts b/plugins/upliftai/tsup.config.ts new file mode 100644 index 00000000..8ca20961 --- /dev/null +++ b/plugins/upliftai/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'tsup'; + +import defaults from '../../tsup.config'; + +export default defineConfig({ + ...defaults, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 90406faf..9d644e61 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -189,6 +189,9 @@ importers: '@livekit/agents-plugin-silero': specifier: workspace:* version: link:../plugins/silero + '@livekit/agents-plugin-upliftai': + specifier: workspace:^ + version: link:../plugins/upliftai '@livekit/noise-cancellation-node': specifier: ^0.1.9 version: 0.1.9 @@ -531,6 +534,31 @@ importers: specifier: ^5.0.0 version: 5.4.5 + plugins/upliftai: + dependencies: + socket.io-client: + specifier: ^4.7.2 + version: 4.8.1 + devDependencies: + '@livekit/agents': + specifier: workspace:* + version: link:../../agents + '@livekit/rtc-node': + specifier: ^0.13.12 + version: 0.13.13 + '@microsoft/api-extractor': + specifier: ^7.35.0 + version: 7.43.7(@types/node@20.19.17) + '@types/node': + specifier: ^20.0.0 + version: 20.19.17 + tsup: + specifier: ^8.3.5 + version: 8.4.0(@microsoft/api-extractor@7.43.7(@types/node@20.19.17))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5) + typescript: + specifier: ^5.0.0 + version: 5.4.5 + packages: '@ampproject/remapping@2.3.0': @@ -1206,92 +1234,78 @@ packages: resolution: {integrity: sha512-RXwd0CgG+uPRX5YYrkzKyalt2OJYRiJQ8ED/fi1tq9WQW2jsQIn0tqrlR5l5dr/rjqq6AHAxURhj2DVjyQWSOA==} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.0': resolution: {integrity: sha512-mWd2uWvDtL/nvIzThLq3fr2nnGfyr/XMXlq8ZJ9WMR6PXijHlC3ksp0IpuhK6bougvQrchUAfzRLnbsen0Cqvw==} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.0': resolution: {integrity: sha512-Xod/7KaDDHkYu2phxxfeEPXfVXFKx70EAFZ0qyUdOjCcxbjqyJOEUpDe6RIyaunGxT34Anf9ue/wuWOqBW2WcQ==} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.0': resolution: {integrity: sha512-eMKfzDxLGT8mnmPJTNMcjfO33fLiTDsrMlUVcp6b96ETbnJmd4uvZxVJSKPQfS+odwfVaGifhsB07J1LynFehw==} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.0': resolution: {integrity: sha512-ZW3FPWIc7K1sH9E3nxIGB3y3dZkpJlMnkk7z5tu1nSkBoCgw2nSRTFHI5pB/3CQaJM0pdzMF3paf9ckKMSE9Tg==} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.0': resolution: {integrity: sha512-UG+LqQJbf5VJ8NWJ5Z3tdIe/HXjuIdo4JeVNADXBFuG7z9zjoegpzzGIyV5zQKi4zaJjnAd2+g2nna8TZvuW9Q==} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.0': resolution: {integrity: sha512-SRYOLR7CXPgNze8akZwjoGBoN1ThNZoqpOgfnOxmWsklTGVfJiGJoC/Lod7aNMGA1jSsKWM1+HRX43OP6p9+6Q==} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-linux-arm64@0.34.3': resolution: {integrity: sha512-QdrKe3EvQrqwkDrtuTIjI0bu6YEJHTgEeqdzI3uWJOH6G1O8Nl1iEeVYRGdj1h5I21CqxSvQp1Yv7xeU3ZewbA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [glibc] '@img/sharp-linux-arm@0.34.3': resolution: {integrity: sha512-oBK9l+h6KBN0i3dC8rYntLiVfW8D8wH+NPNT3O/WBHeW0OQWCjfWksLUaPidsrDKpJgXp3G3/hkmhptAW0I3+A==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] - libc: [glibc] '@img/sharp-linux-ppc64@0.34.3': resolution: {integrity: sha512-GLtbLQMCNC5nxuImPR2+RgrviwKwVql28FWZIW1zWruy6zLgA5/x2ZXk3mxj58X/tszVF69KK0Is83V8YgWhLA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@img/sharp-linux-s390x@0.34.3': resolution: {integrity: sha512-3gahT+A6c4cdc2edhsLHmIOXMb17ltffJlxR0aC2VPZfwKoTGZec6u5GrFgdR7ciJSsHT27BD3TIuGcuRT0KmQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] - libc: [glibc] '@img/sharp-linux-x64@0.34.3': resolution: {integrity: sha512-8kYso8d806ypnSq3/Ly0QEw90V5ZoHh10yH0HnrzOCr6DKAPI6QVHvwleqMkVQ0m+fc7EH8ah0BB0QPuWY6zJQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.3': resolution: {integrity: sha512-vAjbHDlr4izEiXM1OTggpCcPg9tn4YriK5vAjowJsHwdBIdx0fYRsURkxLG2RLm9gyBq66gwtWI8Gx0/ov+JKQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] - libc: [musl] '@img/sharp-linuxmusl-x64@0.34.3': resolution: {integrity: sha512-gCWUn9547K5bwvOn9l5XGAEjVTTRji4aPTqLzGXHvIr6bIDZKNTA34seMPgM0WmSf+RYBH411VavCejp3PkOeQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] - libc: [musl] '@img/sharp-wasm32@0.34.3': resolution: {integrity: sha512-+CyRcpagHMGteySaWos8IbnXcHgfDn7pO2fiC2slJxvNq9gDipYBN42/RagzctVRKgxATmfqOSulgZv5e1RdMg==} @@ -1419,14 +1433,12 @@ packages: engines: {node: '>= 10'} cpu: [arm64] os: [linux] - libc: [glibc] '@livekit/rtc-node-linux-x64-gnu@0.13.13': resolution: {integrity: sha512-B/SgbeBRobpA5LqmDEoBJHpRXePpoF4RO4F0zJf9BdkDhOR0j77p6hD0ZiOuPTRoBzUqukpsTszp+lZnHoNmiA==} engines: {node: '>= 10'} cpu: [x64] os: [linux] - libc: [glibc] '@livekit/rtc-node-win32-x64-msvc@0.13.13': resolution: {integrity: sha512-ygVYV4eHczs3QdaW/p0ADhhm7InUDhFaCYk8OzzIn056ZibZPXzvPizCThZqs8VsDniA01MraZF3qhZZb8IyRg==} @@ -1567,121 +1579,101 @@ packages: resolution: {integrity: sha512-3reX2fUHqN7sffBNqmEyMQVj/CKhIHZd4y631duy0hZqI8Qoqf6lTtmAKvJFYa6bhU95B1D0WgzHkmTg33In0A==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-gnueabihf@4.40.0': resolution: {integrity: sha512-y/qUMOpJxBMy8xCXD++jeu8t7kzjlOCkoxxajL58G62PJGBZVl/Gwpm7JK9+YvlB701rcQTzjUZ1JgUoPTnoQA==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.17.2': resolution: {integrity: sha512-uSqpsp91mheRgw96xtyAGP9FW5ChctTFEoXP0r5FAzj/3ZRv3Uxjtc7taRQSaQM/q85KEKjKsZuiZM3GyUivRg==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm-musleabihf@4.40.0': resolution: {integrity: sha512-GoCsPibtVdJFPv/BOIvBKO/XmwZLwaNWdyD8TKlXuqp0veo2sHE+A/vpMQ5iSArRUz/uaoj4h5S6Pn0+PdhRjg==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.17.2': resolution: {integrity: sha512-EMMPHkiCRtE8Wdk3Qhtciq6BndLtstqZIroHiiGzB3C5LDJmIZcSzVtLRbwuXuUft1Cnv+9fxuDtDxz3k3EW2A==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-gnu@4.40.0': resolution: {integrity: sha512-L5ZLphTjjAD9leJzSLI7rr8fNqJMlGDKlazW2tX4IUF9P7R5TMQPElpH82Q7eNIDQnQlAyiNVfRPfP2vM5Avvg==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.17.2': resolution: {integrity: sha512-NMPylUUZ1i0z/xJUIx6VUhISZDRT+uTWpBcjdv0/zkp7b/bQDF+NfnfdzuTiB1G6HTodgoFa93hp0O1xl+/UbA==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-musl@4.40.0': resolution: {integrity: sha512-ATZvCRGCDtv1Y4gpDIXsS+wfFeFuLwVxyUBSLawjgXK2tRE6fnsQEkE4csQQYWlBlsFztRzCnBvWVfcae/1qxQ==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.40.0': resolution: {integrity: sha512-wG9e2XtIhd++QugU5MD9i7OnpaVb08ji3P1y/hNbxrQ3sYEelKJOq1UJ5dXczeo6Hj2rfDEL5GdtkMSVLa/AOg==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.17.2': resolution: {integrity: sha512-T19My13y8uYXPw/L/k0JYaX1fJKFT/PWdXiHr8mTbXWxjVF1t+8Xl31DgBBvEKclw+1b00Chg0hxE2O7bTG7GQ==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.40.0': resolution: {integrity: sha512-vgXfWmj0f3jAUvC7TZSU/m/cOE558ILWDzS7jBhiCAFpY2WEBn5jqgbqvmzlMjtp8KlLcBlXVD2mkTSEQE6Ixw==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.17.2': resolution: {integrity: sha512-BOaNfthf3X3fOWAB+IJ9kxTgPmMqPPH5f5k2DcCsRrBIbWnaJCgX2ll77dV1TdSy9SaXTR5iDXRL8n7AnoP5cg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.40.0': resolution: {integrity: sha512-uJkYTugqtPZBS3Z136arevt/FsKTF/J9dEMTX/cwR7lsAW4bShzI2R0pJVw+hcBTWF4dxVckYh72Hk3/hWNKvA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.40.0': resolution: {integrity: sha512-rKmSj6EXQRnhSkE22+WvrqOqRtk733x3p5sWpZilhmjnkHkpeCgWsFFo0dGnUGeA+OZjRl3+VYq+HyCOEuwcxQ==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.17.2': resolution: {integrity: sha512-W0UP/x7bnn3xN2eYMql2T/+wpASLE5SjObXILTMPUBDB/Fg/FxC+gX4nvCfPBCbNhz51C+HcqQp2qQ4u25ok6g==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-s390x-gnu@4.40.0': resolution: {integrity: sha512-SpnYlAfKPOoVsQqmTFJ0usx0z84bzGOS9anAC0AZ3rdSo3snecihbhFTlJZ8XMwzqAcodjFU4+/SM311dqE5Sw==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.17.2': resolution: {integrity: sha512-Hy7pLwByUOuyaFC6mAr7m+oMC+V7qyifzs/nW2OJfC8H4hbCzOX07Ov0VFk/zP3kBsELWNFi7rJtgbKYsav9QQ==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.40.0': resolution: {integrity: sha512-RcDGMtqF9EFN8i2RYN2W+64CdHruJ5rPqrlYw+cgM3uOVPSsnAQps7cpjXe9be/yDp8UC7VLoCoKC8J3Kn2FkQ==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.17.2': resolution: {integrity: sha512-h1+yTWeYbRdAyJ/jMiVw0l6fOOm/0D1vNLui9iPuqgRGnXA0u21gAqOyB5iHjlM9MMfNOm9RHCQ7zLIzT0x11Q==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-linux-x64-musl@4.40.0': resolution: {integrity: sha512-HZvjpiUmSNx5zFgwtQAV1GaGazT2RWvqeDi0hV+AtC8unqqDSsaFjPxfsO6qPtKRRg25SisACWnJ37Yio8ttaw==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.17.2': resolution: {integrity: sha512-tmdtXMfKAjy5+IQsVtDiCfqbynAQE/TQRpWdVataHmhMb9DCoJxp9vLcCBjEQWMiUYxO1QprH/HbY9ragCEFLA==} @@ -1758,6 +1750,9 @@ packages: '@sinclair/typebox@0.27.8': resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + '@socket.io/component-emitter@3.1.2': + resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} + '@trivago/prettier-plugin-sort-imports@4.3.0': resolution: {integrity: sha512-r3n0onD3BTOVUNPhR4lhVK4/pABGpbA7bW3eumZnYdKaHkf1qEC+Mag6DPbGNuuh0eG8AaYj+YqmVHSiGslaTQ==} peerDependencies: @@ -1797,6 +1792,9 @@ packages: '@types/node@18.19.64': resolution: {integrity: sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==} + '@types/node@20.19.17': + resolution: {integrity: sha512-gfehUI8N1z92kygssiuWvLiwcbOB3IRktR6hTDgJlXMYh5OvkPSRmgfoBUmfZt+vhwJtX7v1Yw4KvvAf7c5QKQ==} + '@types/node@22.15.30': resolution: {integrity: sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==} @@ -2365,6 +2363,13 @@ packages: end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + engine.io-client@6.6.3: + resolution: {integrity: sha512-T0iLjnyNWahNyv/lcjS2y4oE358tVS/SYQNxYXGAJ9/GLgH4VCvOQ/mhTjqU88mLZCQgiG8RIegFHYCdVC+j5w==} + + engine.io-parser@5.2.3: + resolution: {integrity: sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==} + engines: {node: '>=10.0.0'} + enhanced-resolve@5.16.1: resolution: {integrity: sha512-4U5pNsuDl0EhuZpq46M5xPslstkviJuhrdobaRDBk2Jy2KO37FDAJl4lb2KlNabxT0m4MTK2UHNrsAcphE8nyw==} engines: {node: '>=10.13.0'} @@ -3884,6 +3889,14 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} + socket.io-client@4.8.1: + resolution: {integrity: sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==} + engines: {node: '>=10.0.0'} + + socket.io-parser@4.2.4: + resolution: {integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==} + engines: {node: '>=10.0.0'} + sonic-boom@3.8.1: resolution: {integrity: sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg==} @@ -4437,6 +4450,18 @@ packages: utf-8-validate: optional: true + ws@8.17.1: + resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + ws@8.18.3: resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==} engines: {node: '>=10.0.0'} @@ -4449,6 +4474,10 @@ packages: utf-8-validate: optional: true + xmlhttprequest-ssl@2.1.2: + resolution: {integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==} + engines: {node: '>=0.4.0'} + yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} @@ -5243,6 +5272,14 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 + '@microsoft/api-extractor-model@7.28.17(@types/node@20.19.17)': + dependencies: + '@microsoft/tsdoc': 0.14.2 + '@microsoft/tsdoc-config': 0.16.2 + '@rushstack/node-core-library': 4.3.0(@types/node@20.19.17) + transitivePeerDependencies: + - '@types/node' + '@microsoft/api-extractor-model@7.28.17(@types/node@22.15.30)': dependencies: '@microsoft/tsdoc': 0.14.2 @@ -5251,6 +5288,24 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@microsoft/api-extractor@7.43.7(@types/node@20.19.17)': + dependencies: + '@microsoft/api-extractor-model': 7.28.17(@types/node@20.19.17) + '@microsoft/tsdoc': 0.14.2 + '@microsoft/tsdoc-config': 0.16.2 + '@rushstack/node-core-library': 4.3.0(@types/node@20.19.17) + '@rushstack/rig-package': 0.5.2 + '@rushstack/terminal': 0.11.0(@types/node@20.19.17) + '@rushstack/ts-command-line': 4.21.0(@types/node@20.19.17) + lodash: 4.17.21 + minimatch: 3.0.8 + resolve: 1.22.8 + semver: 7.5.4 + source-map: 0.6.1 + typescript: 5.4.2 + transitivePeerDependencies: + - '@types/node' + '@microsoft/api-extractor@7.43.7(@types/node@22.15.30)': dependencies: '@microsoft/api-extractor-model': 7.28.17(@types/node@22.15.30) @@ -5459,6 +5514,17 @@ snapshots: transitivePeerDependencies: - '@types/node' + '@rushstack/node-core-library@4.3.0(@types/node@20.19.17)': + dependencies: + fs-extra: 7.0.1 + import-lazy: 4.0.0 + jju: 1.4.0 + resolve: 1.22.8 + semver: 7.5.4 + z-schema: 5.0.5 + optionalDependencies: + '@types/node': 20.19.17 + '@rushstack/node-core-library@4.3.0(@types/node@22.15.30)': dependencies: fs-extra: 7.0.1 @@ -5482,6 +5548,13 @@ snapshots: resolve: 1.22.8 strip-json-comments: 3.1.1 + '@rushstack/terminal@0.11.0(@types/node@20.19.17)': + dependencies: + '@rushstack/node-core-library': 4.3.0(@types/node@20.19.17) + supports-color: 8.1.1 + optionalDependencies: + '@types/node': 20.19.17 + '@rushstack/terminal@0.11.0(@types/node@22.15.30)': dependencies: '@rushstack/node-core-library': 4.3.0(@types/node@22.15.30) @@ -5489,6 +5562,15 @@ snapshots: optionalDependencies: '@types/node': 22.15.30 + '@rushstack/ts-command-line@4.21.0(@types/node@20.19.17)': + dependencies: + '@rushstack/terminal': 0.11.0(@types/node@20.19.17) + '@types/argparse': 1.0.38 + argparse: 1.0.10 + string-argv: 0.3.2 + transitivePeerDependencies: + - '@types/node' + '@rushstack/ts-command-line@4.21.0(@types/node@22.15.30)': dependencies: '@rushstack/terminal': 0.11.0(@types/node@22.15.30) @@ -5500,6 +5582,8 @@ snapshots: '@sinclair/typebox@0.27.8': {} + '@socket.io/component-emitter@3.1.2': {} + '@trivago/prettier-plugin-sort-imports@4.3.0(prettier@3.2.5)': dependencies: '@babel/generator': 7.17.7 @@ -5539,6 +5623,10 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@20.19.17': + dependencies: + undici-types: 6.21.0 + '@types/node@22.15.30': dependencies: undici-types: 6.21.0 @@ -6155,6 +6243,20 @@ snapshots: dependencies: once: 1.4.0 + engine.io-client@6.6.3: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.4 + engine.io-parser: 5.2.3 + ws: 8.17.1 + xmlhttprequest-ssl: 2.1.2 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + engine.io-parser@5.2.3: {} + enhanced-resolve@5.16.1: dependencies: graceful-fs: 4.2.11 @@ -7965,6 +8067,24 @@ snapshots: slash@3.0.0: {} + socket.io-client@4.8.1: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.4 + engine.io-client: 6.6.3 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-parser@4.2.4: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + sonic-boom@3.8.1: dependencies: atomic-sleep: 1.0.0 @@ -8195,6 +8315,34 @@ snapshots: tslib@2.6.2: {} + tsup@8.4.0(@microsoft/api-extractor@7.43.7(@types/node@20.19.17))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5): + dependencies: + bundle-require: 5.1.0(esbuild@0.25.2) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.0 + esbuild: 0.25.2 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(postcss@8.4.38)(tsx@4.20.4) + resolve-from: 5.0.0 + rollup: 4.40.0 + source-map: 0.8.0-beta.0 + sucrase: 3.35.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.12 + tree-kill: 1.2.2 + optionalDependencies: + '@microsoft/api-extractor': 7.43.7(@types/node@20.19.17) + postcss: 8.4.38 + typescript: 5.4.5 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + tsup@8.4.0(@microsoft/api-extractor@7.43.7(@types/node@22.15.30))(postcss@8.4.38)(tsx@4.20.4)(typescript@5.4.5): dependencies: bundle-require: 5.1.0(esbuild@0.25.2) @@ -8554,8 +8702,12 @@ snapshots: ws@8.17.0: {} + ws@8.17.1: {} + ws@8.18.3: {} + xmlhttprequest-ssl@2.1.2: {} + yallist@4.0.0: {} yallist@5.0.0: {} diff --git a/turbo.json b/turbo.json index df1ac0e6..45b58a6a 100644 --- a/turbo.json +++ b/turbo.json @@ -34,7 +34,9 @@ "GOOGLE_GENAI_API_KEY", "GOOGLE_GENAI_USE_VERTEXAI", "GOOGLE_CLOUD_PROJECT", - "GOOGLE_CLOUD_LOCATION" + "GOOGLE_CLOUD_LOCATION", + "UPLIFTAI_API_KEY", + "UPLIFTAI_BASE_URL" ], "pipeline": { "build": {