diff --git a/client/components/App.jsx b/client/components/App.jsx index fb1079451..9845794f9 100644 --- a/client/components/App.jsx +++ b/client/components/App.jsx @@ -12,10 +12,13 @@ export default function App() { const audioElement = useRef(null); async function startSession() { + const bot = window.location.href.split('/')[4].split('?')[0].split('#')[0] + const api = bot ? `https://feedbot-${bot}-app.azurewebsites.net` : 'http://localhost:7071' + // Get an ephemeral key from the Fastify server - const tokenResponse = await fetch("/token"); + const tokenResponse = await fetch(api+"/api/messages/realtime/token", {method: 'POST'}); const data = await tokenResponse.json(); - const EPHEMERAL_KEY = data.client_secret.value; + const EPHEMERAL_KEY = data.value; // Create a peer connection const pc = new RTCPeerConnection(); @@ -39,9 +42,9 @@ export default function App() { const offer = await pc.createOffer(); await pc.setLocalDescription(offer); - const baseUrl = "https://api.openai.com/v1/realtime"; - const model = "gpt-4o-realtime-preview-2024-12-17"; - const sdpResponse = await fetch(`${baseUrl}?model=${model}`, { + const baseUrl = "https://api.openai.com/v1/realtime/calls"; + const model = "gpt-realtime"; + const sdpResponse = await fetch(`${baseUrl}?model=${model}`,{ method: "POST", body: offer.sdp, headers: { @@ -56,6 +59,20 @@ export default function App() { }; await pc.setRemoteDescription(answer); + const location = sdpResponse.headers.get("Location"); + const callId = location?.split("/").pop(); + console.log('callId', callId); + + fetch(api+'/api/messages/realtime/calls', { + //fetch('https://feedbot-master-realtime-voice-app.azurewebsites.net/api/messages/realtime/calls?code=cb4ba4ab-93f9-4048-bd10-c4bda0b175d0', { + method: 'POST', + headers: {'Content-Type': 'application/json', 'Authorization': `Bearer ${EPHEMERAL_KEY}`}, + body: JSON.stringify({call_id: callId}) + }) + .then(response => response.text()) + .then(response => console.log('Bot hosting response', response)) + .catch(err => console.error(err)); + peerConnection.current = pc; } @@ -127,7 +144,7 @@ export default function App() { <> @@ -148,12 +165,12 @@ export default function App() {
- + />*/}
diff --git a/client/components/ToolPanel.jsx b/client/components/ToolPanel.jsx index f236f43e0..9be9ead9b 100644 --- a/client/components/ToolPanel.jsx +++ b/client/components/ToolPanel.jsx @@ -1,35 +1,51 @@ import { useEffect, useState } from "react"; -const functionDescription = ` -Call this function when a user asks for a color palette. -`; - const sessionUpdate = { type: "session.update", session: { + instructions: + "Řekni přesně: 'Děkuji, zapsáno. Přeji pěkný den!'", + voice: "marin", + temperature: 0.6, + input_audio_transcription: { + model: "whisper-1", + }, tools: [ { type: "function", - name: "display_color_palette", - description: functionDescription, + name: "get_appointment_slots", + description: + "Použij tuto funkci pro získání dostupných termínů, volitelně s filtrem daného dne.", parameters: { type: "object", strict: true, properties: { - theme: { + date: { type: "string", - description: "Description of the theme for the color scheme.", + description: "Preferované datum pro schůzku", }, - colors: { - type: "array", - description: "Array of five hex color codes based on the theme.", - items: { - type: "string", - description: "Hex color code", - }, + }, + }, + }, + { + type: "function", + name: "create_appointment", + description: + "Použij tuto funkci pro založení schůzky, povinná je volba data+času a volitelně poznámka", + parameters: { + type: "object", + strict: true, + properties: { + dateTime: { + type: "string", + description: "Vybrané datum a čas schůzky", + }, + note: { + type: "string", + description: "Volitelná poznámka", }, }, - required: ["theme", "colors"], + required: ["dateTime"], }, }, ], @@ -37,27 +53,15 @@ const sessionUpdate = { }, }; -function FunctionCallOutput({ functionCallOutput }) { - const { theme, colors } = JSON.parse(functionCallOutput.arguments); - - const colorBoxes = colors.map((color) => ( -
-

- {color} -

-
- )); - +function FunctionCallOutput({ functionCallOutputs }) { return (
-

Theme: {theme}

- {colorBoxes} 
-        {JSON.stringify(functionCallOutput, null, 2)}
+
); @@ -69,7 +73,7 @@ export default function ToolPanel({ events, }) { const [functionAdded, setFunctionAdded] = useState(false); - const [functionCallOutput, setFunctionCallOutput] = useState(null); + const [functionCallOutputs, setFunctionCallOutputs] = useState([]); useEffect(() => { if (!events || events.length === 0) return; @@ -77,6 +81,7 @@ export default function ToolPanel({ const firstEvent = events[events.length - 1]; if (!functionAdded && firstEvent.type === "session.created") { sendClientEvent(sessionUpdate); + sendClientEvent({ type: "response.create" }); setFunctionAdded(true); } @@ -86,19 +91,62 @@ export default function ToolPanel({ mostRecentEvent.response.output ) { mostRecentEvent.response.output.forEach((output) => { + let args = {}; + try { + args = JSON.parse(output.arguments || "{}"); + } catch (e) { + console.warn("Failed to parse arguments", output); + } + + if ( + output.type === "function_call" && + output.name === "get_appointment_slots" + ) { + setFunctionCallOutputs([...functionCallOutputs, output]); + setTimeout(() => { + sendClientEvent({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: output.call_id, + output: + `{"slots": ["${args.date}T10:00:00","${args.date}T14:30:00"]}` + /*!args.date || args.date === "2025-02-25"*/ + /*true + ? '{"slots": ["2025-02-2510:00:00","2025-02-25T14:30:00"]}' + : args.date === "2025-02-25" ? '{"slots": ["2025-02-2511:00:00","2025-02-25T15:30:00", "2025-02-25T17:00:00"]}' : '{"slots": []}',*/ + }, + }); + sendClientEvent({ + type: "response.create", + response: { + instructions: + "Dej uživateli na výběr jeden z navržených termínů [\"2025-02-2510:00:00\",\"2025-02-25T14:30:00\"]. Hodiny čti správně česky v prvním pádě např. 'v deset hodin', 've čtrnáct třicet' apod. Pokud nejsou žádné k dispozici tak požádej o jiný den případně o nejbližší možný. Nenebádej k výběru dní, pro které nemáš nalezené sloty.", + }, + }); + }, 200); + } + if ( output.type === "function_call" && - output.name === "display_color_palette" + output.name === "create_appointment" && + args.dateTime ) { - setFunctionCallOutput(output); + setFunctionCallOutputs([...functionCallOutputs, output]); setTimeout(() => { + sendClientEvent({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: output.call_id, + output: '{"selectedSlot": "' + args.dateTime + '"}', + }, + }); sendClientEvent({ type: "response.create", response: { - instructions: ` - ask for feedback about the color palette - don't repeat - the colors, just ask if they like the colors. - `, + instructions: + "Finálně potvrď uživateli objednání na vybraný čas (čti česky v prvním pádě např. 'čtrnáct třicet') zeptej se jestli nechce něco přidat do poznámky. Pokud ano tak znovu zavolej funkci create_appointment, pokud ne tak se jen hezky rozluč.", }, }); }, 500); @@ -110,17 +158,17 @@ export default function ToolPanel({ useEffect(() => { if (!isSessionActive) { setFunctionAdded(false); - setFunctionCallOutput(null); + setFunctionCallOutputs([]); } }, [isSessionActive]); return (
-

Color Palette Tool

+

Tool outputs

{isSessionActive ? ( - functionCallOutput ? ( - + functionCallOutputs.length > 0 ? ( + ) : (

Ask for advice on a color palette...

) diff --git a/client/entry-client.jsx b/client/entry-client.jsx new file mode 100644 index 000000000..a1f8925f8 --- /dev/null +++ b/client/entry-client.jsx @@ -0,0 +1,11 @@ +import { StrictMode } from "react"; +import ReactDOM from "react-dom/client"; +import App from "./components/App"; +import "./base.css"; + +ReactDOM.hydrateRoot( + document.getElementById("root"), + + + , +); \ No newline at end of file diff --git a/client/index.html b/client/index.html index e82bf38b7..cbf4e5908 100644 --- a/client/index.html +++ b/client/index.html @@ -2,10 +2,10 @@ - OpenAI Realtime Console - - - + Feedyou Realtime Console + + +
- + diff --git a/package.json b/package.json index a5b97a666..b803ed826 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "start": "node server.js", "build": "npm run build:client && npm run build:server", "build:client": "vite build --outDir dist/client --ssrManifest", - "build:server": "vite build --outDir dist/server --ssr /index.js", + "build:server": "vite build --outDir dist/server --ssr ./server.js", "devinstall": "zx ../../devinstall.mjs -- node server.js --dev", "lint": "eslint . --ext .js,.jsx --fix" }, diff --git a/server.js b/server.js index b97a81f8c..3cf086f51 100644 --- a/server.js +++ b/server.js @@ -32,19 +32,34 @@ await server.vite.ready(); // Server-side API route to return an ephemeral realtime session token server.get("/token", async () => { - const r = await fetch("https://api.openai.com/v1/realtime/sessions", { - method: "POST", - headers: { - Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, - "Content-Type": "application/json", + const sessionConfig = JSON.stringify({ + session: { + type: "realtime", + model: "gpt-realtime", + audio: { + input: { + turn_detection: { type: "semantic_vad", create_response: true } + }, + output: { + voice: "marin", + }, + }, + tracing: "auto" }, - body: JSON.stringify({ - model: "gpt-4o-realtime-preview-2024-12-17", - voice: "verse", - }), }); + const response = await fetch( + "https://api.openai.com/v1/realtime/client_secrets", + { + method: "POST", + headers: { + Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, + "Content-Type": "application/json", + }, + body: sessionConfig, + } + ); - return new Response(r.body, { + return new Response(response.body, { status: 200, headers: { "Content-Type": "application/json", diff --git a/src/pages/token.json.ts b/src/pages/token.json.ts index 5dabb2b1b..f19dd760c 100644 --- a/src/pages/token.json.ts +++ b/src/pages/token.json.ts @@ -1,3 +1,4 @@ +// WARN unused? export async function GET() { const r = await fetch("https://api.openai.com/v1/realtime/sessions", { method: "POST", @@ -6,8 +7,8 @@ export async function GET() { "Content-Type": "application/json", }, body: JSON.stringify({ - model: "gpt-4o-realtime-preview-2024-12-17", - voice: "verse", + model: "gpt-realtime", + voice: "marin" }), }); diff --git a/vite.config.js b/vite.config.js index ae27af278..2c498047e 100644 --- a/vite.config.js +++ b/vite.config.js @@ -7,6 +7,7 @@ import viteFastifyReact from "@fastify/react/plugin"; const path = fileURLToPath(import.meta.url); export default { + base: 'https://feedyou.blob.core.windows.net/realtime-console', root: join(dirname(path), "client"), plugins: [viteReact(), viteFastifyReact()], ssr: {