From 13c20197fe97af1ae1a6b7c6a351aead902e9c66 Mon Sep 17 00:00:00 2001 From: Vitor Franklin Date: Thu, 2 Jul 2026 07:51:07 -0300 Subject: [PATCH 1/3] Adjusting frontend and backend to deal with audio and video separated --- backend/app/streaming/ffmpeg_webrtc.py | 1 + frontend/app.js | 87 +++++++++++++++++++------- frontend/index.html | 1 + 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/backend/app/streaming/ffmpeg_webrtc.py b/backend/app/streaming/ffmpeg_webrtc.py index 4b74f4c..96fae01 100644 --- a/backend/app/streaming/ffmpeg_webrtc.py +++ b/backend/app/streaming/ffmpeg_webrtc.py @@ -56,6 +56,7 @@ def start(self, game: Any) -> None: "-fps_mode", "passthrough", *profile.filter_args, *profile.codec_args, + "-af", "aresample=async=1000:first_pts=0", "-c:a", "libopus", "-b:a", "96k", "-pkt_size", "1200", "-f", "rtsp", settings.MEDIAMTX_RTSP_URL, diff --git a/frontend/app.js b/frontend/app.js index d7c9ebb..af1183c 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -20,7 +20,14 @@ const cifsUsernameInput = document.getElementById("cifs-username"); const cifsPasswordInput = document.getElementById("cifs-password"); let whepUrl = null; -let peerConnection = null; +//let peerConnection = null; -- + +const gameVideo = document.getElementById("game-video"); +const gameAudio = document.getElementById("game-audio"); + +let whepUrl = null; +let videoPeerConnection = null; +let audioPeerConnection = null; // Mesmo vocabulario do backend (backend/app/input/keymap.py) - nomes // logicos que correspondem aos binds input_player1_* do RetroArch. @@ -211,6 +218,35 @@ async function scanLibrary() { } } +async function connectWhepTrack(kind, onTrack) { + const pc = new RTCPeerConnection(); + + pc.ontrack = onTrack; + pc.addTransceiver(kind, { direction: "recvonly" }); + + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + + const response = await fetch(whepUrl, { + method: "POST", + headers: { "Content-Type": "application/sdp" }, + body: offer.sdp, + }); + + if (!response.ok) { + pc.close(); + throw new Error(`WHEP ${kind} falhou: ${response.status}`); + } + + const answerSdp = await response.text(); + await pc.setRemoteDescription({ + type: "answer", + sdp: answerSdp, + }); + + return pc; +} + function showPlayView() { idleView.classList.add("hidden"); playView.classList.remove("hidden"); @@ -225,38 +261,41 @@ async function connectWhep() { if (!whepUrl) { await loadConfig(); } - const pc = new RTCPeerConnection(); - peerConnection = pc; - - pc.ontrack = (event) => { - gameVideo.srcObject = event.streams[0]; - }; - pc.addTransceiver("video", { direction: "recvonly" }); - pc.addTransceiver("audio", { direction: "recvonly" }); - const offer = await pc.createOffer(); - await pc.setLocalDescription(offer); - - const resp = await fetch(whepUrl, { - method: "POST", - headers: { "Content-Type": "application/sdp" }, - body: offer.sdp, + videoPeerConnection = await connectWhepTrack("video", (event) => { + gameVideo.srcObject = new MediaStream([event.track]); }); - if (!resp.ok) { - throw new Error(`WHEP falhou: ${resp.status}`); - } + try { + audioPeerConnection = await connectWhepTrack("audio", (event) => { + gameAudio.srcObject = new MediaStream([event.track]); - const answerSdp = await resp.text(); - await pc.setRemoteDescription({ type: "answer", sdp: answerSdp }); + gameAudio.play().catch((error) => { + console.warn("Autoplay do áudio bloqueado:", error); + }); + }); + } catch (error) { + // Falha do áudio não deve impedir o jogo. + console.warn("Conexão de áudio indisponível:", error); + audioPeerConnection = null; + } } function disconnectWhep() { - if (peerConnection) { - peerConnection.close(); - peerConnection = null; + if (videoPeerConnection) { + videoPeerConnection.close(); + videoPeerConnection = null; } + + if (audioPeerConnection) { + audioPeerConnection.close(); + audioPeerConnection = null; + } + gameVideo.srcObject = null; + + gameAudio.pause(); + gameAudio.srcObject = null; } function connectInputSocket() { diff --git a/frontend/index.html b/frontend/index.html index 846c756..19c96d4 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -45,6 +45,7 @@

RetroHost

+
From f2ee40fdd305235b3408abda1ebf8c212bbaa067 Mon Sep 17 00:00:00 2001 From: Vitor Franklin Date: Thu, 2 Jul 2026 07:58:00 -0300 Subject: [PATCH 2/3] Fixing mistakes --- frontend/app.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/app.js b/frontend/app.js index af1183c..a8017a1 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -4,6 +4,7 @@ const scanBtn = document.getElementById("scan-btn"); const idleView = document.getElementById("idle-view"); const playView = document.getElementById("play-view"); const gameVideo = document.getElementById("game-video"); +const gameAudio = document.getElementById("game-audio"); const fullscreenBtn = document.getElementById("fullscreen-btn"); const endGameBtn = document.getElementById("end-game-btn"); const backgroundGameBanner = document.getElementById("background-game-banner"); @@ -22,10 +23,6 @@ const cifsPasswordInput = document.getElementById("cifs-password"); let whepUrl = null; //let peerConnection = null; -- -const gameVideo = document.getElementById("game-video"); -const gameAudio = document.getElementById("game-audio"); - -let whepUrl = null; let videoPeerConnection = null; let audioPeerConnection = null; From 3c3e9a56062799b231aa3bc27d8f34c91c42491e Mon Sep 17 00:00:00 2001 From: Vitor Franklin Date: Thu, 2 Jul 2026 20:23:24 -0300 Subject: [PATCH 3/3] Documentation and adjusting homegame name --- ARCHITECTURE.md | 23 ++++++++++++++++++++++- Dockerfile | 4 ++-- README.md | 12 +++++++----- SECURITY.md | 8 ++++---- backend/app/core/config.py | 2 +- backend/app/input/uinput_keyboard.py | 2 +- backend/app/main.py | 2 +- backend/app/services/storage.py | 2 +- compose.yml | 5 +++-- config/mediamtx.yml | 2 +- container/sdl_input_preload.c | 2 +- scripts/docker-entrypoint.sh | 2 +- scripts/setup_streaming.sh | 2 +- scripts/whep_test.html | 2 +- 14 files changed, 47 insertions(+), 23 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b3d6977..e7eff90 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -44,9 +44,12 @@ Same principle as Steam Link, Moonlight, or Stadia: capture, encode, transmit, d └─────────────────────────────────────────────────────────────────────────┼──┼───┘ │ │ Browser ───── WebSocket (input) ────────────────────────┘ │ - ◄──── WebRTC video+audio (WHEP) ───────────────────┘ + ◄──── WebRTC video (WHEP) ────────────────────────┘ + ◄──── WebRTC audio (WHEP, separate connection) ────┘ ``` +Video and audio are fetched over **two independent WHEP requests / `RTCPeerConnection`s**, not two tracks on one connection — see "Why two WebRTC connections" below. + --- ## Components @@ -66,8 +69,26 @@ Same principle as Steam Link, Moonlight, or Stadia: capture, encode, transmit, d - `-bf 0` is set on **all** encoder profiles. WebRTC rejects H.264 streams that contain B-frames ("WebRTC doesn't support H264 streams with B-frames"). `-tune zerolatency` in libx264 is supposed to disable them but is not guaranteed across all builds — explicit `-bf 0` is the reliable fix. - `-fflags nobuffer -flags low_delay` and `-fps_mode passthrough` reduce buffering latency. - GOP of 15 frames (`-g 15`) was validated to reduce jitter buffer from ~200 ms to ~100 ms on the Pi. +- `-af aresample=async=1000:first_pts=0` resamples audio to a constant rate anchored at PTS 0, compensating for drift introduced by the FIFO/Matroska path now that audio and video are consumed by two independent WebRTC connections (see below) with no shared clock to resync them. - Implemented in `backend/app/streaming/ffmpeg_webrtc.py` (`FFmpegStreamingProvider`). +### 2.1 Why two WebRTC connections (video/audio split) + +Initially, a single `RTCPeerConnection` carried both the video and audio tracks (two transceivers on one connection), matching how WHEP is normally used. Measured with `RTCPeerConnection.getStats()` on the Pi 3, this configuration showed a **video jitter buffer of ~289-459 ms** — far above what the encoder alone accounts for (see "Known hardware limits" below). + +Root cause: when audio and video share one `RTCPeerConnection`, the browser's jitter buffer logic holds the video queue back to preserve lip-sync with the audio track's own (larger) buffering requirements — inflating video latency to match audio, not the other way around. + +**Fix**: the frontend opens two independent WHEP requests against the same `whep_url`, each negotiating a single `recvonly` transceiver (`connectWhepTrack("video", ...)` and `connectWhepTrack("audio", ...)` in `frontend/app.js`), resulting in two separate `RTCPeerConnection`s and two `