Macos playback issue

apps/desktop/src-tauri/src/frame_ws.rs

@@ -7,7 +7,7 @@ static TOTAL_BYTES_SENT: AtomicU64 = AtomicU64::new(0);
 static TOTAL_FRAMES_SENT: AtomicU32 = AtomicU32::new(0);
 static LAST_LOG_TIME: AtomicU64 = AtomicU64::new(0);
 
-const DOWNSCALE_PERCENT: u32 = 50;
+const DOWNSCALE_PERCENT: u32 = 75;
 const NV12_FORMAT_MAGIC: u32 = 0x4e563132;
 
 fn downscale_and_convert_to_nv12(
@@ -197,8 +197,6 @@ pub async fn create_watch_frame_ws(
             }
         }
 
-        let mut last_frame_number: Option<u32> = None;
-
         loop {
             tokio::select! {
                 msg = socket.recv() => {
@@ -217,10 +215,6 @@ pub async fn create_watch_frame_ws(
                 _ = camera_rx.changed() => {
                     let frame_opt = camera_rx.borrow_and_update().clone();
                     if let Some(frame) = frame_opt {
-                        if last_frame_number == Some(frame.frame_number) {
-                            continue;
-                        }
-                        last_frame_number = Some(frame.frame_number);
 
                         let (nv12_data, scaled_width, scaled_height) = downscale_and_convert_to_nv12(
                             &frame.data,

apps/desktop/src-tauri/src/lib.rs

@@ -1631,8 +1631,16 @@ async fn set_project_config(
 async fn update_project_config_in_memory(
     editor_instance: WindowEditorInstance,
     config: ProjectConfiguration,
+    frame_number: Option<u32>,
+    fps: Option<u32>,
+    resolution_base: Option<XY<u32>>,
 ) -> Result<(), String> {
     editor_instance.project_config.0.send(config).ok();
+    if let (Some(frame), Some(f), Some(res)) = (frame_number, fps, resolution_base) {
+        editor_instance.preview_tx.send_modify(|v| {
+            *v = Some((frame, f, res));
+        });
+    }
     Ok(())
 }
 

apps/desktop/src/routes/editor/Editor.tsx

@@ -208,9 +208,21 @@ function Inner() {
 
 	const doConfigUpdate = async (time: number) => {
 		const config = serializeProjectConfiguration(project);
-		await commands.updateProjectConfigInMemory(config);
-		canvasControls()?.resetFrameState();
-		renderFrame(time);
+		const frameNumber = Math.max(Math.floor(time * FPS), 0);
+		const resBase = previewResolutionBase();
+		try {
+			await commands.updateProjectConfigInMemory(
+				config,
+				frameNumber,
+				FPS,
+				resBase,
+			);
+		} catch (e) {
+			console.error(
+				"[Editor] doConfigUpdate - ERROR sending config to Rust:",
+				e,
+			);
+		}
 	};
 	const throttledConfigUpdate = throttle(doConfigUpdate, 1000 / FPS);
 	const trailingConfigUpdate = debounce(doConfigUpdate, 1000 / FPS + 16);
@@ -220,10 +232,13 @@ function Inner() {
 	};
 	createEffect(
 		on(
-			() => trackDeep(project),
 			() => {
-				updateConfigAndRender(editorState.playbackTime);
+				trackDeep(project);
+			},
+			() => {
+				updateConfigAndRender(frameNumberToRender());
 			},
+			{ defer: true },
 		),
 	);
 

apps/desktop/src/routes/editor/Player.tsx

@@ -461,7 +461,10 @@ function PreviewCanvas() {
 
 	const hasRenderedFrame = () => canvasControls()?.hasRenderedFrame() ?? false;
 
-	const canvasTransferredRef = { current: false };
+	const canvasInitializedRef = { current: false };
+	const [canvasRef, setCanvasRef] = createSignal<HTMLCanvasElement | null>(
+		null,
+	);
 
 	const [canvasContainerRef, setCanvasContainerRef] =
 		createSignal<HTMLDivElement>();
@@ -487,103 +490,99 @@ function PreviewCanvas() {
 		}
 	});
 
-	const isWindows = navigator.userAgent.includes("Windows");
-
-	const initCanvas = (canvas: HTMLCanvasElement) => {
-		if (canvasTransferredRef.current) return;
+	createEffect(() => {
+		const canvas = canvasRef();
 		const controls = canvasControls();
-		if (!controls) return;
+		console.warn("[Player] Canvas init effect", {
+			hasCanvas: !!canvas,
+			hasControls: !!controls,
+			alreadyInit: canvasInitializedRef.current,
+		});
+		if (canvasInitializedRef.current || !canvas || !controls) return;
 
-		if (isWindows) {
-			controls.initDirectCanvas(canvas);
-			canvasTransferredRef.current = true;
-			return;
-		}
+		console.warn("[Player] Initializing canvas", {
+			canvasId: canvas.id,
+			isConnected: canvas.isConnected,
+		});
+		controls.initDirectCanvas(canvas);
+		canvasInitializedRef.current = true;
+		console.warn("[Player] Canvas initialized successfully");
+	});
 
-		try {
-			const offscreen = canvas.transferControlToOffscreen();
-			controls.initCanvas(offscreen);
-			canvasTransferredRef.current = true;
-		} catch (e) {
-			console.error("[PreviewCanvas] Failed to transfer canvas:", e);
+	const padding = 4;
+	const frameWidth = () => latestFrame()?.width ?? 1920;
+	const frameHeight = () => latestFrame()?.height ?? 1080;
+
+	const availableWidth = () =>
+		Math.max(debouncedBounds().width - padding * 2, 0);
+	const availableHeight = () =>
+		Math.max(debouncedBounds().height - padding * 2, 0);
+
+	const containerAspect = () => {
+		const width = availableWidth();
+		const height = availableHeight();
+		if (width === 0 || height === 0) return 1;
+		return width / height;
+	};
+
+	const frameAspect = () => {
+		const width = frameWidth();
+		const height = frameHeight();
+		if (width === 0 || height === 0) return containerAspect();
+		return width / height;
+	};
+
+	const size = () => {
+		let width: number;
+		let height: number;
+		if (frameAspect() < containerAspect()) {
+			height = availableHeight();
+			width = height * frameAspect();
+		} else {
+			width = availableWidth();
+			height = width / frameAspect();
 		}
+
+		return { width, height };
 	};
 
+	const hasFrame = () => !!latestFrame();
+
 	return (
 		<div
 			ref={setCanvasContainerRef}
 			class="relative flex-1 justify-center items-center"
 			style={{ contain: "layout style" }}
 		>
-			<Show when={latestFrame()}>
-				{(currentFrame) => {
-					const padding = 4;
-					const frameWidth = () => currentFrame().width;
-					const frameHeight = () => currentFrame().height;
-
-					const availableWidth = () =>
-						Math.max(debouncedBounds().width - padding * 2, 0);
-					const availableHeight = () =>
-						Math.max(debouncedBounds().height - padding * 2, 0);
-
-					const containerAspect = () => {
-						const width = availableWidth();
-						const height = availableHeight();
-						if (width === 0 || height === 0) return 1;
-						return width / height;
-					};
-
-					const frameAspect = () => {
-						const width = frameWidth();
-						const height = frameHeight();
-						if (width === 0 || height === 0) return containerAspect();
-						return width / height;
-					};
-
-					const size = () => {
-						let width: number;
-						let height: number;
-						if (frameAspect() < containerAspect()) {
-							height = availableHeight();
-							width = height * frameAspect();
-						} else {
-							width = availableWidth();
-							height = width / frameAspect();
-						}
-
-						return { width, height };
-					};
-
-					return (
-						<div class="flex overflow-hidden absolute inset-0 justify-center items-center h-full">
-							<div
-								class="relative"
-								style={{
-									width: `${size().width}px`,
-									height: `${size().height}px`,
-									contain: "strict",
-								}}
-							>
-								<canvas
-									style={{
-										width: `${size().width}px`,
-										height: `${size().height}px`,
-										"image-rendering": "auto",
-										"background-color": "#000000",
-										...(hasRenderedFrame() ? gridStyle : {}),
-									}}
-									ref={initCanvas}
-									id="canvas"
-									width={frameWidth()}
-									height={frameHeight()}
-								/>
-								<MaskOverlay size={size()} />
-								<TextOverlay size={size()} />
-							</div>
-						</div>
-					);
-				}}
-			</Show>
+			<div
+				class="flex overflow-hidden absolute inset-0 justify-center items-center h-full"
+				style={{ visibility: hasFrame() ? "visible" : "hidden" }}
+			>
+				<div
+					class="relative"
+					style={{
+						width: `${size().width}px`,
+						height: `${size().height}px`,
+						contain: "strict",
+					}}
+				>
+					<canvas
+						style={{
+							width: `${size().width}px`,
+							height: `${size().height}px`,
+							"image-rendering": "auto",
+							"background-color": "#000000",
+							...(hasRenderedFrame() ? gridStyle : {}),
+						}}
+						ref={setCanvasRef}
+						id="canvas"
+					/>
+					<Show when={hasFrame()}>
+						<MaskOverlay size={size()} />
+						<TextOverlay size={size()} />
+					</Show>
+				</div>
+			</div>
 		</div>
 	);
 }

apps/desktop/src/utils/frame-worker.ts

@@ -621,6 +621,85 @@ function renderLoop() {
 		const frame = frameToRender;
 
 		if (frame.mode === "webgpu" && !webgpuRenderer) {
+			if (renderMode === "pending") {
+				_rafId = requestAnimationFrame(renderLoop);
+				return;
+			}
+			if (renderMode === "canvas2d" && offscreenCanvas && offscreenCtx) {
+				frameQueue.splice(frameIndex, 1);
+				lastRenderedFrameNumber = frame.timing.frameNumber;
+
+				if (
+					offscreenCanvas.width !== frame.width ||
+					offscreenCanvas.height !== frame.height
+				) {
+					offscreenCanvas.width = frame.width;
+					offscreenCanvas.height = frame.height;
+				}
+
+				let rgbaData: Uint8ClampedArray;
+				if (frame.pixelFormat === "nv12") {
+					rgbaData = convertNv12ToRgba(
+						frame.data,
+						frame.width,
+						frame.height,
+						frame.yStride,
+					);
+				} else {
+					const expectedRowBytes = frame.width * 4;
+					if (frame.strideBytes === expectedRowBytes) {
+						rgbaData = frame.data;
+					} else {
+						const expectedLength = expectedRowBytes * frame.height;
+						if (!strideBuffer || strideBufferSize < expectedLength) {
+							strideBuffer = new Uint8ClampedArray(expectedLength);
+							strideBufferSize = expectedLength;
+						}
+						for (let row = 0; row < frame.height; row += 1) {
+							const srcStart = row * frame.strideBytes;
+							const destStart = row * expectedRowBytes;
+							strideBuffer.set(
+								frame.data.subarray(srcStart, srcStart + expectedRowBytes),
+								destStart,
+							);
+						}
+						rgbaData = strideBuffer.subarray(0, expectedLength);
+					}
+				}
+
+				if (
+					!cachedImageData ||
+					cachedWidth !== frame.width ||
+					cachedHeight !== frame.height
+				) {
+					cachedImageData = new ImageData(frame.width, frame.height);
+					cachedWidth = frame.width;
+					cachedHeight = frame.height;
+				}
+				cachedImageData.data.set(rgbaData);
+				offscreenCtx.putImageData(cachedImageData, 0, 0);
+
+				if (frame.releaseCallback) {
+					frame.releaseCallback();
+				}
+
+				self.postMessage({
+					type: "frame-rendered",
+					width: frame.width,
+					height: frame.height,
+				} satisfies FrameRenderedMessage);
+
+				const shouldContinue =
+					frameQueue.length > 0 ||
+					(useSharedBuffer && consumer && !consumer.isShutdown());
+
+				if (shouldContinue) {
+					_rafId = requestAnimationFrame(renderLoop);
+				} else {
+					rafRunning = false;
+				}
+				return;
+			}
 			_rafId = requestAnimationFrame(renderLoop);
 			return;
 		}