How to get a callback when a frame is decompressed? #632

MotivaCG · 2024-07-23T17:54:43Z

I need a per frame synchronization so HTMLVideoElement requestVideoFrameCallback is not enough and I need a signal that is triggered once the frame has been decoded.

Is there something like that on ogv.js? (I don't think so)
Is there any easy modification to make that? (Maybe with a tip I could do it and commit it back as a patch)

I really appreciate any help you can provide.

bvibber · 2024-07-23T22:06:09Z

I think the simplest way is to hack OGVWrapperCodec.decodeFrame to wrap a custom callback. Something like this ought to do it on top of a stock ogv.js distribution, or you could modify the main code and rebuild it:

// player._codec is an OGVWrapperCodec but this class is not exposed globally
let proto = Object.getPrototypeOf(player._codec);
proto._decodeFrameOrig = proto.decodeFrame;
proto.decodeFrame = function(callback) {
  this._decodeFrameOrig((ok) => {
    if (ok) {
      let frame = this.frameBuffer;
      // <- add custom processing
    }
    callback(ok);
  }]);
}

Note this will fire while decoding during seeking operations as well as when buffering up data for playback.

Let me know if you have any trouble!

MotivaCG · 2024-07-24T17:04:13Z

Great, I've made this change on a basic javascript implementation in a basic HTML file, out of the ogv.js api:

player.addEventListener('loadedmetadata', () => {
    let proto = Object.getPrototypeOf(player._codec);
    if (!proto._decodeFrameOrig) {
        proto._decodeFrameOrig = proto.decodeFrame;
        proto.decodeFrame = function(callback) {
            this._decodeFrameOrig((ok) => {
                if (ok) {
                    let frame = this.frameBuffer;
                    // Add custom processing here
                    console.log('Decoded frame k:', frame.keyframeTimestamp," t:", frame.timestamp);
                }
                callback(ok);
            });
        }
    }
});

I have a couple of questions:

Is there any potential issue or recommendation against modifying the decodeFrame function directly within the HTML?

Specifically, are there any best practices or potential pitfalls I should be aware of when implementing this directly in the HTML?

Is there a way to use this custom decoding with a video element?

My ultimate goal is to use the video element as a texture in Babylon.js. Are there any examples or recommended approaches for integrating ogv.js with a web engine for this purpose?

Thank you in advance!

bvibber · 2024-07-24T17:42:27Z

by modifying directly do you mean in the ogv.js blob? should be safe enough, just remember you'll need to redo it on any updates. ;)
i did some experiments with sending the output into an actual <video> element using canvas.captureStream(), with mixed results. this might be more reliable now; I think I took out the experimental code but you can wrap around it by getting at the player's canvas sub-element and doing something like:

let canvas = player._canvas;
let frameRate = 30;
let stream = canvas.captureStream(frameRate);
let url = URL.createObjectURL(stream);

let video = ...; // your target <video> element
video.src = url;
video.muted = true; // so you can start the video-only stream programmatically
video.play();

Alternately, you might be able to use the canvas directly as a texture source, skipping the pipe into the video element.

https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/captureStream

MotivaCG · 2024-07-24T22:06:13Z

I've made that directly into the html

<!doctype html>
<html>
<head>
	<meta charset="UTF-8">
	<meta name="viewport" content="initial-scale=1.0, user-scalable=no" />
	<title>Minimal Ogv.js example</title>
	<script src="lib/ogv.js?version=OGV_VERSION"></script>
</head>
<body>

<div id="videoarea"></div>

<select onchange="switchmedia(this)">
	<option value="media/llama-drama-av1.webm">Llama drama (AV1, Opus; 180p)</option>
	<option value="media/curiosity.ogv">Curiosity's Seven Minutes of Terror (Vorbis, Theora; 160p)</option>
	<option value="media/ehren-paper_lights-96.opus">Ehren - Paper Lights (Opus)</option>
	<option value="media/pixel_aspect_ratio.ogg">Theora Test suite pixel_aspect_ratio.ogg</option>
	<option value="media/BotellaTapon.webm">BotellaTapon.webm</option>
</select>
<div>
<button onclick="play(sampleFile);">Play</button><button onclick="pause();">Pause/Unpause</button>
</div>


<script language="JavaScript">

var player = null;
var paused = false;
var sampleFile = 'media/llama-drama-av1.webm';

player = new OGVPlayer();
//document.getElementById("videoarea").appendChild(player);
// Hack para añadir callback personalizado al decodificar fotogramas
player.addEventListener('loadedmetadata', () => {
                let proto = Object.getPrototypeOf(player._codec);
                if (!proto._decodeFrameOrig) {
                    proto._decodeFrameOrig = proto.decodeFrame;
                    proto.decodeFrame = function(callback) {
                        this._decodeFrameOrig((ok) => {
                            if (ok) {
                                let frame = this.frameBuffer;
                                // Añade aquí el procesamiento personalizado del fotograma
                                console.log('Fotograma decodificado k:', frame.keyframeTimestamp," t:", frame.timestamp);
                            }
                            callback(ok);
                        });
                    }
                }
            });


function play(src) {
	//player = document.createElement('video');
	player.src = src;
	player.play();	
}

function pause() {
	if(player) {
		if(!paused) {
			player.pause();
			paused = true;
		} else {
			player.play();
			paused = false;
		}
	}
}

function switchmedia(src) {
	sampleFile = src.value;
	play(sampleFile);
}

</script>

</body></html>

But I'm not sure if it will introduce some performance penalties.

"modifying directly do you mean in the ogv.js blob?"
Do you mean the minimized one? I'm worried that this could be hard since ... it is packed and minimized.

Maybe I could do that on the player and run makefile again.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

How to get a callback when a frame is decompressed? #632

How to get a callback when a frame is decompressed? #632

MotivaCG commented Jul 23, 2024

bvibber commented Jul 23, 2024 •

edited

Loading

MotivaCG commented Jul 24, 2024 •

edited

Loading

bvibber commented Jul 24, 2024 •

edited

Loading

MotivaCG commented Jul 24, 2024

How to get a callback when a frame is decompressed? #632

How to get a callback when a frame is decompressed? #632

Comments

MotivaCG commented Jul 23, 2024

bvibber commented Jul 23, 2024 • edited Loading

MotivaCG commented Jul 24, 2024 • edited Loading

bvibber commented Jul 24, 2024 • edited Loading

MotivaCG commented Jul 24, 2024

bvibber commented Jul 23, 2024 •

edited

Loading

MotivaCG commented Jul 24, 2024 •

edited

Loading

bvibber commented Jul 24, 2024 •

edited

Loading