Merge pull request #530 from chcunningham/audio-video-player-sample

Add audio-video-player sample

Author: chcunningham

samples/_headers

@@ -0,0 +1,4 @@
+/*
+  Cross-Origin-Opener-Policy: same-origin
+  Cross-Origin-Embedder-Policy: require-corp
+  

samples/audio-video-player/audio_renderer.js

@@ -0,0 +1,229 @@
+import { MP4PullDemuxer } from "./mp4_pull_demuxer.js";
+import { RingBuffer } from "./third_party/ringbufjs/ringbuf.js";
+
+const DATA_BUFFER_DECODE_TARGET_DURATION = 0.3;
+const DATA_BUFFER_DURATION = 0.6;
+const DECODER_QUEUE_SIZE_MAX = 5;
+const ENABLE_DEBUG_LOGGING = false;
+
+function debugLog(msg) {
+  if (!ENABLE_DEBUG_LOGGING) {
+    return;
+  }
+  console.debug(msg);
+}
+
+export class AudioRenderer {
+  async initialize(fileUri) {
+    this.fillInProgress = false;
+    this.playing = false;
+
+    this.demuxer = new MP4PullDemuxer(fileUri);
+
+    let trackInfo = await this.demuxer.getAudioTrackInfo();
+    this.demuxer.selectAudio();
+
+    this.decoder = new AudioDecoder({
+      output: this.bufferAudioData.bind(this),
+      error: e => console.error(e)
+    });
+    const config = {
+      codec: trackInfo.codec,
+      sampleRate: trackInfo.sampleRate,
+      numberOfChannels: trackInfo.numberOfChannels,
+      description: trackInfo.extradata
+    };
+    this.sampleRate = trackInfo.sampleRate;
+    this.channelCount = trackInfo.numberOfChannels;
+
+    debugLog(config);
+
+    console.assert(AudioDecoder.isConfigSupported(config));
+    this.decoder.configure(config);
+
+    // Initialize the ring buffer between the decoder and the real-time audio
+    // rendering thread. The AudioRenderer has buffer space for approximately
+    // 500ms of decoded audio ahead.
+    let sampleCountIn500ms =
+      DATA_BUFFER_DURATION * this.sampleRate * trackInfo.numberOfChannels;
+    let sab = RingBuffer.getStorageForCapacity(
+      sampleCountIn500ms,
+      Float32Array
+    );
+    this.ringbuffer = new RingBuffer(sab, Float32Array);
+    this.interleavingBuffers = [];
+
+    this.init_resolver = null;
+    let promise = new Promise(resolver => (this.init_resolver = resolver));
+
+    this.fillDataBuffer();
+    return promise;
+  }
+
+  play() {
+    // resolves when audio has effectively started: this can take some time if using
+    // bluetooth, for example.
+    debugLog("playback start");
+    this.playing = true;
+    this.fillDataBuffer();
+  }
+
+  pause() {
+    debugLog("playback stop");
+    this.playing = false;
+  }
+
+  makeChunk(sample) {
+    const type = sample.is_sync ? "key" : "delta";
+    const pts_us = (sample.cts * 1000000) / sample.timescale;
+    const duration_us = (sample.duration * 1000000) / sample.timescale;
+    return new EncodedAudioChunk({
+      type: type,
+      timestamp: pts_us,
+      duration: duration_us,
+      data: sample.data
+    });
+  }
+
+  async fillDataBuffer() {
+    // This method is called from multiple places to ensure the buffer stays
+    // healthy. Sometimes these calls may overlap, but at any given point only
+    // one call is desired.
+    if (this.fillInProgress)
+      return;
+
+    this.fillInProgress = true;
+    // This should be this file's ONLY call to the *Internal() variant of this method.
+    await this.fillDataBufferInternal();
+    this.fillInProgress = false;
+  }
+
+  async fillDataBufferInternal() {
+    debugLog(`fillDataBufferInternal()`);
+
+    if (this.decoder.decodeQueueSize >= DECODER_QUEUE_SIZE_MAX) {
+      debugLog('\tdecoder saturated');
+      // Some audio decoders are known to delay output until the next input.
+      // Make sure the DECODER_QUEUE_SIZE is big enough to avoid stalling on the
+      // return below. We're relying on decoder output callback to trigger
+      // another call to fillDataBuffer().
+      console.assert(DECODER_QUEUE_SIZE_MAX >= 2);
+      return;
+    }
+
+    let usedBufferElements = this.ringbuffer.capacity() - this.ringbuffer.available_write();
+    let usedBufferSecs = usedBufferElements / (this.channelCount * this.sampleRate);
+    let pcntOfTarget = 100 * usedBufferSecs / DATA_BUFFER_DECODE_TARGET_DURATION;
+    if (usedBufferSecs >= DATA_BUFFER_DECODE_TARGET_DURATION) {
+      debugLog(`\taudio buffer full usedBufferSecs: ${usedBufferSecs} pcntOfTarget: ${pcntOfTarget}`);
+
+      // When playing, schedule timeout to periodically refill buffer. Don't
+      // bother scheduling timeout if decoder already saturated. The output
+      // callback will call us back to keep filling.
+      if (this.playing)
+        // Timeout to arrive when buffer is half empty.
+        setTimeout(this.fillDataBuffer.bind(this), 1000 * usedBufferSecs / 2);
+
+      // Initialize() is done when the buffer fills for the first time.
+      if (this.init_resolver) {
+        this.init_resolver();
+        this.init_resolver = null;
+      }
+
+      // Buffer full, so no further work to do now.
+      return;
+    }
+
+    // Decode up to the buffering target or until decoder is saturated.
+    while (usedBufferSecs < DATA_BUFFER_DECODE_TARGET_DURATION &&
+      this.decoder.decodeQueueSize < DECODER_QUEUE_SIZE_MAX) {
+      debugLog(`\tMore samples. usedBufferSecs:${usedBufferSecs} < target:${DATA_BUFFER_DECODE_TARGET_DURATION}.`);
+      let sample = await this.demuxer.readSample();
+      this.decoder.decode(this.makeChunk(sample));
+
+      // NOTE: awaiting the demuxer.readSample() above will also give the
+      // decoder output callbacks a chance to run, so we may see usedBufferSecs
+      // increase.
+      usedBufferElements = this.ringbuffer.capacity() - this.ringbuffer.available_write();
+      usedBufferSecs = usedBufferElements / (this.channelCount * this.sampleRate);
+    }
+
+    if (ENABLE_DEBUG_LOGGING) {
+      let logPrefix = usedBufferSecs >= DATA_BUFFER_DECODE_TARGET_DURATION ?
+          '\tbuffered enough' : '\tdecoder saturated';
+      pcntOfTarget = 100 * usedBufferSecs / DATA_BUFFER_DECODE_TARGET_DURATION;
+      debugLog(logPrefix + `; bufferedSecs:${usedBufferSecs} pcntOfTarget: ${pcntOfTarget}`);
+    }
+  }
+
+  bufferHealth() {
+    return (1 - this.ringbuffer.available_write() / this.ringbuffer.capacity()) * 100;
+  }
+
+  // From a array of Float32Array containing planar audio data `input`, writes
+  // interleaved audio data to `output`. Start the copy at sample
+  // `inputOffset`: index of the sample to start the copy from
+  // `inputSamplesToCopy`: number of input samples to copy
+  // `output`: a Float32Array to write the samples to
+  // `outputSampleOffset`: an offset in `output` to start writing
+  interleave(inputs, inputOffset, inputSamplesToCopy, output, outputSampleOffset) {
+    if (inputs.length * inputs[0].length < output.length) {
+      throw `not enough space in destination (${inputs.length * inputs[0].length} < ${output.length}})`
+    }
+    let channelCount = inputs.length;
+    let outIdx = outputSampleOffset;
+    let inputIdx = Math.floor(inputOffset / channelCount);
+    var channel = inputOffset % channelCount;
+    for (var i = 0; i < inputSamplesToCopy; i++) {
+      output[outIdx++] = inputs[channel][inputIdx];
+      if (++channel == inputs.length) {
+        channel = 0;
+        inputIdx++;
+      }
+    }
+  }
+
+  bufferAudioData(data) {
+    if (this.interleavingBuffers.length != data.numberOfChannels) {
+      this.interleavingBuffers = new Array(this.channelCount);
+      for (var i = 0; i < this.interleavingBuffers.length; i++) {
+        this.interleavingBuffers[i] = new Float32Array(data.numberOfFrames);
+      }
+    }
+
+    debugLog(`bufferAudioData() ts:${data.timestamp} durationSec:${data.duration / 1000000}`);
+    // Write to temporary planar arrays, and interleave into the ring buffer.
+    for (var i = 0; i < this.channelCount; i++) {
+      data.copyTo(this.interleavingBuffers[i], { planeIndex: i });
+    }
+    // Write the data to the ring buffer. Because it wraps around, there is
+    // potentially two copyTo to do.
+    let wrote = this.ringbuffer.writeCallback(
+      data.numberOfFrames * data.numberOfChannels,
+      (first_part, second_part) => {
+        this.interleave(this.interleavingBuffers, 0, first_part.length, first_part, 0);
+        this.interleave(this.interleavingBuffers, first_part.length, second_part.length, second_part, 0);
+      }
+    );
+
+    // FIXME - this could theoretically happen since we're pretty agressive
+    // about saturating the decoder without knowing the size of the
+    // AudioData.duration vs ring buffer capacity.
+    console.assert(wrote == data.numberOfChannels * data.numberOfFrames, 'Buffer full, dropping data!')
+
+    // Logging maxBufferHealth below shows we currently max around 73%, so we're
+    // safe from the assert above *for now*. We should add an overflow buffer
+    // just to be safe.
+    // let bufferHealth = this.bufferHealth();
+    // if (!('maxBufferHealth' in this))
+    //   this.maxBufferHealth = 0;
+    // if (bufferHealth > this.maxBufferHealth) {
+    //   this.maxBufferHealth = bufferHealth;
+    //   console.log(`new maxBufferHealth:${this.maxBufferHealth}`);
+    // }
+
+    // fillDataBuffer() gives up if too much decode work is queued. Keep trying
+    // now that we've finished some.
+    this.fillDataBuffer();
+  }
+}

samples/audio-video-player/audio_video_player.html

@@ -0,0 +1,156 @@
+<!doctype html>
+<style>
+  body {
+    font-family: sans-serif;
+    color: #444;
+    font-weight: 300;
+    font-size:  larger;
+  }
+  button {
+    font-size: larger;
+  }
+  #controls {
+    margin-bottom: 10px;
+  }
+  #loading {
+    font-size: 2em;
+  }
+  .monospace {
+    font-family: monospace;
+  }
+  div#container {
+    margin: 0 auto 0 auto;
+    max-width: 60em;
+    padding: 1em 1.5em 1.3em 1.5em;
+  }
+  canvas {
+    outline: 1px solid black;
+  }
+</style>
+<div id=container>
+  <p>
+    This sample combines WebCodecs and WebAudio to create a media player that
+    renders synchronized audio and video.
+  </p>
+  <p>
+    Check out the <a href='../video-decode-display/'>Video Decoding and Display
+    </a> demo for a simpler introduction to video decoding and rendering. View
+    <a href='https://youtu.be/U8T5U8sN5d4?t=1572'>this video presentation</a>
+    for an overview of audio rendering stack.
+  </p>
+  <p>
+    This sample requires <a href='https://web.dev/cross-origin-isolation-guide'>
+    cross origin isolation</a> to use
+    <span class='monospace'>SharedArrayBuffer</span>. You may use
+    <span class='monospace'>node server.js</span> to host this sample locally
+    with the appropriate HTTP headers.
+  </p>
+  <div id=controls>
+    <p id=loading>Loading...</p>
+    <button disabled=true>Play</button>
+    <label for=volume>Volume</label>
+    <input id=volume type=range value=0.8 min=0 max=1.0 step=0.01></input>
+  </div>
+  <canvas width=1280 height=720></canvas>
+</div>
+<script type="module">
+import { WebAudioController } from "./web_audio_controller.js";
+
+// Transfer canvas to offscreen. Painting will be performed by worker without
+// blocking the Window main thread.
+window.$ = document.querySelector.bind(document);
+let canvas = $("canvas");
+let offscreenCanvas = canvas.transferControlToOffscreen();
+
+// Instantiate the "media worker" and start loading the files. The worker will
+// house and drive the demuxers and decoders.
+let mediaWorker = new Worker('./media_worker.js');
+mediaWorker.postMessage({command: 'initialize',
+                         audioFile: 'bbb_audio_aac_frag.mp4',
+                         videoFile: 'bbb_video_avc_frag.mp4',
+                         canvas: offscreenCanvas},
+                        {transfer: [offscreenCanvas]});
+
+// Wait for worker initialization. Use metadata to init the WebAudioController.
+let initResolver = null;
+let initDone = new Promise(resolver => (initResolver = resolver));
+let audioController = new WebAudioController();
+mediaWorker.addEventListener('message', (e) => {
+  console.assert(e.data.command == 'initialize-done');
+  audioController.initialize(e.data.sampleRate, e.data.channelCount,
+                      e.data.sharedArrayBuffer);
+  initResolver();
+  initResolver = null;
+});
+await initDone;
+
+// Set up volume slider.
+$('#volume').onchange = (e) => { audioController.setVolume(e.target.value); }
+
+// Enable play now that we're loaded
+let playButton = $('button');
+let loadingElement = $('#loading');
+playButton.disabled = false;
+loadingElement.innerText = 'Ready! Click play.'
+
+playButton.onclick = () => {
+  if (playButton.innerText == "Play") {
+    console.log("playback start");
+
+    // Audio can only start in reaction to a user-gesture.
+    audioController.play().then(() => console.log('playback started'));
+    mediaWorker.postMessage({
+        command: 'play',
+        mediaTimeSecs: audioController.getMediaTimeInSeconds(),
+        mediaTimeCapturedAtHighResTimestamp:
+            performance.now() + performance.timeOrigin
+    });
+
+    sendMediaTimeUpdates(true);
+
+    playButton.innerText = "Pause";
+
+  } else {
+    console.log("playback pause");
+    // Resolves when audio has effectively stopped, this can take some time if
+    // using bluetooth, for example.
+    audioController.pause().then(() => { console.log("playback paused");
+      // Wait to pause worker until context suspended to ensure we continue
+      // filling audio buffer while audio is playing.
+      mediaWorker.postMessage({command: 'pause'});
+    });
+
+    sendMediaTimeUpdates(false);
+
+    playButton.innerText = "Play"
+  }
+}
+
+// Helper function to periodically send the current media time to the media
+// worker. Ideally we would instead compute the media time on the worker thread,
+// but this requires WebAudio interfaces to be exposed on the WorkerGlobalScope.
+// See https://github.com/WebAudio/web-audio-api/issues/2423
+let mediaTimeUpdateInterval = null;
+function sendMediaTimeUpdates(enabled) {
+  if (enabled) {
+    // Local testing shows this interval (1 second) is frequent enough that the
+    // estimated media time between updates drifts by less than 20 msec. Lower
+    // values didn't produce meaningfully lower drift and have the downside of
+    // waking up the main thread more often. Higher values could make av sync
+    // glitches more noticeable when changing the output device.
+    const UPDATE_INTERVAL = 1000;
+    mediaTimeUpdateInterval = setInterval(() => {
+      mediaWorker.postMessage({
+          command: 'update-media-time',
+          mediaTimeSecs: audioController.getMediaTimeInSeconds(),
+          mediaTimeCapturedAtHighResTimestamp:
+              performance.now() + performance.timeOrigin
+      });
+    }, UPDATE_INTERVAL);
+  } else {
+    clearInterval(mediaTimeUpdateInterval);
+    mediaTimeUpdateInterval = null;
+  }
+}
+</script>
+</html>