From 4d9ceb96061f4b81d16cae29d85eda334458b8b6 Mon Sep 17 00:00:00 2001 From: Djuffin Date: Thu, 28 Sep 2023 16:08:34 +0000 Subject: [PATCH] =?UTF-8?q?Deploying=20to=20gh-pages=20from=20@=20w3c/webc?= =?UTF-8?q?odecs@2a0d426206e4d4b11e6b54457fcb873ba4bcb648=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- samples/encode-decode-worker/index.html | 10 ++ samples/encode-decode-worker/js/main.js | 74 +++++++----- .../encode-decode-worker/js/stream_worker.js | 108 +++++++++++++++++- 3 files changed, 162 insertions(+), 30 deletions(-) diff --git a/samples/encode-decode-worker/index.html b/samples/encode-decode-worker/index.html index f6b62b8e..4c6094f7 100644 --- a/samples/encode-decode-worker/index.html +++ b/samples/encode-decode-worker/index.html @@ -75,6 +75,12 @@

WebCodecs in Worker + RVFC

value=3000> +
+ + +
+

Codec:

@@ -155,6 +161,10 @@

WebCodecs in Worker + RVFC

+
+ +
+
diff --git a/samples/encode-decode-worker/js/main.js b/samples/encode-decode-worker/js/main.js index 8bfa3b0f..4b063efe 100644 --- a/samples/encode-decode-worker/js/main.js +++ b/samples/encode-decode-worker/js/main.js @@ -20,6 +20,7 @@ let display_metrics = { }; const rate = document.querySelector('#rate'); +const framer = document.querySelector('#framer'); const connectButton = document.querySelector('#connect'); const stopButton = document.querySelector('#stop'); const codecButtons = document.querySelector('#codecButtons'); @@ -28,9 +29,13 @@ const modeButtons = document.querySelector('#modeButtons'); const decHwButtons = document.querySelector('#decHwButtons'); const encHwButtons = document.querySelector('#encHwButtons'); const chart2_div = document.getElementById('chart2_div'); +const chart3_div = document.getElementById('chart3_div'); +const chart4_div = document.getElementById('chart4_div'); const videoSelect = document.querySelector('select#videoSource'); const selectors = [videoSelect]; chart2_div.style.display = "none"; +chart3_div.style.display = "none"; +chart4_div.style.display = "none"; connectButton.disabled = false; stopButton.disabled = true; @@ -189,6 +194,8 @@ function stop() { stopButton.disabled = true; connectButton.disabled = true; chart2_div.style.display = "initial"; + chart3_div.style.display = "initial"; + chart4_div.style.display = "initial"; streamWorker.postMessage({ type: "stop" }); try { inputStream.cancel(); @@ -231,35 +238,42 @@ document.addEventListener('DOMContentLoaded', async function(event) { addToEventLog('Worker created.'); streamWorker.addEventListener('message', function(e) { + let labels = ''; if (e.data.severity != 'chart'){ addToEventLog('Worker msg: ' + e.data.text, e.data.severity); } else { - // draw the glass-glass latency chart - metrics_report(); - const e2eX = e2e.all.map(item => item[0]); - const e2eY = e2e.all.map(item => item[1]); - const labels = e2e.all.map((item, index) => { - return Object.keys(display_metrics.all[index]).map(key => { - return `${key}: ${display_metrics.all[index][key]}`; - }).join('
'); + if (e.data.text == '') { + metrics_report(); // sets e2e.all and display_metrics + e.data.text = JSON.stringify(e2e.all); + labels = e2e.all.map((item, index) => { + return Object.keys(display_metrics.all[index]).map(key => { + return `${key}: ${display_metrics.all[index][key]}`; + }).join('
'); + }); + } + const parsed = JSON.parse(e.data.text); + const x = parsed.map(item => item[0]); + const y = parsed.map(item => item[1]); + // TODO: more options needed from https://plotly.com/javascript/line-and-scatter + Plotly.newPlot(e.data.div, [{ + x, + y, + text: labels, + mode: 'markers', + type: 'scatter', + }], { + xaxis: { + title: e.data.x, + autorange: true, + range: [0, Math.max.apply(null, x) + 100 /* + a bit, 10%-ish to make it look good */], + }, + yaxis: { + title: e.data.y, + autorange: true, + //range: [0, Math.max.apply(null, y) /* + a bit, 10%-ish to make it look good */], + }, + title: e.data.label, }); - Plotly.newPlot(chart2_div, [{ - x: e2eX, - y: e2eY, - text: labels, - mode: 'markers', - type: 'scatter', - }], { - xaxis: { - title: 'Frame number', - autorange: true, - }, - yaxis: { - title: 'Glass-Glass-Latency (ms)', - autorange: true, - }, - title: 'Glass-Glass Latency (ms) versus Frame Number', - }); } }, false); @@ -279,6 +293,7 @@ document.addEventListener('DOMContentLoaded', async function(event) { resButtons.style.display = "none"; modeButtons.style.display = "none"; rateInput.style.display = "none"; + frameInput.style.display = "none"; keyInput.style.display = "none"; startMedia(); } @@ -290,6 +305,9 @@ document.addEventListener('DOMContentLoaded', async function(event) { // Collect the bitrate const rate = document.getElementById('rate').value; + // Collect the framerate + const framer = document.getElementById('framer').value; + // Collect the keyframe gap const keygap = document.getElementById('keygap').value; @@ -346,6 +364,7 @@ document.addEventListener('DOMContentLoaded', async function(event) { let ssrcArr = new Uint32Array(1); window.crypto.getRandomValues(ssrcArr); const ssrc = ssrcArr[0]; + const framerat = Math.min(framer, ts.frameRate/vConfig.framerateScale) ; const config = { alpha: "discard", @@ -357,7 +376,7 @@ document.addEventListener('DOMContentLoaded', async function(event) { hardwareAcceleration: encHw, decHwAcceleration: decHw, bitrate: rate, - framerate: ts.frameRate/vConfig.framerateScale, + framerate: framerat, keyInterval: vConfig.keyInterval, ssrc: ssrc }; @@ -368,7 +387,8 @@ document.addEventListener('DOMContentLoaded', async function(event) { switch(preferredCodec){ case "H264": - config.codec = "avc1.42002A"; // baseline profile, level 4.2 + config.codec = "avc1.42002A"; // baseline profile, level 4.2 + /* config.codec = "avc1.640028"; */ config.avc = { format: "annexb" }; config.pt = 1; break; diff --git a/samples/encode-decode-worker/js/stream_worker.js b/samples/encode-decode-worker/js/stream_worker.js index d5238b44..f57fc53c 100644 --- a/samples/encode-decode-worker/js/stream_worker.js +++ b/samples/encode-decode-worker/js/stream_worker.js @@ -2,6 +2,28 @@ let encoder, decoder, pl, started = false, stopped = false; +let enc_aggregate = { + all: [], +}; + +let enc_time = { + all: [], + min: Number.MAX_VALUE, + max: 0, + sum: 0 +}; + +let dec_aggregate = { + all: [], +}; + +let dec_time = { + all: [], + min: Number.MAX_VALUE, + max: 0, + sum: 0 +}; + let encqueue_aggregate = { all: [], min: Number.MAX_VALUE, @@ -18,6 +40,10 @@ let decqueue_aggregate = { sum: 0, }; +function enc_update(data) { + enc_aggregate.all.push(data); +} + function encqueue_update(duration) { encqueue_aggregate.all.push(duration); encqueue_aggregate.min = Math.min(encqueue_aggregate.min, duration); @@ -25,6 +51,33 @@ function encqueue_update(duration) { encqueue_aggregate.sum += duration; } +function enc_report() { + enc_aggregate.all.sort((a, b) => { + return (100000 * (a.timestamp - b.timestamp) + a.output - b.output); + }); + const len = enc_aggregate.all.length; + if (len < 2) return; + for (let i = 1; i < len ; i++ ) { + if ((enc_aggregate.all[i].output == 1) && (enc_aggregate.all[i-1].output == 0) && (enc_aggregate.all[i].timestamp == enc_aggregate.all[i-1].timestamp)) { + const timestamp = enc_aggregate.all[i].timestamp; + const enc_delay = enc_aggregate.all[i].time - enc_aggregate.all[i-1].time; + const data = [timestamp, enc_delay]; + enc_time.all.push(data); + enc_time.min = Math.min(enc_time.min, enc_delay); + enc_time.max = Math.max(enc_time.max, enc_delay); + enc_time.sum += enc_delay; + } + } + const avg = enc_time.sum / enc_time.all.length; + //self.postMessage({text: 'Encode Time Data dump: ' + JSON.stringify(enc_time.all)}); + return { + count: enc_time.all.length, + min: enc_time.min, + avg: avg, + max: enc_time.max + }; +} + function encqueue_report() { encqueue_aggregate.all.sort(); const len = encqueue_aggregate.all.length; @@ -47,6 +100,37 @@ function encqueue_report() { }; } +function dec_update(data) { + dec_aggregate.all.push(data); +} + +function dec_report() { + dec_aggregate.all.sort((a, b) => { + return (100000 * (a.timestamp - b.timestamp) + a.output - b.output); + }); + const len = dec_aggregate.all.length; + if (len < 2) return; + for (let i = 1; i < len ; i++ ) { + if ((dec_aggregate.all[i].output == 1) && (dec_aggregate.all[i-1].output == 0) && (dec_aggregate.all[i].timestamp == dec_aggregate.all[i-1].timestamp)) { + const timestamp = dec_aggregate.all[i].timestamp; + const dec_delay = dec_aggregate.all[i].time - dec_aggregate.all[i-1].time; + const data = [timestamp, dec_delay]; + dec_time.all.push(data); + dec_time.min = Math.min(dec_time.min, dec_delay); + dec_time.max = Math.max(dec_time.max, dec_delay); + dec_time.sum += dec_delay; + } + } + const avg = dec_time.sum / dec_time.all.length; + //self.postMessage({text: 'Decode Time Data dump: ' + JSON.stringify(dec_time.all)}); + return { + count: dec_time.all.length, + min: dec_time.min, + avg: avg, + max: dec_time.max + }; +} + function decqueue_update(duration) { decqueue_aggregate.all.push(duration); decqueue_aggregate.min = Math.min(decqueue_aggregate.min, duration); @@ -114,9 +198,13 @@ class pipeline { return new TransformStream({ start(controller) { this.decoder = decoder = new VideoDecoder({ - output: frame => controller.enqueue(frame), + output: (frame) => { + const after = performance.now(); + dec_update({output: 1, timestamp: frame.timestamp, time: after}); + controller.enqueue(frame); + }, error: (e) => { - self.postMessage({severity: 'fatal', text: `Init Decoder error: ${e.message}`}); + self.postMessage({severity: 'fatal', text: `Decoder error: ${e.message}`}); } }); }, @@ -139,6 +227,8 @@ class pipeline { try { const queue = this.decoder.decodeQueueSize; decqueue_update(queue); + const before = performance.now(); + dec_update({output: 0, timestamp: chunk.timestamp, time: before}); this.decoder.decode(chunk); } catch (e) { self.postMessage({severity: 'fatal', text: 'Derror size: ' + chunk.byteLength + ' seq: ' + chunk.seqNo + ' kf: ' + chunk.keyframeIndex + ' delta: ' + chunk.deltaframeIndex + ' dur: ' + chunk.duration + ' ts: ' + chunk.timestamp + ' ssrc: ' + chunk.ssrc + ' pt: ' + chunk.pt + ' tid: ' + chunk.temporalLayerId + ' type: ' + chunk.type}); @@ -175,6 +265,10 @@ class pipeline { config: decoderConfig }; controller.enqueue(configChunk); + } + if (chunk.type != 'config'){ + const after = performance.now(); + enc_update({output: 1, timestamp: chunk.timestamp, time: after}); } chunk.temporalLayerId = 0; if (cfg.svc) { @@ -218,6 +312,8 @@ class pipeline { if (this.encoder.state != "closed") { const queue = this.encoder.encodeQueueSize; encqueue_update(queue); + const before = performance.now(); + enc_update({output: 0, timestamp: frame.timestamp, time: before}); this.encoder.encode(frame, { keyFrame: insert_keyframe }); } } catch(e) { @@ -236,10 +332,16 @@ class pipeline { this.stopped = true; const len = encqueue_aggregate.all.length; if (len > 1) { + const enc_stats = enc_report(); const encqueue_stats = encqueue_report(); + const dec_stats = dec_report(); const decqueue_stats = decqueue_report(); - self.postMessage({severity: 'chart'}); + self.postMessage({severity: 'chart', x: 'Frame Number', y: 'Glass-Glass Latency', label: 'Glass-Glass Latency (ms) by Frame Number', div: 'chart2_div', text: ''}); + self.postMessage({severity: 'chart', x: 'Timestamp', y: 'Encoding Time', label: 'Encoding Time (ms) by Timestamp', div: 'chart3_div', text: JSON.stringify(enc_time.all)}); + self.postMessage({severity: 'chart', x: 'Timestamp', y: 'Decoding Time', label: 'Decoding Time (ms) by Timestamp', div: 'chart4_div', text: JSON.stringify(dec_time.all)}); + self.postMessage({text: 'Encoder Time report: ' + JSON.stringify(enc_stats)}); self.postMessage({text: 'Encoder Queue report: ' + JSON.stringify(encqueue_stats)}); + self.postMessage({text: 'Decoder Time report: ' + JSON.stringify(dec_stats)}); self.postMessage({text: 'Decoder Queue report: ' + JSON.stringify(decqueue_stats)}); } self.postMessage({text: 'stop(): frame, encoder and decoder closed'});