Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add encode/decode time graphs to encode-decode sample #718

Merged
merged 1 commit into from
Sep 28, 2023
Merged

Add encode/decode time graphs to encode-decode sample #718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions samples/encode-decode-worker/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ <h2>WebCodecs in Worker + RVFC</h2>
value=3000>
</div>

<div id="frameInput">
<label for="framer">framerate: </label>
<input type="text" name="framer" id="framer" minlength=2 maxlength=3 size=3
value=30>
</div>

<div id="codecButtons">
<p>Codec:</p>
<input type="radio" id="H264" name="codec" value="H264" onchange="getCodecValue(this)">
Expand Down Expand Up @@ -155,6 +161,10 @@ <h2>WebCodecs in Worker + RVFC</h2>

<div id="chart2_div" style="width: 900px; height: 500px;"></div>

<div id="chart3_div" style="width: 900px; height: 500px;"></div>

<div id="chart4_div" style="width: 900px; height: 500px;"></div>

<div class="select">
<label for="videoSource">Video source: </label><select id="videoSource"></select>
</div>
Expand Down
74 changes: 47 additions & 27 deletions samples/encode-decode-worker/js/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ let display_metrics = {
};

const rate = document.querySelector('#rate');
const framer = document.querySelector('#framer');
const connectButton = document.querySelector('#connect');
const stopButton = document.querySelector('#stop');
const codecButtons = document.querySelector('#codecButtons');
Expand All @@ -28,9 +29,13 @@ const modeButtons = document.querySelector('#modeButtons');
const decHwButtons = document.querySelector('#decHwButtons');
const encHwButtons = document.querySelector('#encHwButtons');
const chart2_div = document.getElementById('chart2_div');
const chart3_div = document.getElementById('chart3_div');
const chart4_div = document.getElementById('chart4_div');
const videoSelect = document.querySelector('select#videoSource');
const selectors = [videoSelect];
chart2_div.style.display = "none";
chart3_div.style.display = "none";
chart4_div.style.display = "none";
connectButton.disabled = false;
stopButton.disabled = true;

Expand Down Expand Up @@ -189,6 +194,8 @@ function stop() {
stopButton.disabled = true;
connectButton.disabled = true;
chart2_div.style.display = "initial";
chart3_div.style.display = "initial";
chart4_div.style.display = "initial";
streamWorker.postMessage({ type: "stop" });
try {
inputStream.cancel();
Expand Down Expand Up @@ -231,35 +238,42 @@ document.addEventListener('DOMContentLoaded', async function(event) {
addToEventLog('Worker created.');

streamWorker.addEventListener('message', function(e) {
let labels = '';
if (e.data.severity != 'chart'){
addToEventLog('Worker msg: ' + e.data.text, e.data.severity);
} else {
// draw the glass-glass latency chart
metrics_report();
const e2eX = e2e.all.map(item => item[0]);
const e2eY = e2e.all.map(item => item[1]);
const labels = e2e.all.map((item, index) => {
return Object.keys(display_metrics.all[index]).map(key => {
return `${key}: ${display_metrics.all[index][key]}`;
}).join('<br>');
if (e.data.text == '') {
metrics_report(); // sets e2e.all and display_metrics
e.data.text = JSON.stringify(e2e.all);
labels = e2e.all.map((item, index) => {
return Object.keys(display_metrics.all[index]).map(key => {
return `${key}: ${display_metrics.all[index][key]}`;
}).join('<br>');
});
}
const parsed = JSON.parse(e.data.text);
const x = parsed.map(item => item[0]);
const y = parsed.map(item => item[1]);
// TODO: more options needed from https://plotly.com/javascript/line-and-scatter
Plotly.newPlot(e.data.div, [{
x,
y,
text: labels,
mode: 'markers',
type: 'scatter',
}], {
xaxis: {
title: e.data.x,
autorange: true,
range: [0, Math.max.apply(null, x) + 100 /* + a bit, 10%-ish to make it look good */],
},
yaxis: {
title: e.data.y,
autorange: true,
//range: [0, Math.max.apply(null, y) /* + a bit, 10%-ish to make it look good */],
},
title: e.data.label,
});
Plotly.newPlot(chart2_div, [{
x: e2eX,
y: e2eY,
text: labels,
mode: 'markers',
type: 'scatter',
}], {
xaxis: {
title: 'Frame number',
autorange: true,
},
yaxis: {
title: 'Glass-Glass-Latency (ms)',
autorange: true,
},
title: 'Glass-Glass Latency (ms) versus Frame Number',
});
}
}, false);

Expand All @@ -279,6 +293,7 @@ document.addEventListener('DOMContentLoaded', async function(event) {
resButtons.style.display = "none";
modeButtons.style.display = "none";
rateInput.style.display = "none";
frameInput.style.display = "none";
keyInput.style.display = "none";
startMedia();
}
Expand All @@ -290,6 +305,9 @@ document.addEventListener('DOMContentLoaded', async function(event) {
// Collect the bitrate
const rate = document.getElementById('rate').value;

// Collect the framerate
const framer = document.getElementById('framer').value;

// Collect the keyframe gap
const keygap = document.getElementById('keygap').value;

Expand Down Expand Up @@ -346,6 +364,7 @@ document.addEventListener('DOMContentLoaded', async function(event) {
let ssrcArr = new Uint32Array(1);
window.crypto.getRandomValues(ssrcArr);
const ssrc = ssrcArr[0];
const framerat = Math.min(framer, ts.frameRate/vConfig.framerateScale) ;

const config = {
alpha: "discard",
Expand All @@ -357,7 +376,7 @@ document.addEventListener('DOMContentLoaded', async function(event) {
hardwareAcceleration: encHw,
decHwAcceleration: decHw,
bitrate: rate,
framerate: ts.frameRate/vConfig.framerateScale,
framerate: framerat,
keyInterval: vConfig.keyInterval,
ssrc: ssrc
};
Expand All @@ -368,7 +387,8 @@ document.addEventListener('DOMContentLoaded', async function(event) {

switch(preferredCodec){
case "H264":
config.codec = "avc1.42002A"; // baseline profile, level 4.2
config.codec = "avc1.42002A"; // baseline profile, level 4.2
/* config.codec = "avc1.640028"; */
config.avc = { format: "annexb" };
config.pt = 1;
break;
Expand Down
108 changes: 105 additions & 3 deletions samples/encode-decode-worker/js/stream_worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,28 @@

let encoder, decoder, pl, started = false, stopped = false;

let enc_aggregate = {
all: [],
};

let enc_time = {
all: [],
min: Number.MAX_VALUE,
max: 0,
sum: 0
};

let dec_aggregate = {
all: [],
};

let dec_time = {
all: [],
min: Number.MAX_VALUE,
max: 0,
sum: 0
};

let encqueue_aggregate = {
all: [],
min: Number.MAX_VALUE,
Expand All @@ -18,13 +40,44 @@ let decqueue_aggregate = {
sum: 0,
};

function enc_update(data) {
enc_aggregate.all.push(data);
}

function encqueue_update(duration) {
encqueue_aggregate.all.push(duration);
encqueue_aggregate.min = Math.min(encqueue_aggregate.min, duration);
encqueue_aggregate.max = Math.max(encqueue_aggregate.max, duration);
encqueue_aggregate.sum += duration;
}

function enc_report() {
enc_aggregate.all.sort((a, b) => {
return (100000 * (a.timestamp - b.timestamp) + a.output - b.output);
});
const len = enc_aggregate.all.length;
if (len < 2) return;
for (let i = 1; i < len ; i++ ) {
if ((enc_aggregate.all[i].output == 1) && (enc_aggregate.all[i-1].output == 0) && (enc_aggregate.all[i].timestamp == enc_aggregate.all[i-1].timestamp)) {
const timestamp = enc_aggregate.all[i].timestamp;
const enc_delay = enc_aggregate.all[i].time - enc_aggregate.all[i-1].time;
const data = [timestamp, enc_delay];
enc_time.all.push(data);
enc_time.min = Math.min(enc_time.min, enc_delay);
enc_time.max = Math.max(enc_time.max, enc_delay);
enc_time.sum += enc_delay;
}
}
const avg = enc_time.sum / enc_time.all.length;
//self.postMessage({text: 'Encode Time Data dump: ' + JSON.stringify(enc_time.all)});
return {
count: enc_time.all.length,
min: enc_time.min,
avg: avg,
max: enc_time.max
};
}

function encqueue_report() {
encqueue_aggregate.all.sort();
const len = encqueue_aggregate.all.length;
Expand All @@ -47,6 +100,37 @@ function encqueue_report() {
};
}

function dec_update(data) {
dec_aggregate.all.push(data);
}

function dec_report() {
dec_aggregate.all.sort((a, b) => {
return (100000 * (a.timestamp - b.timestamp) + a.output - b.output);
});
const len = dec_aggregate.all.length;
if (len < 2) return;
for (let i = 1; i < len ; i++ ) {
if ((dec_aggregate.all[i].output == 1) && (dec_aggregate.all[i-1].output == 0) && (dec_aggregate.all[i].timestamp == dec_aggregate.all[i-1].timestamp)) {
const timestamp = dec_aggregate.all[i].timestamp;
const dec_delay = dec_aggregate.all[i].time - dec_aggregate.all[i-1].time;
const data = [timestamp, dec_delay];
dec_time.all.push(data);
dec_time.min = Math.min(dec_time.min, dec_delay);
dec_time.max = Math.max(dec_time.max, dec_delay);
dec_time.sum += dec_delay;
}
}
const avg = dec_time.sum / dec_time.all.length;
//self.postMessage({text: 'Decode Time Data dump: ' + JSON.stringify(dec_time.all)});
return {
count: dec_time.all.length,
min: dec_time.min,
avg: avg,
max: dec_time.max
};
}

function decqueue_update(duration) {
decqueue_aggregate.all.push(duration);
decqueue_aggregate.min = Math.min(decqueue_aggregate.min, duration);
Expand Down Expand Up @@ -114,9 +198,13 @@ class pipeline {
return new TransformStream({
start(controller) {
this.decoder = decoder = new VideoDecoder({
output: frame => controller.enqueue(frame),
output: (frame) => {
const after = performance.now();
dec_update({output: 1, timestamp: frame.timestamp, time: after});
controller.enqueue(frame);
},
error: (e) => {
self.postMessage({severity: 'fatal', text: `Init Decoder error: ${e.message}`});
self.postMessage({severity: 'fatal', text: `Decoder error: ${e.message}`});
}
});
},
Expand All @@ -139,6 +227,8 @@ class pipeline {
try {
const queue = this.decoder.decodeQueueSize;
decqueue_update(queue);
const before = performance.now();
dec_update({output: 0, timestamp: chunk.timestamp, time: before});
this.decoder.decode(chunk);
} catch (e) {
self.postMessage({severity: 'fatal', text: 'Derror size: ' + chunk.byteLength + ' seq: ' + chunk.seqNo + ' kf: ' + chunk.keyframeIndex + ' delta: ' + chunk.deltaframeIndex + ' dur: ' + chunk.duration + ' ts: ' + chunk.timestamp + ' ssrc: ' + chunk.ssrc + ' pt: ' + chunk.pt + ' tid: ' + chunk.temporalLayerId + ' type: ' + chunk.type});
Expand Down Expand Up @@ -175,6 +265,10 @@ class pipeline {
config: decoderConfig
};
controller.enqueue(configChunk);
}
if (chunk.type != 'config'){
const after = performance.now();
enc_update({output: 1, timestamp: chunk.timestamp, time: after});
}
chunk.temporalLayerId = 0;
if (cfg.svc) {
Expand Down Expand Up @@ -218,6 +312,8 @@ class pipeline {
if (this.encoder.state != "closed") {
const queue = this.encoder.encodeQueueSize;
encqueue_update(queue);
const before = performance.now();
enc_update({output: 0, timestamp: frame.timestamp, time: before});
this.encoder.encode(frame, { keyFrame: insert_keyframe });
}
} catch(e) {
Expand All @@ -236,10 +332,16 @@ class pipeline {
this.stopped = true;
const len = encqueue_aggregate.all.length;
if (len > 1) {
const enc_stats = enc_report();
const encqueue_stats = encqueue_report();
const dec_stats = dec_report();
const decqueue_stats = decqueue_report();
self.postMessage({severity: 'chart'});
self.postMessage({severity: 'chart', x: 'Frame Number', y: 'Glass-Glass Latency', label: 'Glass-Glass Latency (ms) by Frame Number', div: 'chart2_div', text: ''});
self.postMessage({severity: 'chart', x: 'Timestamp', y: 'Encoding Time', label: 'Encoding Time (ms) by Timestamp', div: 'chart3_div', text: JSON.stringify(enc_time.all)});
self.postMessage({severity: 'chart', x: 'Timestamp', y: 'Decoding Time', label: 'Decoding Time (ms) by Timestamp', div: 'chart4_div', text: JSON.stringify(dec_time.all)});
self.postMessage({text: 'Encoder Time report: ' + JSON.stringify(enc_stats)});
self.postMessage({text: 'Encoder Queue report: ' + JSON.stringify(encqueue_stats)});
self.postMessage({text: 'Decoder Time report: ' + JSON.stringify(dec_stats)});
self.postMessage({text: 'Decoder Queue report: ' + JSON.stringify(decqueue_stats)});
}
self.postMessage({text: 'stop(): frame, encoder and decoder closed'});
Expand Down
Loading