Skip to content

Commit

Permalink
improve test comparison report
Browse files Browse the repository at this point in the history
  • Loading branch information
@liabru
liabru committed Mar 10, 2024
1 parent f74dafd commit 962fba5
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 35 deletions.
32 changes: 21 additions & 11 deletions test/ExampleWorker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ const runExample = options => {
let render;
let extrinsicCapture;

const bodyOverlap = (bodyA, bodyB) => {
const collision = Matter.Collision.collides(bodyA, bodyB);
return collision ? collision.depth : 0;
const pairOverlap = (pair) => {
const collision = Matter.Collision.collides(pair.bodyA, pair.bodyB);
return collision ? Math.max(collision.depth - pair.slop, 0) : -1;
};

for (i = 0; i < options.repeats; i += 1) {
Expand Down Expand Up @@ -66,20 +66,30 @@ const runExample = options => {
timeDeltaAverage = smoothExp(timeDeltaAverage, timeDelta);
}

if (j === 1) {
const pairsList = engine.pairs.list;
const pairsListLength = engine.pairs.list.length;
let overlapTotalUpdate = 0;
let overlapCountUpdate = 0;

for (let p = 0; p < pairsListLength; p += 1) {
const pair = pairsList[p];
const pairsList = engine.pairs.list;
const pairsListLength = engine.pairs.list.length;

if (pair.isActive && !pair.isSensor) {
overlapTotal += bodyOverlap(pair.bodyA, pair.bodyB);
overlapCount += 1;
for (let p = 0; p < pairsListLength; p += 1) {
const pair = pairsList[p];

if (pair.isActive && !pair.isSensor){
const overlap = pairOverlap(pair);

if (overlap >= 0) {
overlapTotalUpdate += overlap;
overlapCountUpdate += 1;
}
}
}

if (overlapCountUpdate > 0) {
overlapTotal += overlapTotalUpdate / overlapCountUpdate;
overlapCount += 1;
}

if (!extrinsicCapture && engine.timing.timestamp >= 1000) {
extrinsicCapture = captureExtrinsics(engine, Matter);
extrinsicCapture.updates = j;
Expand Down
89 changes: 65 additions & 24 deletions test/TestTools.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,22 @@ const equalityThreshold = 1;
const colors = { Red: 31, Green: 32, Yellow: 33, White: 37, BrightWhite: 90, BrightCyan: 36 };

const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildVersion, save, benchmark) => {
const performanceDev = capturePerformanceTotals(capturesDev);
const performanceBuild = capturePerformanceTotals(capturesBuild);
const {
durationChange,
memoryChange,
overlapChange
} = captureBenchmark(capturesDev, capturesBuild);

const perfChange = noiseThreshold(1 - (performanceDev.duration / performanceBuild.duration), 0.01);
const memoryChange = noiseThreshold((performanceDev.memory / performanceBuild.memory) - 1, 0.01);
const overlapChange = (performanceDev.overlap / (performanceBuild.overlap || 1)) - 1;
const filesizeChange = (devSize / buildSize) - 1;

const similaritys = extrinsicSimilarity(capturesDev, capturesBuild);
const similarityAverage = extrinsicSimilarityAverage(similaritys);
const similarityEntries = Object.entries(similaritys);
similarityEntries.sort((a, b) => a[1] - b[1]);

const firstCapture = Object.entries(capturesDev)[0][1];
const updates = firstCapture.extrinsic.updates;

const similaritys = extrinsicSimilarity(capturesDev, capturesBuild);
const similarityAverage = extrinsicSimilarityAverage(similaritys);
const similarityAveragePerUpdate = Math.pow(1, -1 / updates) * Math.pow(similarityAverage, 1 / updates);
const similarityEntries = Object.entries(similaritys);
similarityEntries.sort((a, b) => a[1] - b[1]);

const devIntrinsicsChanged = {};
const buildIntrinsicsChanged = {};
Expand All @@ -51,8 +50,6 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV
})
.sort((a, b) => a.name.localeCompare(b.name));

const formatColor = green => green ? colors.Green : colors.Yellow;

const report = (breakEvery, format) => [
[`Output sample comparison estimates of ${similarityEntries.length} examples`,
`against previous release ${format('matter-js@' + buildVersion, colors.Yellow)}:`
Expand All @@ -70,7 +67,7 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV

...(benchmark ? [
`\n${format('Performance', colors.White)}`,
` ${format(formatPercent(perfChange), formatColor(perfChange >= 0))}%`,
` ${format(formatPercent(durationChange), formatColor(durationChange >= 0))}%`,

` ${format('Memory', colors.White)} `,
` ${format(formatPercent(memoryChange), formatColor(memoryChange <= 0))}%`,
Expand Down Expand Up @@ -102,9 +99,13 @@ const comparisonReport = (capturesDev, capturesBuild, devSize, buildSize, buildV
};

const similarityRatings = similarity => similarity < equalityThreshold ? color('●', colors.Yellow) : '·';

const changeRatings = isChanged => isChanged ? color('◆', colors.White) : '·';

const color = (text, number) => number ? `\x1b[${number}m${text}\x1b[0m` : text;

const formatColor = isGreen => isGreen ? colors.Green : colors.Yellow;

const formatPercent = (val, showSign=true, showFractional=false, padStart=6) => {
let fractionalSign = '';

Expand All @@ -127,10 +128,35 @@ const noiseThreshold = (val, threshold) => {
return sign * Math.max(0, magnitude - threshold) / (1 - threshold);
};

const median = (values, lower, upper) => {
const valuesSorted = values.slice(0).sort();

return mean(valuesSorted.slice(
Math.floor(valuesSorted.length * lower),
Math.floor(valuesSorted.length * upper)
));
};

const mean = (values) => {
const valuesLength = values.length;
let result = 0;

for (let i = 0; i < valuesLength; i += 1) {
result += values[i];
}

return (result / valuesLength) || 0;
};

const smoothExp = (last, current) => {
const delta = current - last;
const sign = delta < 0 ? -1 : 1;
const magnitude = Math.abs(delta);

if (magnitude < 1) {
return last + 0.01 * delta;
}

return last + Math.sqrt(magnitude) * sign;
};

Expand All @@ -143,20 +169,35 @@ const equals = (a, b) => {
return true;
};

const capturePerformanceTotals = (captures) => {
const totals = {
duration: 0,
overlap: 0,
memory: 0
};
const captureBenchmark = (capturesDev, capturesBuild) => {
const overlapChanges = [];

let durationDev = 0;
let durationBuild = 0;
let memoryDev = 0;
let memoryBuild = 0;

for (const name in capturesDev) {
durationDev += capturesDev[name].duration;
durationBuild += capturesBuild[name].duration;

memoryDev += capturesDev[name].memory;
memoryBuild += capturesBuild[name].memory;

for (const [ name ] of Object.entries(captures)) {
totals.duration += captures[name].duration;
totals.overlap += captures[name].overlap;
totals.memory += captures[name].memory;
if (capturesBuild[name].overlap > 0.1 && capturesDev[name].overlap > 0.1){
overlapChanges.push(capturesDev[name].overlap / capturesBuild[name].overlap);
}
};

return totals;
const durationChange = 1 - noiseThreshold(durationDev / durationBuild, 0.02);
const memoryChange = noiseThreshold(memoryDev / memoryBuild, 0.02) - 1;
const overlapChange = noiseThreshold(median(overlapChanges, 0.45, 0.55), 0.001) - 1;

return {
durationChange,
memoryChange,
overlapChange
};
};

const extrinsicSimilarity = (currentCaptures, referenceCaptures, key='extrinsic') => {
Expand Down

0 comments on commit 962fba5

Please sign in to comment.