diff --git a/.github/workflows/build-preview.yml b/.github/workflows/build-preview.yml index d1ffd9423..afa0c84af 100644 --- a/.github/workflows/build-preview.yml +++ b/.github/workflows/build-preview.yml @@ -36,6 +36,10 @@ jobs: echo "MAIN_STATS<> $GITHUB_ENV echo -e "$MAIN_STATS" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV + MAIN_BENCH=$(node scripts/benchmark.js -j) + echo "MAIN_BENCH<> $GITHUB_ENV + echo -e "$MAIN_BENCH" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV - name: Checkout PR Branch 🛎️ uses: actions/checkout@v3 - name: Use Node.js 18.16.1 @@ -61,6 +65,10 @@ jobs: echo "PR_STATS<> $GITHUB_ENV echo -e "$PR_STATS" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV + PR_BENCH=$(node scripts/benchmark.js -j) + echo "PR_BENCH<> $GITHUB_ENV + echo -e "$PR_BENCH" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV - name: Compare Stats id: compare-stats run: | @@ -68,6 +76,9 @@ jobs: echo '${{ env.MAIN_STATS }}' echo '${{ env.PR_STATS }}' npm exec ts-node scripts/stats_compare '${{ env.MAIN_STATS }}' '${{ env.PR_STATS }}' > pr/stats-difference.md + echo '${{ env.MAIN_BENCH }}' + echo '${{ env.PR_BENCH }}' + node scripts/benchmark_compare '${{ env.MAIN_BENCH }}' '${{ env.PR_BENCH }}' >> pr/stats-difference.md - name: Save PR artifacts env: PR_NUMBER: ${{ github.event.pull_request.number }} diff --git a/package.json b/package.json index 192a64e38..c59041f41 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "shieldlib" ], "scripts": { + "benchmark_action": "node scripts/benchmark_cmd.js", "build:shieldlib": "cd shieldlib && node scripts/build.js", "build:code": "exec ts-node scripts/build", "build": "run-s clean-build sprites build:shieldlib build:code taginfo status_map", @@ -30,6 +31,7 @@ "sprites": "node scripts/sprites.js", "start": "run-s clean-build build:shieldlib sprites shields serve", "stats": "node scripts/stats.js", + "stats_action": "node scripts/stats_cmd.js", "style": "node scripts/generate_style.js -o dist/style.json", "status_map": "node scripts/status_map.js", "taginfo": "exec ts-node scripts/taginfo", diff --git a/scripts/benchmark.js b/scripts/benchmark.js index abe4a0e96..91c18a570 100644 --- a/scripts/benchmark.js +++ b/scripts/benchmark.js @@ -1,59 +1,8 @@ -import Benchmark from "benchmark"; -import { expression } from "@maplibre/maplibre-gl-style-spec"; - import { build } from "../src/layer/index.js"; -import { VectorTile } from "@mapbox/vector-tile"; -import Pbf from "pbf"; - -const layers = build(["en"]) - .filter((layer) => layer["source-layer"] === "transportation" && layer.filter) - .map((layer) => expression.createExpression(layer.filter).value.expression); - -const suite = new Benchmark.Suite(); - -async function addTest(name, z, x, y) { - const tile = await ( - await fetch( - `https://d1zqyi8v6vm8p9.cloudfront.net/planet/${z}/${x}/${y}.mvt` - ) - ).arrayBuffer(); - - const transportation = new VectorTile(new Pbf(tile)).layers["transportation"]; - const features = []; +import { calcBenchmarkJSON } from "./benchmark_json.js"; - for (let i = 0; i < transportation.length; i++) { - const feature = transportation.feature(i); - features.push({ - type: feature.type, - properties: feature.properties, - geometry: [], - }); - } - suite.add(`evaluate expressions ${name}`, () => { - let num = 0; - const context = { - properties: () => context.feature.properties, - geometryType: () => context.feature.type, - }; - for (const layer of layers) { - for (const feature of features) { - context.feature = feature; - if (layer.evaluate(context)) { - num++; - } - } - } - }); -} +const style = build(["en"]); -await addTest("nyc z12", 12, 1207, 1539); -await addTest("boston z12", 12, 1239, 1514); -await addTest("kansas z14", 14, 3707, 6302); +let benchmarks = await calcBenchmarkJSON(style); -suite - .on("error", (event) => console.log(event.target.error)) - .on("cycle", (event) => { - const time = 1_000 / event.target.hz; - console.log(`${time.toPrecision(4)}ms ${event.target}`); - }) - .run(); +console.log(benchmarks); diff --git a/scripts/benchmark_cmd.js b/scripts/benchmark_cmd.js new file mode 100644 index 000000000..61e8f15ff --- /dev/null +++ b/scripts/benchmark_cmd.js @@ -0,0 +1,5 @@ +import { calcBenchmarkJSON } from "./benchmark_json.js"; +import { readFileSync } from "fs"; + +const style = JSON.parse(readFileSync(process.argv[2], "utf8")); +process.stdout.write(JSON.stringify(await calcBenchmarkJSON(style))); diff --git a/scripts/benchmark_compare.js b/scripts/benchmark_compare.js new file mode 100644 index 000000000..4c28f98a2 --- /dev/null +++ b/scripts/benchmark_compare.js @@ -0,0 +1,38 @@ +import { pctFormat, timingFormat, durationFormat } from "./compare_func.js"; + +const stats1 = JSON.parse(process.argv[2]); +const stats2 = JSON.parse(process.argv[3]); + +for (const tile in stats2) { + const tileStats1 = stats1[tile]; + const tileStats2 = stats2[tile]; + let tilePerf = ` +## Performance for ${tile} + +| layer | #features | main (/feature) | PR (/feature) | main (/tile) | PR (/tile) | change (/tile) | % change | +|-------|----------:|----------------:|--------------:|-------------:|-----------:|---------------:|---------:| +`; + + for (const layer in tileStats2) { + let perf1 = tileStats1[layer]; + let perf2 = tileStats2[layer]; + let featTime1 = perf1.time / perf1.featureCount; + let featTime2 = perf2.time / perf2.featureCount; + let tileDiff = perf2.time - perf1.time; + tilePerf += `${layer}|${perf2.featureCount}|${( + 1_000 * featTime1 + ).toLocaleString(undefined, durationFormat)}μs|${( + 1_000 * featTime2 + ).toLocaleString(undefined, durationFormat)}μs|${perf1.time.toLocaleString( + undefined, + durationFormat + )}ms|${perf2.time.toLocaleString( + undefined, + durationFormat + )}ms|${tileDiff.toLocaleString(undefined, timingFormat)}ms|${( + tileDiff / perf1.time + ).toLocaleString(undefined, pctFormat)} +`; + } + console.log(tilePerf); +} diff --git a/scripts/benchmark_json.js b/scripts/benchmark_json.js new file mode 100644 index 000000000..3537cd38b --- /dev/null +++ b/scripts/benchmark_json.js @@ -0,0 +1,91 @@ +import Benchmark from "benchmark"; +import { expression } from "@maplibre/maplibre-gl-style-spec"; +import { VectorTile } from "@mapbox/vector-tile"; +import Pbf from "pbf"; +import perfLocations from "./performance.json" assert { type: "json" }; + +function getStyleLayerExpressions(layers, layerName) { + let expressions = layers + .filter((layer) => layer["source-layer"] === layerName && layer.filter) + .map((layer) => expression.createExpression(layer.filter).value.expression); + if (expressions) { + return expressions; + } else { + return []; + } +} + +async function addTest(suite, style, name, z, x, y) { + const tile = await ( + await fetch( + `https://d1zqyi8v6vm8p9.cloudfront.net/planet/${z}/${x}/${y}.mvt` + ) + ).arrayBuffer(); + + const vtile = new VectorTile(new Pbf(tile)); + + for (const layerName in vtile.layers) { + const thisLayer = vtile.layers[layerName]; + const features = []; + const layers = getStyleLayerExpressions(style.layers, layerName); + + for (let i = 0; i < thisLayer.length; i++) { + const feature = thisLayer.feature(i); + features.push({ + type: feature.type, + properties: feature.properties, + geometry: [], + }); + } + suite.add(`${name}#${layerName}#${thisLayer.length}`, () => { + let num = 0; + const context = { + properties: () => context.feature.properties, + geometryType: () => context.feature.type, + globals: { + zoom: z, + }, + }; + for (const layer of layers) { + for (const feature of features) { + context.feature = feature; + if (layer.evaluate(context)) { + num++; + } + } + } + }); + } +} + +export async function calcBenchmarkJSON(style) { + const suite = new Benchmark.Suite(); + + for (let i in perfLocations) { + let test = perfLocations[i]; + await addTest(suite, style, test.name, test.z, test.x, test.y); + } + + const performanceTest = {}; + + suite + .on("error", (event) => console.log(event.target.error)) + .on("cycle", (event) => { + const time = 1_000 / event.target.hz; + const targetParts = event.target.name.split("#"); + const suiteName = targetParts[0]; + const sourceLayer = targetParts[1]; + const featureCount = targetParts[2]; + if (!performanceTest[suiteName]) { + performanceTest[suiteName] = {}; + } + const perfResult = { + featureCount, + time, + }; + performanceTest[suiteName][sourceLayer] = perfResult; + }) + .run(); + + return performanceTest; +} diff --git a/scripts/compare_func.js b/scripts/compare_func.js new file mode 100644 index 000000000..f888c7f3b --- /dev/null +++ b/scripts/compare_func.js @@ -0,0 +1,34 @@ +export const pctFormat = { + style: "percent", + minimumFractionDigits: 1, + maximumFractionDigits: 1, + signDisplay: "exceptZero", +}; + +export const timingFormat = { + style: "decimal", + minimumFractionDigits: 3, + maximumFractionDigits: 3, + signDisplay: "exceptZero", +}; + +export const durationFormat = { + style: "decimal", + minimumFractionDigits: 3, + maximumFractionDigits: 3, + signDisplay: "never", +}; + +export function calculateDifference(object1, object2) { + const difference = {}; + + for (const key in object1) { + if (typeof object1[key] === "object") { + difference[key] = calculateDifference(object1[key], object2[key]); + } else if (typeof object1[key] === "number") { + difference[key] = object2[key] - object1[key]; + } + } + + return difference; +} diff --git a/scripts/performance.json b/scripts/performance.json new file mode 100644 index 000000000..be5b361e8 --- /dev/null +++ b/scripts/performance.json @@ -0,0 +1,14 @@ +[ + { + "name": "Manhattan z14", + "z": 14, + "x": 4825, + "y": 6157 + }, + { + "name": "Fortaleza, Brazil z14", + "z": 14, + "x": 6435, + "y": 8361 + } +] diff --git a/scripts/stats.js b/scripts/stats.js index b75ef7f01..a10dc6ff8 100644 --- a/scripts/stats.js +++ b/scripts/stats.js @@ -1,6 +1,7 @@ import * as Style from "../src/js/style.js"; import config from "../src/config.js"; import { Command, Option } from "commander"; +import { calcStatsJSON } from "./stats_json.js"; const program = new Command(); program @@ -38,43 +39,20 @@ const style = Style.build( ); const layers = style.layers; -const layerCount = layers.length; if (opts.layerCount) { + const layerCount = layers.length; console.log(layerCount); process.exit(); } -const styleSize = JSON.stringify(layers).length; - if (opts.layerSize) { + const styleSize = JSON.stringify(layers).length; console.log(styleSize); process.exit(); } -const layerMap = new Map(); - -const stats = { - layerCount, - styleSize, - layerGroup: {}, -}; - -for (let i = 0; i < layerCount; i++) { - const layer = layers[i]; - layerMap.set(layer.id, layers[i]); - const layerSize = JSON.stringify(layer).length; - const layerGroup = layer["source-layer"] || layer.source || layer.type; - if (stats.layerGroup[layerGroup]) { - stats.layerGroup[layerGroup].size += layerSize; - stats.layerGroup[layerGroup].layerCount++; - } else { - stats.layerGroup[layerGroup] = { - size: layerSize, - layerCount: 1, - }; - } -} +let stats = calcStatsJSON(style); if (opts.allJson) { process.stdout.write(JSON.stringify(stats, null, opts.pretty ? 2 : null)); diff --git a/scripts/stats_cmd.js b/scripts/stats_cmd.js new file mode 100644 index 000000000..511c5d60b --- /dev/null +++ b/scripts/stats_cmd.js @@ -0,0 +1,5 @@ +import { calcStatsJSON } from "./stats_json.js"; +import { readFileSync } from "fs"; + +const style = JSON.parse(readFileSync(process.argv[2], "utf8")); +process.stdout.write(JSON.stringify(calcStatsJSON(style))); diff --git a/scripts/stats_json.js b/scripts/stats_json.js new file mode 100644 index 000000000..3e9bea8c6 --- /dev/null +++ b/scripts/stats_json.js @@ -0,0 +1,29 @@ +export function calcStatsJSON(style) { + const layerMap = new Map(); + const layers = style.layers; + const layerCount = layers.length; + const styleSize = JSON.stringify(style).length; + + const stats = { + layerCount, + styleSize, + layerGroup: {}, + }; + + for (let i = 0; i < layerCount; i++) { + const layer = layers[i]; + layerMap.set(layer.id, layers[i]); + const layerSize = JSON.stringify(layer).length; + const layerGroup = layer["source-layer"] || layer.source || layer.type; + if (stats.layerGroup[layerGroup]) { + stats.layerGroup[layerGroup].size += layerSize; + stats.layerGroup[layerGroup].layerCount++; + } else { + stats.layerGroup[layerGroup] = { + size: layerSize, + layerCount: 1, + }; + } + } + return stats; +} diff --git a/test/test_benchmark_compare.sh b/test/test_benchmark_compare.sh new file mode 100755 index 000000000..72119706b --- /dev/null +++ b/test/test_benchmark_compare.sh @@ -0,0 +1,11 @@ +#!/bin/sh +DATA1=$(cat <<-EOM +{"world z0":{"water":{"featureCount":"2","time":0.054234511861310974},"landcover":{"featureCount":"11","time":0.25328191520102906},"place":{"featureCount":"42","time":1.0943271995293875},"water_name":{"featureCount":"6","time":0.16907226338813058},"boundary":{"featureCount":"33","time":0.8796750853046593}},"kansas z14":{"landcover":{"featureCount":"1","time":0.03050667578182351},"transportation_name":{"featureCount":"5","time":0.2833461815632759},"transportation":{"featureCount":"4","time":0.2337247301196497},"building":{"featureCount":"1","time":0.036974993808978514}}} +EOM +) +DATA2=$(cat <<-EOM +{"world z0":{"water":{"featureCount":"2","time":0.055234511861310974},"landcover":{"featureCount":"11","time":0.25828191520102906},"place":{"featureCount":"42","time":1.0743271995293875},"water_name":{"featureCount":"6","time":0.16907226338813058},"boundary":{"featureCount":"33","time":0.8796750853046593}},"kansas z14":{"landcover":{"featureCount":"1","time":0.03050667578182351},"transportation_name":{"featureCount":"5","time":0.2833461815632759},"transportation":{"featureCount":"4","time":0.2337247301196497},"building":{"featureCount":"1","time":0.036974993808978514}}} +EOM +) + +node scripts/benchmark_compare.js "$DATA1" "$DATA2"