Performance benchmarks

.github/workflows/build-preview.yml

@@ -36,6 +36,10 @@ jobs:
           echo "MAIN_STATS<<EOF" >> $GITHUB_ENV
           echo -e "$MAIN_STATS" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
+          MAIN_BENCH=$(node scripts/benchmark.js -j)
+          echo "MAIN_BENCH<<EOF" >> $GITHUB_ENV
+          echo -e "$MAIN_BENCH" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
       - name: Checkout PR Branch 🛎️
         uses: actions/checkout@v3
       - name: Use Node.js 18.16.1
@@ -61,13 +65,20 @@ jobs:
           echo "PR_STATS<<EOF" >> $GITHUB_ENV
           echo -e "$PR_STATS" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
+          PR_BENCH=$(node scripts/benchmark.js -j)
+          echo "PR_BENCH<<EOF" >> $GITHUB_ENV
+          echo -e "$PR_BENCH" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
       - name: Compare Stats
         id: compare-stats
         run: |
           mkdir -p pr
           echo '${{ env.MAIN_STATS }}'
           echo '${{ env.PR_STATS }}'
           npm exec ts-node scripts/stats_compare '${{ env.MAIN_STATS }}' '${{ env.PR_STATS }}' > pr/stats-difference.md
+          echo '${{ env.MAIN_BENCH }}'
+          echo '${{ env.PR_BENCH }}'
+          node scripts/benchmark_compare '${{ env.MAIN_BENCH }}' '${{ env.PR_BENCH }}' >> pr/stats-difference.md          
       - name: Save PR artifacts
         env:
           PR_NUMBER: ${{ github.event.pull_request.number }}

package.json

@@ -11,6 +11,7 @@
     "shieldlib"
   ],
   "scripts": {
+    "benchmark_action": "node scripts/benchmark_cmd.js",
     "build:shieldlib": "cd shieldlib && node scripts/build.js",
     "build:code": "exec ts-node scripts/build",
     "build": "run-s clean-build sprites build:shieldlib build:code taginfo status_map",
@@ -30,6 +31,7 @@
     "sprites": "node scripts/sprites.js",
     "start": "run-s clean-build build:shieldlib sprites shields serve",
     "stats": "node scripts/stats.js",
+    "stats_action": "node scripts/stats_cmd.js",
     "style": "node scripts/generate_style.js -o dist/style.json",
     "status_map": "node scripts/status_map.js",
     "taginfo": "exec ts-node scripts/taginfo",

scripts/benchmark.js

@@ -1,59 +1,8 @@
-import Benchmark from "benchmark";
-import { expression } from "@maplibre/maplibre-gl-style-spec";
-
 import { build } from "../src/layer/index.js";
-import { VectorTile } from "@mapbox/vector-tile";
-import Pbf from "pbf";
-
-const layers = build(["en"])
-  .filter((layer) => layer["source-layer"] === "transportation" && layer.filter)
-  .map((layer) => expression.createExpression(layer.filter).value.expression);
-
-const suite = new Benchmark.Suite();
-
-async function addTest(name, z, x, y) {
-  const tile = await (
-    await fetch(
-      `https://d1zqyi8v6vm8p9.cloudfront.net/planet/${z}/${x}/${y}.mvt`
-    )
-  ).arrayBuffer();
-
-  const transportation = new VectorTile(new Pbf(tile)).layers["transportation"];
-  const features = [];
+import { calcBenchmarkJSON } from "./benchmark_json.js";
 
-  for (let i = 0; i < transportation.length; i++) {
-    const feature = transportation.feature(i);
-    features.push({
-      type: feature.type,
-      properties: feature.properties,
-      geometry: [],
-    });
-  }
-  suite.add(`evaluate expressions ${name}`, () => {
-    let num = 0;
-    const context = {
-      properties: () => context.feature.properties,
-      geometryType: () => context.feature.type,
-    };
-    for (const layer of layers) {
-      for (const feature of features) {
-        context.feature = feature;
-        if (layer.evaluate(context)) {
-          num++;
-        }
-      }
-    }
-  });
-}
+const style = build(["en"]);
 
-await addTest("nyc z12", 12, 1207, 1539);
-await addTest("boston z12", 12, 1239, 1514);
-await addTest("kansas z14", 14, 3707, 6302);
+let benchmarks = await calcBenchmarkJSON(style);
 
-suite
-  .on("error", (event) => console.log(event.target.error))
-  .on("cycle", (event) => {
-    const time = 1_000 / event.target.hz;
-    console.log(`${time.toPrecision(4)}ms ${event.target}`);
-  })
-  .run();
+console.log(benchmarks);

scripts/benchmark_cmd.js

@@ -0,0 +1,5 @@
+import { calcBenchmarkJSON } from "./benchmark_json.js";
+import { readFileSync } from "fs";
+
+const style = JSON.parse(readFileSync(process.argv[2], "utf8"));
+process.stdout.write(JSON.stringify(await calcBenchmarkJSON(style)));

scripts/benchmark_compare.js

@@ -0,0 +1,38 @@
+import { pctFormat, timingFormat, durationFormat } from "./compare_func.js";
+
+const stats1 = JSON.parse(process.argv[2]);
+const stats2 = JSON.parse(process.argv[3]);
+
+for (const tile in stats2) {
+  const tileStats1 = stats1[tile];
+  const tileStats2 = stats2[tile];
+  let tilePerf = `
+## Performance for ${tile}
+
+| layer | #features | main (/feature) | PR (/feature) | main (/tile) | PR (/tile) | change (/tile) | % change |
+|-------|----------:|----------------:|--------------:|-------------:|-----------:|---------------:|---------:|
+`;
+
+  for (const layer in tileStats2) {
+    let perf1 = tileStats1[layer];
+    let perf2 = tileStats2[layer];
+    let featTime1 = perf1.time / perf1.featureCount;
+    let featTime2 = perf2.time / perf2.featureCount;
+    let tileDiff = perf2.time - perf1.time;
+    tilePerf += `${layer}|${perf2.featureCount}|${(
+      1_000 * featTime1
+    ).toLocaleString(undefined, durationFormat)}μs|${(
+      1_000 * featTime2
+    ).toLocaleString(undefined, durationFormat)}μs|${perf1.time.toLocaleString(
+      undefined,
+      durationFormat
+    )}ms|${perf2.time.toLocaleString(
+      undefined,
+      durationFormat
+    )}ms|${tileDiff.toLocaleString(undefined, timingFormat)}ms|${(
+      tileDiff / perf1.time
+    ).toLocaleString(undefined, pctFormat)}
+`;
+  }
+  console.log(tilePerf);
+}

scripts/benchmark_json.js

@@ -0,0 +1,91 @@
+import Benchmark from "benchmark";
+import { expression } from "@maplibre/maplibre-gl-style-spec";
+import { VectorTile } from "@mapbox/vector-tile";
+import Pbf from "pbf";
+import perfLocations from "./performance.json" assert { type: "json" };
+
+function getStyleLayerExpressions(layers, layerName) {
+  let expressions = layers
+    .filter((layer) => layer["source-layer"] === layerName && layer.filter)
+    .map((layer) => expression.createExpression(layer.filter).value.expression);
+  if (expressions) {
+    return expressions;
+  } else {
+    return [];
+  }
+}
+
+async function addTest(suite, style, name, z, x, y) {
+  const tile = await (
+    await fetch(
+      `https://d1zqyi8v6vm8p9.cloudfront.net/planet/${z}/${x}/${y}.mvt`
+    )
+  ).arrayBuffer();
+
+  const vtile = new VectorTile(new Pbf(tile));
+
+  for (const layerName in vtile.layers) {
+    const thisLayer = vtile.layers[layerName];
+    const features = [];
+    const layers = getStyleLayerExpressions(style.layers, layerName);
+
+    for (let i = 0; i < thisLayer.length; i++) {
+      const feature = thisLayer.feature(i);
+      features.push({
+        type: feature.type,
+        properties: feature.properties,
+        geometry: [],
+      });
+    }
+    suite.add(`${name}#${layerName}#${thisLayer.length}`, () => {
+      let num = 0;
+      const context = {
+        properties: () => context.feature.properties,
+        geometryType: () => context.feature.type,
+        globals: {
+          zoom: z,
+        },
+      };
+      for (const layer of layers) {
+        for (const feature of features) {
+          context.feature = feature;
+          if (layer.evaluate(context)) {
+            num++;
+          }
+        }
+      }
+    });
+  }
+}
+
+export async function calcBenchmarkJSON(style) {
+  const suite = new Benchmark.Suite();
+
+  for (let i in perfLocations) {
+    let test = perfLocations[i];
+    await addTest(suite, style, test.name, test.z, test.x, test.y);
+  }
+
+  const performanceTest = {};
+
+  suite
+    .on("error", (event) => console.log(event.target.error))
+    .on("cycle", (event) => {
+      const time = 1_000 / event.target.hz;
+      const targetParts = event.target.name.split("#");
+      const suiteName = targetParts[0];
+      const sourceLayer = targetParts[1];
+      const featureCount = targetParts[2];
+      if (!performanceTest[suiteName]) {
+        performanceTest[suiteName] = {};
+      }
+      const perfResult = {
+        featureCount,
+        time,
+      };
+      performanceTest[suiteName][sourceLayer] = perfResult;
+    })
+    .run();
+
+  return performanceTest;
+}

scripts/compare_func.js

@@ -0,0 +1,34 @@
+export const pctFormat = {
+  style: "percent",
+  minimumFractionDigits: 1,
+  maximumFractionDigits: 1,
+  signDisplay: "exceptZero",
+};
+
+export const timingFormat = {
+  style: "decimal",
+  minimumFractionDigits: 3,
+  maximumFractionDigits: 3,
+  signDisplay: "exceptZero",
+};
+
+export const durationFormat = {
+  style: "decimal",
+  minimumFractionDigits: 3,
+  maximumFractionDigits: 3,
+  signDisplay: "never",
+};
+
+export function calculateDifference(object1, object2) {
+  const difference = {};
+
+  for (const key in object1) {
+    if (typeof object1[key] === "object") {
+      difference[key] = calculateDifference(object1[key], object2[key]);
+    } else if (typeof object1[key] === "number") {
+      difference[key] = object2[key] - object1[key];
+    }
+  }
+
+  return difference;
+}

scripts/performance.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "Manhattan z14",
+    "z": 14,
+    "x": 4825,
+    "y": 6157
+  },
+  {
+    "name": "Fortaleza, Brazil z14",
+    "z": 14,
+    "x": 6435,
+    "y": 8361
+  }
+]

scripts/stats.js

@@ -1,6 +1,7 @@
 import * as Style from "../src/js/style.js";
 import config from "../src/config.js";
 import { Command, Option } from "commander";
+import { calcStatsJSON } from "./stats_json.js";
 
 const program = new Command();
 program
@@ -38,43 +39,20 @@ const style = Style.build(
 );
 
 const layers = style.layers;
-const layerCount = layers.length;
 
 if (opts.layerCount) {
+  const layerCount = layers.length;
   console.log(layerCount);
   process.exit();
 }
 
-const styleSize = JSON.stringify(layers).length;
-
 if (opts.layerSize) {
+  const styleSize = JSON.stringify(layers).length;
   console.log(styleSize);
   process.exit();
 }
 
-const layerMap = new Map();
-
-const stats = {
-  layerCount,
-  styleSize,
-  layerGroup: {},
-};
-
-for (let i = 0; i < layerCount; i++) {
-  const layer = layers[i];
-  layerMap.set(layer.id, layers[i]);
-  const layerSize = JSON.stringify(layer).length;
-  const layerGroup = layer["source-layer"] || layer.source || layer.type;
-  if (stats.layerGroup[layerGroup]) {
-    stats.layerGroup[layerGroup].size += layerSize;
-    stats.layerGroup[layerGroup].layerCount++;
-  } else {
-    stats.layerGroup[layerGroup] = {
-      size: layerSize,
-      layerCount: 1,
-    };
-  }
-}
+let stats = calcStatsJSON(style);
 
 if (opts.allJson) {
   process.stdout.write(JSON.stringify(stats, null, opts.pretty ? 2 : null));

scripts/stats_cmd.js

@@ -0,0 +1,5 @@
+import { calcStatsJSON } from "./stats_json.js";
+import { readFileSync } from "fs";
+
+const style = JSON.parse(readFileSync(process.argv[2], "utf8"));
+process.stdout.write(JSON.stringify(calcStatsJSON(style)));

scripts/stats_json.js

@@ -0,0 +1,29 @@
+export function calcStatsJSON(style) {
+  const layerMap = new Map();
+  const layers = style.layers;
+  const layerCount = layers.length;
+  const styleSize = JSON.stringify(style).length;
+
+  const stats = {
+    layerCount,
+    styleSize,
+    layerGroup: {},
+  };
+
+  for (let i = 0; i < layerCount; i++) {
+    const layer = layers[i];
+    layerMap.set(layer.id, layers[i]);
+    const layerSize = JSON.stringify(layer).length;
+    const layerGroup = layer["source-layer"] || layer.source || layer.type;
+    if (stats.layerGroup[layerGroup]) {
+      stats.layerGroup[layerGroup].size += layerSize;
+      stats.layerGroup[layerGroup].layerCount++;
+    } else {
+      stats.layerGroup[layerGroup] = {
+        size: layerSize,
+        layerCount: 1,
+      };
+    }
+  }
+  return stats;
+}