Pre-fetching tiles from server

[erikvullings]

How can we reproduce the crash?

I am developing a simple script to prefetch all tiles from a map server. However, it still has no proper timeout to wait for the tile to render, and it starts to fetch many tiles at once. Until bun crashes. My current vscode launch script and code:

    {
      "type": "bun",
      "internalConsoleOptions": "neverOpen",
      "request": "launch",
      "name": "Debug File",
      "program": "${file}",
      "cwd": "${workspaceFolder}",
      "stopOnEntry": false,
      "watchMode": false,
      "args": [
        "https://sr-data.hex.tno.nl/geoserver226/gwc/service/tms/1.0.0/tno-dbe:zodk_gemeente_vestigingen@EPSG:3857@pbf",
        "3.3",
        "7.2",
        "50.7",
        "53.6",
        "-z 18",
        "-o ./cache",
        "-c 5"
      ]
    }

import { Command } from "commander";
import path from "path";
import { mkdir, writeFile } from "fs/promises";

interface Tile {
  z: number;
  x: number;
  y: number;
}

interface BBox {
  minLon: number;
  maxLon: number;
  minLat: number;
  maxLat: number;
}

// Convert latitude/longitude to tile coordinates
function latLonToTile(
  lat: number,
  lon: number,
  zoom: number
): { x: number; y: number } {
  const latRad = (lat * Math.PI) / 180;
  const n = Math.pow(2, zoom);
  const x = Math.floor(((lon + 180) / 360) * n);
  const y = Math.floor(
    ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * n
  );
  return { x, y };
}

// Generate tiles within bounding box for a specific zoom level
function* generateTilesForZoom(bbox: BBox, zoom: number): Generator<Tile> {
  const minTile = latLonToTile(bbox.minLat, bbox.minLon, zoom);
  const maxTile = latLonToTile(bbox.maxLat, bbox.maxLon, zoom);

  // Ensure correct order of min/max
  const minX = Math.min(minTile.x, maxTile.x);
  const maxX = Math.max(minTile.x, maxTile.x);
  const minY = Math.min(minTile.y, maxTile.y);
  const maxY = Math.max(minTile.y, maxTile.y);

  for (let x = minX; x <= maxX; x++) {
    for (let y = minY; y <= maxY; y++) {
      yield { z: zoom, x, y };
    }
  }
}

// Read stream to Uint8Array
async function readStream(
  reader: ReadableStreamDefaultReader
): Promise<Uint8Array> {
  const chunks: Uint8Array[] = [];
  let totalSize = 0;

  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    chunks.push(value);
    totalSize += value.length;
  }

  const result = new Uint8Array(totalSize);
  let offset = 0;
  for (const chunk of chunks) {
    result.set(chunk, offset);
    offset += chunk.length;
  }
  return result;
}

// Fetch and optionally save tile data
async function fetchTileData(
  tile: Tile,
  serverUrl: string,
  outputDir?: string
) {
  const url = new URL(`/tiles/${tile.z}/${tile.x}/${tile.y}.pbf`, serverUrl);

  try {
    console.log(`Fetching tile ${tile.z}/${tile.x}/${tile.y}`);
    const response = await fetch(url.toString());

    if (!response.ok) {
      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
    }

    if (outputDir && response.body) {
      const reader = response.body.getReader();
      const content = await readStream(reader);

      const filename = path.join(
        outputDir,
        `${tile.z}`,
        `${tile.x}`,
        `${tile.y}.pbf`
      );
      await mkdir(path.dirname(filename), { recursive: true });
      await writeFile(filename, content);
      console.log(`Saved ${filename}`);
    }
  } catch (_error) {
    console.error(
      `Error processing tile ${tile.z}/${tile.x}/${tile.y}:`
      // error
    );
  }
}

async function main() {
  const program = new Command();

  program
    .name("tile-generator")
    .description(
      "Pre-render vector tiles for a map server within specified bounding box"
    )
    .argument("<server>", "Server URL (e.g., http://localhost:8080)")
    .argument("<minLon>", "Minimum longitude", parseFloat)
    .argument("<maxLon>", "Maximum longitude", parseFloat)
    .argument("<minLat>", "Minimum latitude", parseFloat)
    .argument("<maxLat>", "Maximum latitude", parseFloat)
    .option("-m, --min-zoom <number>", "Minimum zoom level", "8")
    .option("-z, --max-zoom <number>", "Maximum zoom level", "18")
    .option("-o, --output <dir>", "Output directory for cached tiles")
    .option("-c, --concurrency <number>", "Number of concurrent requests", "5");

  program.parse();

  const options = program.opts();
  const [server, minLon, maxLon, minLat, maxLat] = program.args;
  const minZoom = parseInt(options.minZoom);
  const maxZoom = parseInt(options.maxZoom);

  if (maxZoom < minZoom) {
    console.error(
      "Error: Maximum zoom level must be greater than the minimum zoom level."
    );
    process.exit(1);
  }
  const concurrency = parseInt(options.concurrency);

  // Example bounds for The Netherlands
  const bbox = {
    minLon: +minLon || 3.3, // Western point
    maxLon: +maxLon || 7.2, // Eastern point
    minLat: +minLat || 50.7, // Southern point
    maxLat: +maxLat || 53.6, // Northern point
  };

  console.log("Starting tile generation with parameters:");
  console.log(`Server: ${server}`);
  console.log(`Bounding box: ${JSON.stringify(bbox)}`);
  console.log(`Min zoom: ${minZoom}`);
  console.log(`Max zoom: ${maxZoom}`);
  console.log(`Output directory: ${options.output || "None (preview mode)"}`);
  console.log(`Concurrency: ${concurrency}`);

  // Generate and process tiles for each zoom level
  for (let zoom = minZoom; zoom <= maxZoom; zoom++) {
    const tiles = Array.from(generateTilesForZoom(bbox, zoom));
    console.log(`Processing zoom level ${zoom} (${tiles.length} tiles)`);

    // Process tiles with concurrency limit
    for (let i = 0; i < tiles.length; i += concurrency) {
      const batch = tiles.slice(i, i + concurrency);
      await Promise.all(
        batch.map((tile) => fetchTileData(tile, server, options.output))
      );
    }
  }

  console.log("Tile generation completed!");
}

if (import.meta.main) {
  main().catch((error) => {
    console.error("Fatal error:", error);
    process.exit(1);
  });
}

Relevant log output

Fetching tile 13/4258/2675
Fetching tile 13/4258/2676
============================================================
Bun v1.1.42 (50eec002) Linux x64
WSL Kernel v5.15.153 | glibc v2.35
CPU: sse42 popcnt avx avx2
Args: "bun" "/home/erik/dev/zodk/prerender/src/index.mts" "https://sr-data.hex.tno.nl/geoserver226/gwc/service/tms/1.0.0/tno-dbe:zodk_gemeente_vestigingen@EPSG:3857@pbf" "3.3" "7.2" "50.7" "53.6" "-z 18" "-o ./cache" "-c 5"
Features: fetch(12890) http_server jsc transpiler_cache tsconfig 
Builtins: "bun:main" "node:child_process" "node:events" "node:fs" "node:fs/promises" "node:path" "node:process" "node:string_decoder" "node:util/types" 
Elapsed: 61489ms | User: 5300ms | Sys: 9447ms
RSS: 1.07GB | Peak: 0.21GB | Commit: 1.07GB | Faults: 49

panic: Illegal instruction at address 0x59D8406
oh no: Bun has crashed. This indicates a bug in Bun, not your code.

Stack Trace (bun.report)

Bun v1.1.42 (50eec00) on linux x86_64 [AutoCommand]

Illegal instruction at address 0x059D8406

• ??
• ld-temp.o:0: bmalloc_heap_config_specialized_try_deallocate_not_small_exclusive_segregated
• ld-temp.o:0: void JSC::MarkedBlock::Handle::specializedSweep<true, (...)1, (...)1, (...)1, (...)0, (...)1, (...)1, JSC::IsoInlinedHeapCellType<JSC::JSString>::DestroyFunc>
• ld-temp.o:0: JSC::MarkedBlock::Handle::sweep
• ld-temp.o:0: JSC::LocalAllocator::tryAllocateIn
• ld-temp.o:0: JSC::LocalAllocator::allocateSlowCase
• LocalAllocatorInlines.h:41: JSC::JSString::create
• JSString.h:963: WTF::Detail::CallableWrapper<Bun::BunInspectorConnection::sendMessageToDebuggerThread(...)::'lambda'(...), void, WebCore::ScriptExecutionContext&>::call
• Function.h:82: Bun__performTask
• event_loop.zig:300: src.bun.js.event_loop.EventLoop.tickQueueWithCount__anon_147067

Features: fetch, http_server, jsc, transpiler_cache, tsconfig, tsconfig

[github-actions]

@erikvullings, thank you for reporting this crash. The latest version of Bun is v1.2.0, but this crash was reported on Bun v1.1.42.

Are you able to reproduce this crash on the latest version of Bun?

bun upgrade

For Bun's internal tracking, this issue is BUN-BKA.