CLDR-17803 site: add sitemap.xml

docs/site/.gitignore

@@ -1,3 +1,4 @@
 /node_modules
 /assets/json
 /assets/vendor
+/sitemap.xml

docs/site/assets/js/build.mjs

@@ -4,6 +4,8 @@ import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { default as process } from "node:process";
 import { default as matter } from "gray-matter";
+import { SitemapStream, streamToPromise } from "sitemap";
+import { Readable } from "node:stream";
 
 const SKIP_THESE = /(node_modules|\.jekyll-cache)/;
 
@@ -40,6 +42,28 @@ async function traverse(d, out) {
   return Promise.all(promises);
 }
 
+
+/** replace a/b/c.md with a/b/c.html */
+function md2html(p) {
+  return p.replace(/\.md$/, ".html");
+}
+
+
+async function writeSiteMaps(out) {
+  // simple list of links
+  const links = await Promise.all(out.all.map(async ({ fullPath, title }) => {
+    const stat = await fs.stat(fullPath);
+    return {
+      url: md2html(`/${fullPath}`),
+      lastmod: stat.mtime.toISOString(),
+    };
+  }));
+  const stream = new SitemapStream({ hostname: 'https://cldr.unicode.org'});
+  const data = (await streamToPromise(Readable.from(links).pipe(stream))).toString();
+  await fs.writeFile('./sitemap.xml', data, 'utf-8');
+  console.log('Wrote sitemap.xml');
+}
+
 async function main() {
   const out = {
     all: [],
@@ -48,6 +72,8 @@ async function main() {
   await fs.mkdir("assets/json/", { recursive: true });
   await traverse(".", out);
   await fs.writeFile("assets/json/tree.json", JSON.stringify(out, null, " "));
+  console.log('Wrote assets/json/tree.json');
+  await writeSiteMaps(out);
 }
 
 main().then(

docs/site/package.json

@@ -14,6 +14,7 @@
   "private": true,
   "dependencies": {
     "gray-matter": "^4.0.3",
+    "sitemap": "^8.0.0",
     "vue": "^3.5.0"
   }
 }

docs/site/robots.txt

@@ -0,0 +1 @@
+Sitemap: https://cldr.unicode.org/sitemap.xml