From 9ba7904866f5e672894ac15e0229a5b971a9b739 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 10 Sep 2024 11:23:22 -0500 Subject: [PATCH 1/9] CLDR-17934 site: create tsv sitemap --- docs/site/.gitignore | 2 + docs/site/assets/js/build.mjs | 104 ++++++++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/docs/site/.gitignore b/docs/site/.gitignore index 8336c7f1d7d..9b916be5cff 100644 --- a/docs/site/.gitignore +++ b/docs/site/.gitignore @@ -3,3 +3,5 @@ /assets/vendor /sitemap.xml /sitemap.md +/sitemap.tsv + diff --git a/docs/site/assets/js/build.mjs b/docs/site/assets/js/build.mjs index c185a19c817..235c6973cc2 100644 --- a/docs/site/assets/js/build.mjs +++ b/docs/site/assets/js/build.mjs @@ -9,6 +9,8 @@ import { Readable } from "node:stream"; const SKIP_THESE = /(node_modules|\.jekyll-cache|^sitemap.*)/; +const SITE = "https://cldr.unicode.org"; + async function processFile(d, fullPath, out) { const f = await fs.readFile(fullPath, "utf-8"); const m = matter(f); @@ -42,11 +44,88 @@ async function traverse(d, out) { return Promise.all(promises); } +/** replace a/b/c.md with a/b */ +function path2dir(p) { + const dir = p.split("/").slice(0, -1).join("/"); + return dir; +} + +/** replace a/b/c.md with a/b/c.html */ +function md2html(p) { + return p.replace(/\.md$/, ".html"); +} + +/** replace a/b/c.html with a/b/c.md */ +function html2md(p) { + return p.replace(/\.html$/, ".md"); +} + /** replace a/b/c.md with a/b/c */ function dropmd(p) { return p.replace(/\.md$/, ""); } +function tabs(n) { + let s = []; + for (let i = 0; i < n; i++) { + s.push("\t"); + } + return s.join(""); +} + +function mkurl(p) { + return `${SITE}/${md2html(p)}`; +} + +const coll = new Intl.Collator(["und"]); + +function writeSiteMapSheet({ all, allDirs }, path, outsheet) { + // write my index + function indexForPath(p) { + if (p === "") { + p = "index.md"; + } else { + p = path2dir(p) + ".md"; + } + return all.findIndex(({ fullPath }) => fullPath === p); + } + const myIndex = indexForPath(path); + if (myIndex === -1) { + throw Error(`Could not find index for ${path}`); + } + const { title, fullPath: indexPath } = all[myIndex]; + // find how how much to indent. + // 'path' is '' or 'foo/' or 'foo/bar/baz/' at this point. + const slashes = path.replace(/[^\/]+/g, ""); // foo/bar/ => // + const indent = tabs(slashes.length); // number of slashes => number of tabs + outsheet.push(`${indent}${title}\t${mkurl(indexPath)}`); + + // now, gather the children. + const children = all.filter(({ fullPath }) => { + if (fullPath === indexPath) return false; // no self-list. + const myDir = path2dir(fullPath); + // would this item be under our dir? + if (`${myDir}.md` === indexPath) return true; + // special case for odd /index subdir + if (indexPath === `index.md` && myDir === "") return true; + return false; + }); + + children.sort((a, b) => coll.compare(a.fullPath, b.fullPath)); + + children.forEach(({ title, fullPath }) => { + // if an index, recurse instead. + const baseName = dropmd(fullPath); // downloads.md -> downloads + if (allDirs.has(baseName)) { + // it's a non-leaf node, recurse. + writeSiteMapSheet({ all, allDirs }, `${baseName}/`, outsheet); + } else { + // write leaf (non-index) child pages + outsheet.push(`${indent}\t${title}\t${mkurl(fullPath)}`); + } + }); +} + async function writeSiteMaps(out) { // simple list of links const links = await Promise.all( @@ -58,15 +137,13 @@ async function writeSiteMaps(out) { }; }) ); - const stream = new SitemapStream({ hostname: "https://cldr.unicode.org" }); + const stream = new SitemapStream({ hostname: SITE }); const data = ( await streamToPromise(Readable.from(links).pipe(stream)) ).toString(); await fs.writeFile("./sitemap.xml", data, "utf-8"); - console.log("Wrote sitemap.xml"); + console.log(`Wrote sitemap.xml with ${links.length} entries`); - /* - const coll = new Intl.Collator(["und"]); const allSorted = [...out.all].sort((a, b) => coll.compare(a.fullPath, b.fullPath) ); @@ -82,7 +159,24 @@ async function writeSiteMaps(out) { "utf-8" ); console.log("Wrote sitemap.md"); - */ + + // now, create sitemap.tsv by walking + const outsheet = []; + const allPaths = out.all.map(({ fullPath }) => fullPath); + // Find all 'directories' (ending with /) + const allDirs = new Set(); + allPaths.forEach((p) => { + const segs = p.split("/").slice(0, -1); // ['', 'dir1'] + for (let n = 0; n <= segs.length; n++) { + // add all parent paths, so: '', dir1, dir1/dir2 etc. + const subpath = segs.slice(0, n).join("/"); + allDirs.add(subpath); + } + }); + + writeSiteMapSheet({ all: out.all, allDirs }, "", outsheet); + await fs.writeFile("./sitemap.tsv", outsheet.join("\n"), "utf-8"); + console.log(`wrote sitemap.tsv with ${outsheet.length} entries`); } async function main() { From 8fb8c935f9e49ec006ae7b32c1ed48bebb235ec8 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 11 Sep 2024 16:49:38 -0500 Subject: [PATCH 2/9] CLDR-17934 site: add Site Map to nav bar --- docs/site/_layouts/page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/site/_layouts/page.html b/docs/site/_layouts/page.html index 857db712665..a89b616329d 100644 --- a/docs/site/_layouts/page.html +++ b/docs/site/_layouts/page.html @@ -17,7 +17,7 @@
This navigation UI is temporary, just to give access to the pages.
- +
Site Map
From 6e6d6f597de235f8f03760e77e17c89fc74b819f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 11 Sep 2024 17:20:03 -0500 Subject: [PATCH 3/9] CLDR-17934 site: checkin initial sitemap.tsv --- docs/site/.gitignore | 2 +- docs/site/assets/js/build.mjs | 6 +- docs/site/sitemap.tsv | 194 ++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 docs/site/sitemap.tsv diff --git a/docs/site/.gitignore b/docs/site/.gitignore index 9b916be5cff..869cbb82391 100644 --- a/docs/site/.gitignore +++ b/docs/site/.gitignore @@ -3,5 +3,5 @@ /assets/vendor /sitemap.xml /sitemap.md -/sitemap.tsv + diff --git a/docs/site/assets/js/build.mjs b/docs/site/assets/js/build.mjs index 235c6973cc2..27cb59de016 100644 --- a/docs/site/assets/js/build.mjs +++ b/docs/site/assets/js/build.mjs @@ -98,7 +98,7 @@ function writeSiteMapSheet({ all, allDirs }, path, outsheet) { // 'path' is '' or 'foo/' or 'foo/bar/baz/' at this point. const slashes = path.replace(/[^\/]+/g, ""); // foo/bar/ => // const indent = tabs(slashes.length); // number of slashes => number of tabs - outsheet.push(`${indent}${title}\t${mkurl(indexPath)}`); + outsheet.push(`${indent}${dropmd(indexPath)}`); // now, gather the children. const children = all.filter(({ fullPath }) => { @@ -111,7 +111,7 @@ function writeSiteMapSheet({ all, allDirs }, path, outsheet) { return false; }); - children.sort((a, b) => coll.compare(a.fullPath, b.fullPath)); + children.sort((a, b) => coll.compare(a.title, b.title)); children.forEach(({ title, fullPath }) => { // if an index, recurse instead. @@ -121,7 +121,7 @@ function writeSiteMapSheet({ all, allDirs }, path, outsheet) { writeSiteMapSheet({ all, allDirs }, `${baseName}/`, outsheet); } else { // write leaf (non-index) child pages - outsheet.push(`${indent}\t${title}\t${mkurl(fullPath)}`); + outsheet.push(`${indent}\t${baseName}`); } }); } diff --git a/docs/site/sitemap.tsv b/docs/site/sitemap.tsv new file mode 100644 index 00000000000..04e33b6f9c7 --- /dev/null +++ b/docs/site/sitemap.tsv @@ -0,0 +1,194 @@ +index + index/acknowledgments + index/charts + ddl + downloads + downloads/cldr-31 + downloads/cldr-32 + downloads/cldr-33 + downloads/cldr-33-1 + downloads/cldr-34 + downloads/cldr-35 + downloads/cldr-36 + downloads/cldr-37 + downloads/cldr-38 + downloads/cldr-39 + downloads/cldr-40 + downloads/cldr-41 + downloads/cldr-42 + downloads/cldr-43 + downloads/cldr-44 + downloads/cldr-45 + downloads/cldr-46 + downloads/brs-copy-en_gb-to-en_001 + index/keyboard-workgroup + index/process + index/process/cldr-data-retention-policy + index/downloads + index/downloads/cldr-43 + index/downloads/cldr-44 + index/cldr-spec + index/cldr-spec/collation-guidelines + index/cldr-spec/core-data-for-new-locales + index/cldr-spec/coverage-levels + index/cldr-spec/currency-process + index/cldr-spec/definitions + index/cldr-spec/picking-the-right-language-code + index/cldr-spec/plural-rules + index/cldr-spec/transliteration-guidelines + index/survey-tool + index/survey-tool/bulk-data-upload + index/survey-tool/faq-and-known-bugs + index/survey-tool/managing-users + index/survey-tool/coverage + index/survey-tool/survey-tool-accounts + index/corrigenda + covered-by-other-projects + index/draft-schedule + translation + translation/characters + translation/characters/character-labels + translation/characters/short-names-and-keywords + translation/characters/typographic-names + translation/core-data + translation/core-data/characters + translation/core-data/numbering-systems + translation/core-data/exemplars + translation/currency-names-and-symbols + translation/currency-names-and-symbols/currency-names + translation/currency-names-and-symbols/special-cases + translation/date-time + translation/date-time/date-times-terminology + translation/date-time/date-time-names + translation/date-time/date-time-patterns + translation/date-time/date-time-symbols + translation/displaynames + translation/displaynames/countryregion-territory-names + translation/displaynames/languagelocale-name-patterns + translation/displaynames/languagelocale-names + translation/displaynames/locale-option-names-key + translation/displaynames/script-names + translation/error-codes + translation/translation-guide-general + translation/translation-guide-general/capitalization + translation/translation-guide-general/default-content + translation/translation-guide-general/references + translation/getting-started + translation/getting-started/data-stability + translation/getting-started/empty-cache + translation/getting-started/errors-and-warnings + translation/getting-started/resolving-errors + translation/getting-started/plurals + translation/getting-started/review-formats + translation/getting-started/guide + translation/getting-started/survey-tool-phases + translation/getting-started/vetting-view + translation/grammatical-inflection + translation/miscellaneous-displaying-lists + translation/number-currency-formats + translation/number-currency-formats/number-and-currency-patterns + translation/number-currency-formats/number-symbols + translation/number-currency-formats/other-patterns + translation/miscellaneous-person-name-formats + translation/time-zones-and-city-names + translation/transforms + translation/unique-translations + translation/units + translation/units/measurement-systems + translation/units/unit-names-and-patterns + translation/language-specific + translation/language-specific/lakota + translation/language-specific/odia + translation/language-specific/persian + development + development/adding-locales + development/creating-the-archive + development/cldr-development-site + development/cldr-development-site/running-cldr-tools + development/cldr-development-site/updating-englishroot + development/cldr-big-red-switch + development/cldr-big-red-switch/generating-charts + development/coding-cldr-tools + development/coding-cldr-tools/documenting-cldr-tools + development/guidance-on-direct-modifications-to-cldr-data + development/development-process + development/development-process/design-proposals + development/development-process/design-proposals/alternate-time-formats + development/development-process/design-proposals/bcp-47-changes-draft + development/development-process/design-proposals/bcp47-syntax-mapping + development/development-process/design-proposals/bcp47-validation-and-canonicalization + development/development-process/design-proposals/bidi-handling-of-structured-text + development/development-process/design-proposals/change-to-sites + development/development-process/design-proposals/chinese-and-other-calendar-support-intercalary-months-year-cycles + development/development-process/design-proposals/consistent-casing + development/development-process/design-proposals/coverage-revision + development/development-process/design-proposals/currency-code-fallback + development/development-process/design-proposals/day-period-design + development/development-process/design-proposals/delimiter-quotation-mark-proposal + development/development-process/design-proposals/english-inheritance + development/development-process/design-proposals/european-ordering-rules-issues + development/development-process/design-proposals/extended-windows-olson-zid-mapping + development/development-process/design-proposals/fractional-plurals + development/development-process/design-proposals/generic-calendar-data + development/development-process/design-proposals/grammar-capitalization-forms-for-datetime-elements-and-others + development/development-process/design-proposals/grapheme-usage + development/development-process/design-proposals/hebrew-months + development/development-process/design-proposals/index-characters + development/development-process/design-proposals/islamic-calendar-types + development/development-process/design-proposals/iso-636-deprecation-requests-draft + development/development-process/design-proposals/json-packaging-approved-by-the-cldr-tc-on-2015-03-25 + development/development-process/design-proposals/language-data-consistency + development/development-process/design-proposals/language-distance-data + development/development-process/design-proposals/list-formatting + development/development-process/design-proposals/locale-format + development/development-process/design-proposals/localized-gmt-format + development/development-process/design-proposals/math-formula-preferences + development/development-process/design-proposals/new-bcp47-extension-t-fields + development/development-process/design-proposals/new-time-zone-patterns + development/development-process/design-proposals/path-filtering + development/development-process/design-proposals/pattern-character-for-related-year + development/development-process/design-proposals/pinyin-fixes + development/development-process/design-proposals/post-mortem + development/development-process/design-proposals/proposed-collation-additions + development/development-process/design-proposals/resolution-of-cldr-files + development/development-process/design-proposals/script-metadata + development/development-process/design-proposals/search-collators + development/development-process/design-proposals/secularneutral-eras + development/development-process/design-proposals/specifying-text-break-variants-in-locale-ids + development/development-process/design-proposals/suggested-exemplar-revisions + development/development-process/design-proposals/supported-numberingsystems + development/development-process/design-proposals/thoughts-on-survey-tool-backend + development/development-process/design-proposals/time-zone-data-reorganization + development/development-process/design-proposals/transform-fallback + development/development-process/design-proposals/transform-keywords + development/development-process/design-proposals/unihan-data + development/development-process/design-proposals/units-pixels-ems-display-resolution + development/development-process/design-proposals/uts-35-splitting + development/development-process/design-proposals/voting + development/development-process/design-proposals/xmb + development/maven + development/new-cldr-developers + development/running-tests + development/running-tools + development/updating-codes + development/updating-codes/likelysubtags-and-default-content + development/updating-codes/update-currency-codes + development/updating-codes/update-language-script-info + development/updating-codes/update-language-script-info/language-script-description + development/updating-codes/update-languagescriptregion-subtags + development/updating-codes/update-time-zone-data-for-zoneparser + development/updating-codes/update-validity-xml + development/updating-codes/external-version-metadata + development/updating-codes/updating-population-gdp-literacy + development/updating-codes/updating-script-metadata + development/updating-codes/updating-subdivision-codes + development/updating-codes/updating-subdivision-translations + development/updating-codes/updating-un-codes + development/updating-dtds + index/json-format-data + index/language-support-levels + index/locale-coverage + index/requesting-additionsupdates-to-cldr-languagepopulation-data + stable-links-info + index/cldr-presentations + index/bcp47-extension \ No newline at end of file From 0c734603e97dd012b1e710b32454d14bba10c999 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 11 Sep 2024 18:41:11 -0500 Subject: [PATCH 4/9] CLDR-17934 site: parse sitemap.tsv --- docs/site/_layouts/page.html | 2 +- docs/site/assets/js/build.mjs | 223 +++++++++++++++++++++------------- 2 files changed, 140 insertions(+), 85 deletions(-) diff --git a/docs/site/_layouts/page.html b/docs/site/_layouts/page.html index a89b616329d..857db712665 100644 --- a/docs/site/_layouts/page.html +++ b/docs/site/_layouts/page.html @@ -17,7 +17,7 @@
This navigation UI is temporary, just to give access to the pages.
-
Site Map
+ diff --git a/docs/site/assets/js/build.mjs b/docs/site/assets/js/build.mjs index 27cb59de016..39ead34e91b 100644 --- a/docs/site/assets/js/build.mjs +++ b/docs/site/assets/js/build.mjs @@ -1,4 +1,4 @@ -// extract site frontmatter, save to json +// extract site frontmatter and read from /sitemap.tsv, save to json import * as fs from "node:fs/promises"; import * as path from "node:path"; @@ -6,11 +6,28 @@ import { default as process } from "node:process"; import { default as matter } from "gray-matter"; import { SitemapStream, streamToPromise } from "sitemap"; import { Readable } from "node:stream"; +import { Dirent } from "node:fs"; +// utilities and constants + +// files to skip const SKIP_THESE = /(node_modules|\.jekyll-cache|^sitemap.*)/; +// final URL of site const SITE = "https://cldr.unicode.org"; +// input file +const SITEMAPFILE = "sitemap.tsv"; + +// utility collator +const coll = new Intl.Collator(["und"]); + +/** + * Directory Crawler: process one directory + * @param {string} d directory paren + * @param {string} fullPath path to this file + * @param {object} out output object + */ async function processFile(d, fullPath, out) { const f = await fs.readFile(fullPath, "utf-8"); const m = matter(f); @@ -22,7 +39,13 @@ async function processFile(d, fullPath, out) { } } -/** process one dirent */ +/** + * Directory Crawler: process one dirent + * @param {string} d directory paren + * @param {object} out output object + * @param {Dirent} e directory entry + * @returns + */ async function processEntry(d, out, e) { const fullpath = path.join(d, e.name); if (SKIP_THESE.test(e.name)) return; @@ -35,6 +58,7 @@ async function processEntry(d, out, e) { } /** + * Directory Crawler: kick off the crawl (or subcrawl) of a directory * @param {string} d path to directory * @param {object} out output struct */ @@ -65,6 +89,11 @@ function dropmd(p) { return p.replace(/\.md$/, ""); } +/** + * + * @param {number} n + * @returns string with n tabs + */ function tabs(n) { let s = []; for (let i = 0; i < n; i++) { @@ -73,60 +102,12 @@ function tabs(n) { return s.join(""); } +/** convert a markdown path to a final URL */ function mkurl(p) { return `${SITE}/${md2html(p)}`; } -const coll = new Intl.Collator(["und"]); - -function writeSiteMapSheet({ all, allDirs }, path, outsheet) { - // write my index - function indexForPath(p) { - if (p === "") { - p = "index.md"; - } else { - p = path2dir(p) + ".md"; - } - return all.findIndex(({ fullPath }) => fullPath === p); - } - const myIndex = indexForPath(path); - if (myIndex === -1) { - throw Error(`Could not find index for ${path}`); - } - const { title, fullPath: indexPath } = all[myIndex]; - // find how how much to indent. - // 'path' is '' or 'foo/' or 'foo/bar/baz/' at this point. - const slashes = path.replace(/[^\/]+/g, ""); // foo/bar/ => // - const indent = tabs(slashes.length); // number of slashes => number of tabs - outsheet.push(`${indent}${dropmd(indexPath)}`); - - // now, gather the children. - const children = all.filter(({ fullPath }) => { - if (fullPath === indexPath) return false; // no self-list. - const myDir = path2dir(fullPath); - // would this item be under our dir? - if (`${myDir}.md` === indexPath) return true; - // special case for odd /index subdir - if (indexPath === `index.md` && myDir === "") return true; - return false; - }); - - children.sort((a, b) => coll.compare(a.title, b.title)); - - children.forEach(({ title, fullPath }) => { - // if an index, recurse instead. - const baseName = dropmd(fullPath); // downloads.md -> downloads - if (allDirs.has(baseName)) { - // it's a non-leaf node, recurse. - writeSiteMapSheet({ all, allDirs }, `${baseName}/`, outsheet); - } else { - // write leaf (non-index) child pages - outsheet.push(`${indent}\t${baseName}`); - } - }); -} - -async function writeSiteMaps(out) { +async function writeXmlSiteMap(out) { // simple list of links const links = await Promise.all( out.all.map(async ({ fullPath, title }) => { @@ -143,42 +124,114 @@ async function writeSiteMaps(out) { ).toString(); await fs.writeFile("./sitemap.xml", data, "utf-8"); console.log(`Wrote sitemap.xml with ${links.length} entries`); +} - const allSorted = [...out.all].sort((a, b) => - coll.compare(a.fullPath, b.fullPath) - ); - await fs.writeFile( - "./sitemap.md", - `---\ntitle: Site Map\n---\n\n` + - allSorted - .map( - ({ fullPath, title }) => - `- [/${fullPath}](/${dropmd(fullPath)}) - ${title}` - ) - .join("\n"), - "utf-8" - ); - console.log("Wrote sitemap.md"); - - // now, create sitemap.tsv by walking - const outsheet = []; - const allPaths = out.all.map(({ fullPath }) => fullPath); - // Find all 'directories' (ending with /) - const allDirs = new Set(); - allPaths.forEach((p) => { - const segs = p.split("/").slice(0, -1); // ['', 'dir1'] - for (let n = 0; n <= segs.length; n++) { - // add all parent paths, so: '', dir1, dir1/dir2 etc. - const subpath = segs.slice(0, n).join("/"); - allDirs.add(subpath); +async function readTsvSiteMap(out) { + console.log(`Reading ${SITEMAPFILE}`); + const lines = (await fs.readFile(SITEMAPFILE, "utf-8")).split("\n"); // don't skip comment lines here so we can get line numbers. + const errors = []; + + // user's specified map + const usermap = { + /* + index: { + parent: null, + title: 'CLDR Site', + children: [ + 'cldr-spec', + 'downloads', + … + ], + }, + 'cldr-spec': { + parent: 'index', + title: …, + children: [ + 'cldr-spec/collation-guidelines', + … + ], + }, + 'cldr-spec/collation-guidelines': { + parent: 'cldr-spec', + title: …, + children: null, + }, + */ + }; + // stack of parents, in order + let parents = []; + let n = 0; + for (let line of lines) { + n++; + const location = `${SITEMAPFILE}:${n}: `; // for errors + // skip comment or blank lines + if (/^[ \t]*#/.test(line) || !line.trim()) continue; + + // # of leading + const tabs = /^[\t]*/.exec(line)[0].length; + // rest of line: the actual path + const path = line.slice(tabs).trim(); + if (usermap[path]) { + errors.push(`${location} duplicate path: ${path}`); + continue; + } + const foundItem = out.all.find(({ fullPath }) => fullPath === `${path}.md`); + if (!foundItem) { + errors.push(`${location} could not find file: ${path}.md`); + continue; + } + if (!foundItem.title) { + errors.push(`${location} missing title in ${path}.md`); + // let this continue on + } + usermap[path] = { + title: foundItem.title ?? path, + }; + const parentCount = parents.length; + if (tabs < parentCount) { + /** + * index [1] + * foo [2] + * + */ + // outdent + if (tabs == 0) { + errors.push(`${location} can't have more than one root page!`); + break; + } + // drop 'n' parents + parents = parents.slice(0, tabs); + } else if (tabs > parentCount) { + // Error - wrong indent + errors.push( + `${location} indent too deep (expected ${parentCount} tabs at most)` + ); + continue; + } + const parent = parents.slice(-1)[0] || null; // calculate parent (null for index page) + usermap[path].parent = parent; + if (parent) { + // not for index + usermap[parent].children = usermap[parent].children ?? []; + usermap[parent].children.push(path); + } + parents.push(path); // for next time + } + out.usermap = usermap; + out.all.forEach(({ fullPath }) => { + if (!usermap[dropmd(fullPath)]) { + errors.push(`${SITEMAPFILE}: missing: ${dropmd(fullPath)}`); } }); - - writeSiteMapSheet({ all: out.all, allDirs }, "", outsheet); - await fs.writeFile("./sitemap.tsv", outsheet.join("\n"), "utf-8"); - console.log(`wrote sitemap.tsv with ${outsheet.length} entries`); + if (errors.length) { + errors.forEach((l) => console.error(l)); + throw Error(`${errors.length} errors reading tsv`); + } else { + console.log(`${SITEMAPFILE} Valid.`); + } } +/** top level async */ async function main() { const out = { all: [], @@ -186,9 +239,11 @@ async function main() { }; await fs.mkdir("assets/json/", { recursive: true }); await traverse(".", out); + await writeXmlSiteMap(out); + await readTsvSiteMap(out); + // write final json asset await fs.writeFile("assets/json/tree.json", JSON.stringify(out, null, " ")); console.log("Wrote assets/json/tree.json"); - await writeSiteMaps(out); } main().then( From c16e81daef8855006a18936604a4e99c9db0cb4f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 12 Sep 2024 18:20:35 -0500 Subject: [PATCH 5/9] CLDR-17934 site: use sitemap to drive nav --- docs/site/assets/js/build.mjs | 2 +- docs/site/assets/js/cldrsite.js | 232 +++++++++++++++----------------- 2 files changed, 107 insertions(+), 127 deletions(-) diff --git a/docs/site/assets/js/build.mjs b/docs/site/assets/js/build.mjs index 39ead34e91b..9eaf376d7b8 100644 --- a/docs/site/assets/js/build.mjs +++ b/docs/site/assets/js/build.mjs @@ -235,13 +235,13 @@ async function readTsvSiteMap(out) { async function main() { const out = { all: [], - dirs: {}, }; await fs.mkdir("assets/json/", { recursive: true }); await traverse(".", out); await writeXmlSiteMap(out); await readTsvSiteMap(out); // write final json asset + delete out.all; //not needed at this phase, so trim out of the deploy await fs.writeFile("assets/json/tree.json", JSON.stringify(out, null, " ")); console.log("Wrote assets/json/tree.json"); } diff --git a/docs/site/assets/js/cldrsite.js b/docs/site/assets/js/cldrsite.js index a93e610782c..67b505dddc2 100644 --- a/docs/site/assets/js/cldrsite.js +++ b/docs/site/assets/js/cldrsite.js @@ -18,55 +18,75 @@ function md2html(p) { return p.replace(/\.md$/, ".html"); } +/** replace a/b/c with to /a/b/c, also '' => '/' */ +function path2url(p) { + if (p === "index") { + return "/"; + } + return `/${p}`; +} + /** replace a/b/c.html with a/b/c.md */ function html2md(p) { return p.replace(/\.html$/, ".md"); } +/** replace a/b/c.md with a/b/c */ +function dropmd(p) { + return p.replace(/\.md$/, ""); +} + +/** replace a/b/c.html with a/b/c */ +function drophtml(p) { + return p.replace(/\.html$/, ""); +} + /** load and cook the site data */ async function siteData() { // load the json const d = await fetch("/assets/json/tree.json"); const j = await d.json(); - const { all } = j; - - // 'all' is an array of { title, fullPath } entries. - // Flat list of paths - const allPaths = all.map(({ fullPath }) => fullPath); - // Find all 'directories' (ending with /) - const allDirs = new Set(); - allPaths.forEach((p) => { - const segs = p.split("/").slice(0, -1); // ['', 'dir1'] - for (let n = 0; n <= segs.length; n++) { - // add all parent paths, so: '', dir1, dir1/dir2 etc. - const subpath = segs.slice(0, n).join("/"); - allDirs.add(subpath); - } - }); - j.allDirs = {}; - j.allIndexes = []; - // allDirs: '', index, downloads, etc… - allDirs.forEach((dir) => { - // presumed index page: /downloads -> /downloads.md - // also / -> /index.md - const dirIndex = `${dir || "index"}.md`; - // console.dir({dir, dirIndex}); - if (allPaths.indexOf(dirIndex) !== -1) { - j.allDirs[dir] = { index: dirIndex }; - j.allIndexes.push(dirIndex); - } else { - console.error(`No index page: ${dirIndex}`); - j.allDirs[dir] = {}; - } - j.allDirs[dir].pages = []; - }); - allPaths.forEach((p) => { - const dir = path2dir(p); - j.allDirs[dir].pages.push(p); - }); - // map md -> title - j.title = {}; - all.forEach(({ title, fullPath }) => (j.title[fullPath] = title)); + const { usermap } = j; + + // TODO: we want to phase out j.all in the deployment. So, we only unmarshall usermap. + + // // 'all' is an array of { title, fullPath } entries. + // // Flat list of paths + // const allPaths = Object.keys(usermap); + // // Find all 'directories' (ending with /) + // const allDirs = new Set(); + // allPaths.forEach((p) => { + // const segs = p.split("/").slice(0, -1); // ['', 'dir1'] + // for (let n = 0; n <= segs.length; n++) { + // // add all parent paths, so: '', dir1, dir1/dir2 etc. + // const subpath = segs.slice(0, n).join("/"); + // allDirs.add(subpath); + // } + // }); + // j.allDirs = {}; + // j.allIndexes = []; + // // allDirs: '', index, downloads, etc… + // allDirs.forEach((dir) => { + // // presumed index page: /downloads -> /downloads.md + // // also / -> /index.md + // const dirIndex = `${dir || "index"}.md`; + // // console.dir({dir, dirIndex}); + // if (allPaths.indexOf(dirIndex) !== -1) { + // j.allDirs[dir] = { index: dirIndex }; + // j.allIndexes.push(dirIndex); + // } else { + // console.error(`No index page: ${dirIndex}`); + // j.allDirs[dir] = {}; + // } + // j.allDirs[dir].pages = []; + // }); + // allPaths.forEach((p) => { + // const dir = path2dir(p); + // j.allDirs[dir].pages.push(p); + // }); + // // map md -> title + // j.title = {}; + // all.forEach(({ title, fullPath }) => (j.title[fullPath] = title)); return j; } @@ -97,109 +117,72 @@ const app = Vue.createApp( path: String, }, computed: { - mdPath() { + /** base path: 'index' or 'downloads/cldr-33' */ + base() { if (this.path) { - return html2md(this.path); - } - return null; - }, - ourDir() { - if (this.path) { - return path2dir(this.path); - } - return ""; - }, - ourIndex() { - if (this.tree?.value) { - // first ARE we an index page? - if (this.tree.value.allIndexes.indexOf(this.mdPath) != -1) { - return this.mdPath; // we are an index - } - return this.tree.value.allDirs[this.ourDir].index; - } - return null; - }, - ourIndexHtml() { - if (this.ourIndex) { - return md2html(this.ourIndex); + return drophtml(this.path); } else { - return null; - } - }, - ourIndexTitle() { - if (this.ourIndex && this.tree?.value) { - return this.tree.value.title[this.ourIndex] || this.ourIndex; - } else { - return null; + return "index"; // '' => 'index' } + return null; }, ourTitle() { if (this.tree?.value) { if (this.path === "") return this.rootTitle; - return this.tree.value.title[html2md(this.path)]; + return this?.tree?.value?.usermap[this.base]?.title; } }, // title of root rootTitle() { - return this.tree?.value?.title["index.md"]; + const usermap = this?.tree?.value?.usermap ?? {}; + return usermap?.index?.title ?? "CLDR"; }, - // list of pages for siblings of this dir - siblingPages() { - if (!this.tree?.value) return []; - let dirForPage = this.ourDir; - if (this.tree.value.allIndexes.indexOf(this.mdPath) != -1) { - const dirPages = Object.entries(this.tree?.value?.allDirs).filter( - ([k, { index }]) => index == this.mdPath - )[0]; - if (dirPages) { - // our page is an index -so, show the subpages instead of the siblings. - dirForPage = dirPages[0]; // the adjusted index - } else { - return []; // no sibling pages; - } - } else { - return []; // no sibling pages - } - let thePages = this.tree?.value?.allDirs[dirForPage].pages ?? []; - if (dirForPage === "") { - thePages = [...thePages, ...this.tree?.value?.allDirs["index"].pages]; - } - const c = new Intl.Collator([]); - const t = this; - return thePages - .map((path) => ({ - path, - html: md2html(path), - title: this.tree.value.title[path] ?? path, - })) - .sort((a, b) => c.compare(a.title, b.title)) - .filter(({ html }) => html != t.path); // skip showing the index page in the subpage list + children() { + const usermap = this?.tree?.value?.usermap; + if (!usermap) return []; // no children + const entry = usermap[this.base]; + const children = entry?.children; + if (!children || !children.length) return []; + return children.map((path) => ({ + path, + href: path2url(path), + title: usermap[path]?.title || path, + })); }, ancestorPages() { const pages = []; // if we are not loaded, or if we're at the root, then exit - if (!this.tree?.value || !this.path || this.path == "index.html") - return pages; + const usermap = this?.tree?.value?.usermap; + if ( + !usermap || + !this.path || + this.path == "index.html" || + this.map == "index" + ) { + return []; + } // traverse - let path = this.path; + let path = drophtml(this.path); // can't be null, empty, or index (see above). Map a/b/c.html to a/b/c do { // calculate the immediate ancestor - const pathMd = html2md(path); - const dir = path2dir(path); - const nextIndex = this.tree.value.allDirs[dir].index || "index.md"; // falls back to top - const nextIndexHtml = md2html(nextIndex); - const nextIndexTitle = this.tree.value.title[nextIndex]; + const nextParentPath = usermap[path]?.parent; + if (!nextParentPath) break; + if (nextParentPath == path) { + console.error("Loop detected!"); + break; + } + const nextParent = usermap[nextParentPath]; + if (!nextParent) break; + const href = path2url(nextParentPath); + const { title } = nextParent || nextParentPath; // prepend pages.push({ - href: "/" + nextIndexHtml, - title: nextIndexTitle, + href, + title, + path: nextParentPath, }); - if (nextIndexHtml == path) { - console.error("Loop detected from " + this.path); - path = "index.html"; // exit - } - path = nextIndexHtml; - } while (path && path != "index.html"); // we iterate over 'path', so html + path = nextParentPath; + } while (path); // we iterate over 'path' until it returns null pages.reverse(); return pages; }, @@ -215,18 +198,15 @@ const app = Vue.createApp( {{ ancestor.title }}❱ -
{{ ourTitle }}
+
{{ ourTitle }}
≡ {{ ourTitle }}
✕
    -
  • - - {{ subpage.title }} - - +
  • + {{ subpage.title }}
    • From 1c0c19f258d3c9a839da10bdecd213e47ccf43c8 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 12 Sep 2024 18:21:19 -0500 Subject: [PATCH 6/9] CLDR-17934 site: de-yellow title on CLDR-17947 --- docs/site/assets/css/page.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/site/assets/css/page.css b/docs/site/assets/css/page.css index 684f7bc8e85..a3a6119b2e7 100644 --- a/docs/site/assets/css/page.css +++ b/docs/site/assets/css/page.css @@ -27,7 +27,7 @@ header .navparent > div { header .title { display: table-cell; - color: yellow; + color: white; font-size: 1.5em; } From d3e73ddb0b5f9cb9723df0295893b6eb1bd28046 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 12 Sep 2024 18:27:22 -0500 Subject: [PATCH 7/9] CLDR-17934 site: use sitemap.tsv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add a ❱ marker for pages that have subpages - fix navigation detection --- docs/site/assets/css/page.css | 22 +++++++++++----------- docs/site/assets/js/cldrsite.js | 4 +++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/site/assets/css/page.css b/docs/site/assets/css/page.css index a3a6119b2e7..6f5b50aa896 100644 --- a/docs/site/assets/css/page.css +++ b/docs/site/assets/css/page.css @@ -41,22 +41,22 @@ header .nav a.uplink { } header .nav div.subpages { - box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.2); - z-index: 1; - background-color: white; - position: absolute; - color: black; - padding: 0.5em; + box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.2); + z-index: 1; + background-color: white; + position: absolute; + color: black; + padding: 0.5em; } div.subpages .hamburger { - left: 1em; - top: 1em; - color: darkslateblue; + left: 1em; + top: 1em; + color: darkslateblue; } .subpages .hamburger:hover { - color: gray; + color: gray; } header .nav ul b { @@ -80,7 +80,7 @@ header .nav ul li { } .subpages .li a { - color: black !important; + color: black !important; } header .message { diff --git a/docs/site/assets/js/cldrsite.js b/docs/site/assets/js/cldrsite.js index 67b505dddc2..54f1422ec83 100644 --- a/docs/site/assets/js/cldrsite.js +++ b/docs/site/assets/js/cldrsite.js @@ -38,7 +38,7 @@ function dropmd(p) { /** replace a/b/c.html with a/b/c */ function drophtml(p) { - return p.replace(/\.html$/, ""); + return p.replace(/\.html$/, "").replace(/\/$/, ""); } /** load and cook the site data */ @@ -147,6 +147,7 @@ const app = Vue.createApp( path, href: path2url(path), title: usermap[path]?.title || path, + children: (usermap[path].children ?? []).length > 0, })); }, ancestorPages() { @@ -208,6 +209,7 @@ const app = Vue.createApp(
  • {{ subpage.title }} + ❱
From f96ad152ea7dc98aed34061185143bafba00cbad Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 12 Sep 2024 18:30:14 -0500 Subject: [PATCH 8/9] CLDR-17934 site: cleanup js --- docs/site/assets/js/cldrsite.js | 52 --------------------------------- 1 file changed, 52 deletions(-) diff --git a/docs/site/assets/js/cldrsite.js b/docs/site/assets/js/cldrsite.js index 54f1422ec83..57342cf5768 100644 --- a/docs/site/assets/js/cldrsite.js +++ b/docs/site/assets/js/cldrsite.js @@ -48,45 +48,6 @@ async function siteData() { const j = await d.json(); const { usermap } = j; - // TODO: we want to phase out j.all in the deployment. So, we only unmarshall usermap. - - // // 'all' is an array of { title, fullPath } entries. - // // Flat list of paths - // const allPaths = Object.keys(usermap); - // // Find all 'directories' (ending with /) - // const allDirs = new Set(); - // allPaths.forEach((p) => { - // const segs = p.split("/").slice(0, -1); // ['', 'dir1'] - // for (let n = 0; n <= segs.length; n++) { - // // add all parent paths, so: '', dir1, dir1/dir2 etc. - // const subpath = segs.slice(0, n).join("/"); - // allDirs.add(subpath); - // } - // }); - // j.allDirs = {}; - // j.allIndexes = []; - // // allDirs: '', index, downloads, etc… - // allDirs.forEach((dir) => { - // // presumed index page: /downloads -> /downloads.md - // // also / -> /index.md - // const dirIndex = `${dir || "index"}.md`; - // // console.dir({dir, dirIndex}); - // if (allPaths.indexOf(dirIndex) !== -1) { - // j.allDirs[dir] = { index: dirIndex }; - // j.allIndexes.push(dirIndex); - // } else { - // console.error(`No index page: ${dirIndex}`); - // j.allDirs[dir] = {}; - // } - // j.allDirs[dir].pages = []; - // }); - // allPaths.forEach((p) => { - // const dir = path2dir(p); - // j.allDirs[dir].pages.push(p); - // }); - // // map md -> title - // j.title = {}; - // all.forEach(({ title, fullPath }) => (j.title[fullPath] = title)); return j; } @@ -226,19 +187,6 @@ const app = Vue.createApp( } ); -// app.component("CldrPage", { -// setup() {}, -// template: `

Hello

-// `, -// }); - -// app.component("CldrList", { -// setup() {}, -// template: ` -//

Hullo

-// `, -// }); - app.mount("#nav"); // load anchor.js From b54bdbed7a8fcee507275317ca580ca661f44415 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 12 Sep 2024 18:33:18 -0500 Subject: [PATCH 9/9] CLDR-17934 site: add comments to sitemap.tsv --- docs/site/sitemap.tsv | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/site/sitemap.tsv b/docs/site/sitemap.tsv index 04e33b6f9c7..f1c4851542e 100644 --- a/docs/site/sitemap.tsv +++ b/docs/site/sitemap.tsv @@ -1,3 +1,10 @@ +# CLDR Site Map. +# This is a comment. +# The file is a TSV, tab separated value. +# There must be a single 'index' entry here at the root. +# Every page must be listed. index/charts means index/charts.md for example. +# If an item has 'sub items' it becomes a directory parent. +# You can comment out lines, but an error will be given at build time if a page is missing. index index/acknowledgments index/charts @@ -191,4 +198,4 @@ index index/requesting-additionsupdates-to-cldr-languagepopulation-data stable-links-info index/cldr-presentations - index/bcp47-extension \ No newline at end of file + index/bcp47-extension