diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index f537eda2a7..7c903de241 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -4,7 +4,6 @@ import { renderToHtmlPage } from "../baker/siteRenderers.js" import { excludeUndefined, urlToSlug, - without, uniq, keyBy, compact, @@ -17,7 +16,6 @@ import { BAKED_GRAPHER_URL, } from "../settings/serverSettings.js" import * as db from "../db/db.js" -import { glob } from "glob" import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js" import { getPostIdFromSlug, @@ -50,9 +48,10 @@ import { import { getAllImages } from "../db/model/Image.js" import { logErrorAndMaybeCaptureInSentry } from "../serverUtils/errorLog.js" -import { getTagToSlugMap } from "./GrapherBakingUtils.js" +import { deleteOldGraphers, getTagToSlugMap } from "./GrapherBakingUtils.js" import { knexRaw } from "../db/db.js" import { getRelatedChartsForVariable } from "../db/model/Chart.js" +import { getAllMultiDimDataPageSlugs } from "../db/model/MultiDimDataPage.js" import pMap from "p-map" const renderDatapageIfApplicable = async ( @@ -286,28 +285,6 @@ const bakeGrapherPage = async ( imageMetadataDictionary ) ) - console.log(outPath) -} - -const deleteOldGraphers = async (bakedSiteDir: string, newSlugs: string[]) => { - // Delete any that are missing from the database - const oldSlugs = glob - .sync(`${bakedSiteDir}/grapher/*.html`) - .map((slug) => - slug.replace(`${bakedSiteDir}/grapher/`, "").replace(".html", "") - ) - const toRemove = without(oldSlugs, ...newSlugs) - // do not delete grapher slugs redirected to explorers - .filter((slug) => !isPathRedirectedToExplorer(`/grapher/${slug}`)) - for (const slug of toRemove) { - const path = `${bakedSiteDir}/grapher/${slug}.html` - console.log(`DELETING ${path}`) - fs.unlink(path, (err) => - err - ? console.error(`Error deleting ${path}`, err) - : console.log(`Deleted ${path}`) - ) - } } export interface BakeSingleGrapherChartArguments { @@ -363,7 +340,8 @@ export const bakeAllChangedGrapherPagesAndDeleteRemovedGraphers = async ( ORDER BY cc.slug ASC` ) - const newSlugs = chartsToBake.map((row) => row.slug) + chartsToBake.splice(20) // TODO + await fs.mkdirp(bakedSiteDir + "/grapher") // Prefetch imageMetadata instead of each grapher page fetching @@ -382,7 +360,7 @@ export const bakeAllChangedGrapherPagesAndDeleteRemovedGraphers = async ( })) const progressBar = new ProgressBar( - "bake grapher page [:bar] :current/:total :elapseds :rate/s :etas :name\n", + "bake grapher page [:bar] :current/:total :elapseds :rate/s :name\n", { width: 20, total: chartsToBake.length + 1, @@ -401,11 +379,18 @@ export const bakeAllChangedGrapherPagesAndDeleteRemovedGraphers = async ( async (knex) => await bakeSingleGrapherChart(job, knex), db.TransactionCloseMode.KeepOpen ) - progressBar.tick({ name: `slug ${job.slug}` }) + progressBar.tick({ name: job.slug }) }, { concurrency: 10 } ) - await deleteOldGraphers(bakedSiteDir, excludeUndefined(newSlugs)) + // Multi-dim data pages are baked into the same directory as graphers + // and they are handled separately. + const multiDimSlugs = await getAllMultiDimDataPageSlugs(knex) + const newSlugs = excludeUndefined([ + ...chartsToBake.map((row) => row.slug), + ...multiDimSlugs, + ]) + await deleteOldGraphers(bakedSiteDir, newSlugs) progressBar.tick({ name: `✅ Deleted old graphers` }) } diff --git a/baker/GrapherBakingUtils.ts b/baker/GrapherBakingUtils.ts index 11aeae74b9..794707a9a1 100644 --- a/baker/GrapherBakingUtils.ts +++ b/baker/GrapherBakingUtils.ts @@ -1,8 +1,11 @@ +import fs from "fs-extra" +import { glob } from "glob" import * as lodash from "lodash" import * as db from "../db/db.js" import md5 from "md5" -import { DbPlainTag, Url } from "@ourworldindata/utils" +import { DbPlainTag, Url, without } from "@ourworldindata/utils" +import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js" // Splits a grapher URL like https://ourworldindata.org/grapher/soil-lifespans?tab=chart // into its slug (soil-lifespans) and queryStr (?tab=chart) @@ -74,3 +77,25 @@ export async function getSlugForTopicTag( return slug } + +export async function deleteOldGraphers( + bakedSiteDir: string, + newSlugs: string[] +) { + // Delete any that are missing from the database + const oldSlugs = glob + .sync(`${bakedSiteDir}/grapher/*.html`) + .map((slug) => + slug.replace(`${bakedSiteDir}/grapher/`, "").replace(".html", "") + ) + const toRemove = without(oldSlugs, ...newSlugs) + // do not delete grapher slugs redirected to explorers + .filter((slug) => !isPathRedirectedToExplorer(`/grapher/${slug}`)) + for (const slug of toRemove) { + const path = `${bakedSiteDir}/grapher/${slug}.html` + console.log(`DELETING ${path}`) + fs.unlink(path, (err) => { + if (err) console.error(`Error deleting ${path}`, err) + }) + } +} diff --git a/baker/MultiDimBaker.tsx b/baker/MultiDimBaker.tsx index 1756c72505..da1903182a 100644 --- a/baker/MultiDimBaker.tsx +++ b/baker/MultiDimBaker.tsx @@ -1,5 +1,6 @@ import fs from "fs-extra" import path from "path" +import ProgressBar from "progress" import { ImageMetadata, MultiDimDataPageConfigPreProcessed, @@ -23,7 +24,7 @@ import { BAKED_BASE_URL, BAKED_GRAPHER_URL, } from "../settings/serverSettings.js" -import { getTagToSlugMap } from "./GrapherBakingUtils.js" +import { deleteOldGraphers, getTagToSlugMap } from "./GrapherBakingUtils.js" import { getVariableMetadata } from "../db/model/Variable.js" import pMap from "p-map" import { @@ -32,6 +33,7 @@ import { resolveFaqsForVariable, } from "./DatapageHelpers.js" import { logErrorAndMaybeCaptureInSentry } from "../serverUtils/errorLog.js" +import { getAllPublishedChartSlugs } from "../db/model/Chart.js" import { getAllMultiDimDataPages, getMultiDimDataPageBySlug, @@ -225,6 +227,14 @@ export const bakeAllMultiDimDataPages = async ( imageMetadata: Record ) => { const multiDimsBySlug = await getAllMultiDimDataPages(knex) + const progressBar = new ProgressBar( + "bake multi-dim page [:bar] :current/:total :elapseds :rate/s :name\n", + { + width: 20, + total: multiDimsBySlug.size + 1, + renderThrottle: 0, + } + ) for (const [slug, row] of multiDimsBySlug.entries()) { await bakeMultiDimDataPage( knex, @@ -233,5 +243,11 @@ export const bakeAllMultiDimDataPages = async ( row.config, imageMetadata ) + progressBar.tick({ name: slug }) } + const publishedSlugs = Object.keys(multiDimsBySlug) + const chartSlugs = await getAllPublishedChartSlugs(knex) + const newSlugs = [...publishedSlugs, ...chartSlugs] + await deleteOldGraphers(bakedSiteDir, newSlugs) + progressBar.tick({ name: `✅ Deleted old multi-dim pages` }) } diff --git a/db/model/Chart.ts b/db/model/Chart.ts index 29d10b87b6..17db69b58e 100644 --- a/db/model/Chart.ts +++ b/db/model/Chart.ts @@ -616,3 +616,20 @@ export const getRedirectsByChartId = async ( ORDER BY id ASC`, [chartId] ) + +export async function getAllPublishedChartSlugs( + knex: db.KnexReadonlyTransaction +): Promise { + const rows = await db.knexRaw<{ + slug: string + }>( + knex, + `-- sql + SELECT cc.slug + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.full->>"$.isPublished" = "true" + ` + ) + return rows.map((row) => row.slug) +} diff --git a/db/model/MultiDimDataPage.ts b/db/model/MultiDimDataPage.ts index 50fc52e0c2..25161092c2 100644 --- a/db/model/MultiDimDataPage.ts +++ b/db/model/MultiDimDataPage.ts @@ -84,6 +84,15 @@ export async function getAllLinkedPublishedMultiDimDataPages( return rows.map(enrichRow) } +export async function getAllMultiDimDataPageSlugs( + knex: KnexReadonlyTransaction +): Promise { + const rows = await knex( + MultiDimDataPagesTableName + ).select("slug") + return rows.map((row) => row.slug) +} + export const getMultiDimDataPageBySlug = async ( knex: KnexReadonlyTransaction, slug: string, diff --git a/devTools/tsconfigs/tsconfig.base.json b/devTools/tsconfigs/tsconfig.base.json index 3507ceb1f3..e6fcc81fa5 100644 --- a/devTools/tsconfigs/tsconfig.base.json +++ b/devTools/tsconfigs/tsconfig.base.json @@ -9,7 +9,23 @@ "declaration": true, "declarationMap": true, - "lib": ["dom", "dom.iterable", "es2020", "es2021", "es2022", "es2023"], + // To get newer APIs like Set.prototype.intersection(), we need to use + // ESNext lib on Node 22 and current version of TypeScript (5.7.2). + // However, using the ESNext option doesn't work with + // @types/node@20.8.3, which we are currently pinned to, to fix an + // unrelated bug with types in Cloudflare Functions, thus we only use + // ESNext.Collection. + // https://github.com/microsoft/TypeScript/issues/59919 + // https://developers.cloudflare.com/workers/languages/typescript/#transitive-loading-of-typesnode-overrides-cloudflareworkers-types + "lib": [ + "dom", + "dom.iterable", + "es2020", + "es2021", + "es2022", + "es2023", + "ESNext.Collection" + ], // Using es2022 as a `target` caused the following error in wrangler: // "Uncaught TypeError: PointVector is not a constructor". // This seems to be related to a change in how classes are compiled in