feat: multi-dim data pages (#3657)

owid · Aug 21, 2024 · 4d0564c · 4d0564c
1 parent 127b286
commit 4d0564c
Show file tree

Hide file tree

Showing 30 changed files with 6,623 additions and 92 deletions.
diff --git a/adminSiteServer/mockSiteRouter.tsx b/adminSiteServer/mockSiteRouter.tsx
@@ -62,6 +62,7 @@ import {
 } from "./plainRouterHelpers.js"
 import { DEFAULT_LOCAL_BAKE_DIR } from "../site/SiteConstants.js"
 import { DATA_INSIGHTS_ATOM_FEED_NAME } from "../site/gdocs/utils.js"
+import { renderMultiDimDataPageBySlug } from "../baker/MultiDimBaker.js"
 
 require("express-async-errors")
 
@@ -201,15 +202,29 @@ getPlainRouteNonIdempotentWithRWTransaction(
     mockSiteRouter,
     "/grapher/:slug",
     async (req, res, trx) => {
-        const entity = await getChartConfigBySlug(trx, req.params.slug)
-        if (!entity) throw new JsonError("No such chart", 404)
+        const chartRow = await getChartConfigBySlug(trx, req.params.slug).catch(
+            console.error
+        )
+        if (chartRow) {
+            // XXX add dev-prod parity for this
+            res.set("Access-Control-Allow-Origin", "*")
 
-        // XXX add dev-prod parity for this
-        res.set("Access-Control-Allow-Origin", "*")
+            const previewDataPageOrGrapherPage =
+                await renderPreviewDataPageOrGrapherPage(chartRow.config, trx)
+            res.send(previewDataPageOrGrapherPage)
+            return
+        } else {
+            const page = await renderMultiDimDataPageBySlug(
+                trx,
+                req.params.slug
+            ).catch(console.error)
+            if (page) {
+                res.send(page)
+                return
+            }
+        }
 
-        const previewDataPageOrGrapherPage =
-            await renderPreviewDataPageOrGrapherPage(entity.config, trx)
-        res.send(previewDataPageOrGrapherPage)
+        throw new JsonError("No such chart", 404)
     }
 )
 

diff --git a/baker/DatapageHelpers.ts b/baker/DatapageHelpers.ts
@@ -9,14 +9,25 @@ import {
     getLastUpdatedFromVariable,
     getNextUpdateFromVariable,
     omitUndefinedValues,
+    partition,
 } from "@ourworldindata/utils"
 import {
     getGdocBaseObjectById,
+    getPublishedGdocBaseObjectBySlug,
     loadGdocFromGdocBase,
 } from "../db/model/Gdoc/GdocFactory.js"
 import { OwidGoogleAuth } from "../db/OwidGoogleAuth.js"
-import { GrapherInterface, OwidGdocBaseInterface } from "@ourworldindata/types"
+import {
+    EnrichedFaq,
+    FaqDictionary,
+    GrapherInterface,
+    OwidGdocBaseInterface,
+} from "@ourworldindata/types"
 import { KnexReadWriteTransaction } from "../db/db.js"
+import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js"
+import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
+import { getSlugForTopicTag } from "./GrapherBakingUtils.js"
+import { getShortPageCitation } from "../site/gdocs/utils.js"
 
 export const getDatapageDataV2 = async (
     variableMetadata: OwidVariableWithSource,
@@ -112,3 +123,101 @@ export const getDatapageGdoc = async (
 
     return datapageGdoc
 }
+
+type EnrichedFaqLookupError = {
+    type: "error"
+    error: string
+}
+
+type EnrichedFaqLookupSuccess = {
+    type: "success"
+    enrichedFaq: EnrichedFaq
+}
+
+type EnrichedFaqLookupResult = EnrichedFaqLookupError | EnrichedFaqLookupSuccess
+
+export const fetchAndParseFaqs = async (
+    knex: KnexReadWriteTransaction, // TODO: this transaction is only RW because somewhere inside it we fetch images
+    faqGdocIds: string[],
+    { isPreviewing }: { isPreviewing: boolean }
+) => {
+    const gdocFetchPromises = faqGdocIds.map((gdocId) =>
+        getDatapageGdoc(knex, gdocId, isPreviewing)
+    )
+    const gdocs = await Promise.all(gdocFetchPromises)
+    const gdocIdToFragmentIdToBlock: Record<string, FaqDictionary> = {}
+    gdocs.forEach((gdoc) => {
+        if (!gdoc) return
+        const faqs = parseFaqs(
+            ("faqs" in gdoc.content && gdoc.content?.faqs) ?? [],
+            gdoc.id
+        )
+        gdocIdToFragmentIdToBlock[gdoc.id] = faqs.faqs
+    })
+
+    return gdocIdToFragmentIdToBlock
+}
+
+export const resolveFaqsForVariable = (
+    gdocIdToFragmentIdToBlock: Record<string, FaqDictionary>,
+    variableMetadata: OwidVariableWithSource
+) => {
+    const resolvedFaqResults: EnrichedFaqLookupResult[] = variableMetadata
+        .presentation?.faqs
+        ? variableMetadata.presentation.faqs.map((faq) => {
+              const enrichedFaq = gdocIdToFragmentIdToBlock[faq.gdocId]?.[
+                  faq.fragmentId
+              ] as EnrichedFaq | undefined
+              if (!enrichedFaq)
+                  return {
+                      type: "error",
+                      error: `Could not find fragment ${faq.fragmentId} in gdoc ${faq.gdocId}`,
+                  }
+              return {
+                  type: "success",
+                  enrichedFaq,
+              }
+          })
+        : []
+
+    const [resolvedFaqs, errors] = partition(
+        resolvedFaqResults,
+        (result) => result.type === "success"
+    ) as [EnrichedFaqLookupSuccess[], EnrichedFaqLookupError[]]
+
+    return { resolvedFaqs, errors }
+}
+
+export const getPrimaryTopic = async (
+    knex: KnexReadWriteTransaction,
+    firstTopicTag: string | undefined
+) => {
+    if (!firstTopicTag) return undefined
+
+    let topicSlug: string
+    try {
+        topicSlug = await getSlugForTopicTag(knex, firstTopicTag)
+    } catch (e) {
+        await logErrorAndMaybeSendToBugsnag(
+            `Data page is using "${firstTopicTag}" as its primary tag, which we are unable to resolve to a tag in the grapher DB`
+        )
+        return undefined
+    }
+
+    if (topicSlug) {
+        const gdoc = await getPublishedGdocBaseObjectBySlug(
+            knex,
+            topicSlug,
+            true
+        )
+        if (gdoc) {
+            const citation = getShortPageCitation(
+                gdoc.content.authors,
+                gdoc.content.title ?? "",
+                gdoc?.publishedAt
+            )
+            return { topicTag: firstTopicTag, citation }
+        }
+    }
+    return undefined
+}
diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx
@@ -11,7 +11,6 @@ import {
     keyBy,
     mergePartialGrapherConfigs,
     compact,
-    partition,
 } from "@ourworldindata/utils"
 import fs from "fs-extra"
 import * as lodash from "lodash"
@@ -37,29 +36,28 @@ import {
     DimensionProperty,
     OwidVariableWithSource,
     OwidChartDimensionInterface,
-    EnrichedFaq,
     FaqEntryData,
-    FaqDictionary,
     ImageMetadata,
-    OwidGdocBaseInterface,
 } from "@ourworldindata/types"
 import ProgressBar from "progress"
 import {
     getVariableData,
     getMergedGrapherConfigForVariable,
     getVariableOfDatapageIfApplicable,
 } from "../db/model/Variable.js"
-import { getDatapageDataV2, getDatapageGdoc } from "./DatapageHelpers.js"
+import {
+    fetchAndParseFaqs,
+    getDatapageDataV2,
+    getPrimaryTopic,
+    resolveFaqsForVariable,
+} from "./DatapageHelpers.js"
 import { Image, getAllImages } from "../db/model/Image.js"
 import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
 
-import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js"
-import { getShortPageCitation } from "../site/gdocs/utils.js"
-import { getSlugForTopicTag, getTagToSlugMap } from "./GrapherBakingUtils.js"
+import { getTagToSlugMap } from "./GrapherBakingUtils.js"
 import { knexRaw } from "../db/db.js"
 import { getRelatedChartsForVariable } from "../db/model/Chart.js"
 import pMap from "p-map"
-import { getPublishedGdocBaseObjectBySlug } from "../db/model/Gdoc/GdocFactory.js"
 
 const renderDatapageIfApplicable = async (
     grapher: GrapherInterface,
@@ -114,18 +112,6 @@ export const renderDataPageOrGrapherPage = async (
     return renderGrapherPage(grapher, knex)
 }
 
-type EnrichedFaqLookupError = {
-    type: "error"
-    error: string
-}
-
-type EnrichedFaqLookupSuccess = {
-    type: "success"
-    enrichedFaq: EnrichedFaq
-}
-
-type EnrichedFaqLookupResult = EnrichedFaqLookupError | EnrichedFaqLookupSuccess
-
 export async function renderDataPageV2(
     {
         variableId,
@@ -158,45 +144,16 @@ export async function renderDataPageV2(
         ? mergePartialGrapherConfigs(grapherConfigForVariable, pageGrapher)
         : pageGrapher ?? {}
 
-    const faqDocs = compact(
+    const faqDocIds = compact(
         uniq(variableMetadata.presentation?.faqs?.map((faq) => faq.gdocId))
     )
-    const gdocFetchPromises = faqDocs.map((gdocId) =>
-        getDatapageGdoc(knex, gdocId, isPreviewing)
+
+    const faqGdocs = await fetchAndParseFaqs(knex, faqDocIds, { isPreviewing })
+
+    const { resolvedFaqs, errors: faqResolveErrors } = resolveFaqsForVariable(
+        faqGdocs,
+        variableMetadata
     )
-    const gdocs = await Promise.all(gdocFetchPromises)
-    const gdocIdToFragmentIdToBlock: Record<string, FaqDictionary> = {}
-    gdocs.forEach((gdoc) => {
-        if (!gdoc) return
-        const faqs = parseFaqs(
-            ("faqs" in gdoc.content && gdoc.content?.faqs) ?? [],
-            gdoc.id
-        )
-        gdocIdToFragmentIdToBlock[gdoc.id] = faqs.faqs
-    })
-
-    const resolvedFaqsResults: EnrichedFaqLookupResult[] = variableMetadata
-        .presentation?.faqs
-        ? variableMetadata.presentation.faqs.map((faq) => {
-              const enrichedFaq = gdocIdToFragmentIdToBlock[faq.gdocId]?.[
-                  faq.fragmentId
-              ] as EnrichedFaq | undefined
-              if (!enrichedFaq)
-                  return {
-                      type: "error",
-                      error: `Could not find fragment ${faq.fragmentId} in gdoc ${faq.gdocId}`,
-                  }
-              return {
-                  type: "success",
-                  enrichedFaq,
-              }
-          })
-        : []
-
-    const [resolvedFaqs, faqResolveErrors] = partition(
-        resolvedFaqsResults,
-        (result) => result.type === "success"
-    ) as [EnrichedFaqLookupSuccess[], EnrichedFaqLookupError[]]
 
     if (faqResolveErrors.length > 0) {
         for (const error of faqResolveErrors) {
@@ -234,32 +191,7 @@ export async function renderDataPageV2(
     )
 
     const firstTopicTag = datapageData.topicTagsLinks?.[0]
-
-    let slug = ""
-    if (firstTopicTag) {
-        try {
-            slug = await getSlugForTopicTag(knex, firstTopicTag)
-        } catch (error) {
-            await logErrorAndMaybeSendToBugsnag(
-                `Datapage with variableId "${variableId}" and title "${datapageData.title.title}" is using "${firstTopicTag}" as its primary tag, which we are unable to resolve to a tag in the grapher DB`
-            )
-        }
-        let gdoc: OwidGdocBaseInterface | undefined = undefined
-        if (slug) {
-            gdoc = await getPublishedGdocBaseObjectBySlug(knex, slug, true)
-        }
-        if (gdoc) {
-            const citation = getShortPageCitation(
-                gdoc.content.authors,
-                gdoc.content.title ?? "",
-                gdoc?.publishedAt
-            )
-            datapageData.primaryTopic = {
-                topicTag: firstTopicTag,
-                citation,
-            }
-        }
-    }
+    datapageData.primaryTopic = await getPrimaryTopic(knex, firstTopicTag)
 
     // Get the charts this variable is being used in (aka "related charts")
     // and exclude the current chart to avoid duplicates