wip: transform explorer server-side

owid · Jul 2, 2024 · 064bcd6 · 064bcd6
1 parent 81ca1c2
commit 064bcd6
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 39 deletions.
diff --git a/baker/ExplorerBaker.tsx b/baker/ExplorerBaker.tsx
@@ -6,6 +6,85 @@ import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.
 import { explorerRedirectTable } from "../explorerAdminServer/ExplorerRedirects.js"
 import { renderExplorerPage } from "./siteRenderers.js"
 import * as db from "../db/db.js"
+import { getVariableIdsByCatalogPath } from "../db/model/Variable.js"
+import { ExplorerGrammar } from "../explorer/ExplorerGrammar.js"
+import { CoreTable } from "@ourworldindata/core-table"
+import { ColumnGrammar } from "../explorer/ColumnGrammar.js"
+
+export const transformExplorerProgramToResolveCatalogPaths = async (
+    program: ExplorerProgram,
+    knex: db.KnexReadonlyTransaction
+): Promise<{
+    program: ExplorerProgram
+    unresolvedCatalogPaths?: Set<string>
+}> => {
+    const { decisionMatrix } = program
+    const { requiredCatalogPaths } = decisionMatrix
+
+    if (requiredCatalogPaths.size === 0) return { program }
+
+    const catalogPathToIndicatorIdMap = await getVariableIdsByCatalogPath(
+        [...requiredCatalogPaths],
+        knex
+    )
+    const unresolvedCatalogPaths = new Set(
+        [...requiredCatalogPaths].filter(
+            (path) => !catalogPathToIndicatorIdMap.get(path)
+        )
+    )
+
+    const colSlugsToUpdate = decisionMatrix.allColumnsWithIndicatorIds.map(
+        (col) => col.slug
+    )
+    // In the decision matrix table, replace any catalog paths with their corresponding indicator ids
+    // If a catalog path is not found, it will be left as is
+    const newDecisionMatrixTable =
+        decisionMatrix.tableWithOriginalColumnNames.replaceCells(
+            colSlugsToUpdate,
+            (val) => {
+                if (typeof val === "string") {
+                    const vals = val.split(" ")
+                    const updatedVals = vals.map(
+                        (val) =>
+                            catalogPathToIndicatorIdMap.get(val)?.toString() ??
+                            val
+                    )
+                    return updatedVals.join(" ")
+                }
+                return val
+            }
+        )
+    const grapherBlockLine = program.getKeywordIndex(
+        ExplorerGrammar.graphers.keyword
+    )
+    const newProgram = program.updateBlock(
+        grapherBlockLine,
+        newDecisionMatrixTable.toMatrix()
+    )
+
+    program.columnDefsByTableSlug.forEach((columnDefs, tableSlug) => {
+        const lineNoInProgram = newProgram.getRowMatchingWords(
+            ExplorerGrammar.columns.keyword,
+            tableSlug
+        )
+        const columnDefTable = new CoreTable(
+            newProgram.getBlock(lineNoInProgram)
+        )
+        const newColumnDefsTable = columnDefTable.replaceCells(
+            [ColumnGrammar.variableId.keyword],
+            (val) => {
+                if (typeof val === "string")
+                    return (
+                        catalogPathToIndicatorIdMap.get(val)?.toString() ?? val
+                    )
+                return val
+            }
+        )
+        newProgram.updateBlock(lineNoInProgram, newColumnDefsTable.toMatrix())
+    })
+
+    return { program: newProgram, unresolvedCatalogPaths }
+}
 
 export const bakeAllPublishedExplorers = async (
     outputFolder: string,

diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx
@@ -92,6 +92,7 @@ import {
     getAndLoadGdocById,
 } from "../db/model/Gdoc/GdocFactory.js"
 import { getVariableIdsByCatalogPath } from "../db/model/Variable.js"
+import { transformExplorerProgramToResolveCatalogPaths } from "./ExplorerBaker.js"
 
 export const renderToHtmlPage = (element: any) =>
     `<!doctype html>${ReactDOMServer.renderToStaticMarkup(element)}`
@@ -697,8 +698,22 @@ export const renderExplorerPage = async (
     knex: KnexReadonlyTransaction,
     urlMigrationSpec?: ExplorerPageUrlMigrationSpec
 ) => {
-    const { requiredGrapherIds, requiredVariableIds, requiredCatalogPaths } =
-        program.decisionMatrix
+    const transformResult = await transformExplorerProgramToResolveCatalogPaths(
+        program,
+        knex
+    )
+    const { program: transformedProgram, unresolvedCatalogPaths } =
+        transformResult
+    if (unresolvedCatalogPaths?.size) {
+        void logErrorAndMaybeSendToBugsnag(
+            new JsonError(
+                `${unresolvedCatalogPaths.size} catalog paths cannot be found for explorer ${transformedProgram.slug}: ${[...unresolvedCatalogPaths].join(", ")}.`
+            )
+        )
+    }
+
+    const { requiredGrapherIds, requiredVariableIds } =
+        transformedProgram.decisionMatrix
 
     type ChartRow = { id: number; config: string }
     let grapherConfigRows: ChartRow[] = []
@@ -728,7 +743,7 @@ export const renderExplorerPage = async (
         if (missingIds.length > 0) {
             void logErrorAndMaybeSendToBugsnag(
                 new JsonError(
-                    `Referenced variable IDs do not exist in the database for explorer ${program.slug}: ${missingIds.join(", ")}.`
+                    `Referenced variable IDs do not exist in the database for explorer ${transformedProgram.slug}: ${missingIds.join(", ")}.`
                 )
             )
         }
@@ -760,37 +775,24 @@ export const renderExplorerPage = async (
             return mergePartialGrapherConfigs(etlConfig, adminConfig)
         })
 
-    const wpContent = program.wpBlockId
+    const wpContent = transformedProgram.wpBlockId
         ? await renderReusableBlock(
-              await getBlockContentFromSnapshot(knex, program.wpBlockId),
-              program.wpBlockId,
+              await getBlockContentFromSnapshot(
+                  knex,
+                  transformedProgram.wpBlockId
+              ),
+              transformedProgram.wpBlockId,
               knex
           )
         : undefined
 
-    const catalogPathToIndicatorIdMap = await getVariableIdsByCatalogPath(
-        [...requiredCatalogPaths],
-        knex
-    )
-    const unresolvedCatalogPaths = [...requiredCatalogPaths].filter(
-        (path) => !catalogPathToIndicatorIdMap[path]
-    )
-    if (unresolvedCatalogPaths.length > 0) {
-        void logErrorAndMaybeSendToBugsnag(
-            new JsonError(
-                `${unresolvedCatalogPaths.length} catalog paths cannot be found for explorer ${program.slug}: ${unresolvedCatalogPaths.join(", ")}.`
-            )
-        )
-    }
-
     return (
         `<!doctype html>` +
         ReactDOMServer.renderToStaticMarkup(
             <ExplorerPage
                 grapherConfigs={grapherConfigs}
                 partialGrapherConfigs={partialGrapherConfigs}
-                catalogPathToIndicatorIdMap={catalogPathToIndicatorIdMap}
-                program={program}
+                program={transformedProgram}
                 wpContent={wpContent}
                 baseUrl={BAKED_BASE_URL}
                 urlMigrationSpec={urlMigrationSpec}

diff --git a/db/model/Variable.ts b/db/model/Variable.ts
@@ -583,7 +583,7 @@ export interface VariableResultView {
 export const getVariableIdsByCatalogPath = async (
     catalogPaths: string[],
     knex: db.KnexReadonlyTransaction
-): Promise<Record<string, number | null>> => {
+): Promise<Map<string, number | null>> => {
     const rows: Pick<DbRawVariable, "id" | "catalogPath">[] = await knex
         .select("id", "catalogPath")
         .from(VariablesTableName)
@@ -594,7 +594,7 @@ export const getVariableIdsByCatalogPath = async (
     // `rowsByPath` only contains the rows that were found, so we need to create
     // a map where all keys from `catalogPaths` are present, and set the value to
     // undefined if no row was found for that catalog path.
-    return Object.fromEntries(
+    return new Map(
         // Sort for good measure and determinism.
         catalogPaths.sort().map((path) => [path, rowsByPath[path]?.id ?? null])
     )

diff --git a/explorer/ExplorerConstants.ts b/explorer/ExplorerConstants.ts
@@ -64,8 +64,6 @@ export const EMBEDDED_EXPLORER_GRAPHER_CONFIGS =
     "\n//EMBEDDED_EXPLORER_GRAPHER_CONFIGS\n"
 export const EMBEDDED_EXPLORER_PARTIAL_GRAPHER_CONFIGS =
     "\n//EMBEDDED_PARTIAL_EXPLORER_GRAPHER_CONFIGS\n"
-export const EMBEDDED_EXPLORER_CATALOG_PATH_TO_INDICATOR_ID_MAP =
-    "\n//EMBEDDED_EXPLORER_CATALOG_PATH_TO_INDICATOR_ID_MAP\n"
 
 export const EXPLORER_EMBEDDED_FIGURE_SELECTOR = "data-explorer-src"
 

diff --git a/explorer/ExplorerDecisionMatrix.ts b/explorer/ExplorerDecisionMatrix.ts
@@ -142,16 +142,23 @@ export class DecisionMatrix {
         )
     }
 
-    get requiredCatalogPaths(): Set<string> {
+    get allColumnsWithIndicatorIds() {
         const indicatorColKeywords = [
             GrapherGrammar.yVariableIds.keyword,
             GrapherGrammar.xVariableId.keyword,
             GrapherGrammar.colorVariableId.keyword,
             GrapherGrammar.sizeVariableId.keyword,
         ]
-        const allIndicators = indicatorColKeywords
-            .flatMap((keyword) => this.table.get(keyword).uniqValues)
+        return this.table
+            .getColumns(indicatorColKeywords)
+            .filter((col) => !col.isMissing)
+    }
+
+    get requiredCatalogPaths(): Set<string> {
+        const allIndicators = this.allColumnsWithIndicatorIds
+            .flatMap((col) => col.uniqValues)
             .flatMap((value) => value.split(" "))
+            .filter((value) => value !== "")
 
         // Assume it's a catalog path if it doesn't look like a number
         const catalogPaths = allIndicators.filter(
@@ -161,6 +168,19 @@ export class DecisionMatrix {
         return new Set(catalogPaths)
     }
 
+    // This is, basically, the inverse of `dropColumnTypes`.
+    get tableWithOriginalColumnNames() {
+        return this.table.renameColumns(
+            Object.fromEntries(
+                [...this.choiceNameToControlTypeMap.entries()].map(
+                    ([choiceName, controlType]) => {
+                        return [choiceName, `${choiceName} ${controlType}`]
+                    }
+                )
+            )
+        )
+    }
+
     choiceNameToControlTypeMap: Map<ChoiceName, ExplorerControlType>
     hash: string
 

diff --git a/gridLang/GridProgram.ts b/gridLang/GridProgram.ts
@@ -197,7 +197,7 @@ export class GridProgram {
         return { startRow, endRow: startRow + numRows, numRows }
     }
 
-    protected getKeywordIndex(key: string) {
+    getKeywordIndex(key: string) {
         return this.lines.findIndex((line) => line[0] === key)
     }
 

diff --git a/site/ExplorerPage.tsx b/site/ExplorerPage.tsx
@@ -9,7 +9,6 @@ import {
 } from "@ourworldindata/utils"
 import React from "react"
 import {
-    EMBEDDED_EXPLORER_CATALOG_PATH_TO_INDICATOR_ID_MAP,
     EMBEDDED_EXPLORER_DELIMITER,
     EMBEDDED_EXPLORER_GRAPHER_CONFIGS,
     EMBEDDED_EXPLORER_PARTIAL_GRAPHER_CONFIGS,
@@ -29,7 +28,6 @@ interface ExplorerPageSettings {
     wpContent?: string
     grapherConfigs: GrapherInterface[]
     partialGrapherConfigs: GrapherInterface[]
-    catalogPathToIndicatorIdMap: Record<string, number | null>
     baseUrl: string
     urlMigrationSpec?: ExplorerPageUrlMigrationSpec
 }
@@ -61,7 +59,6 @@ export const ExplorerPage = (props: ExplorerPageSettings) => {
         program,
         grapherConfigs,
         partialGrapherConfigs,
-        catalogPathToIndicatorIdMap,
         baseUrl,
         urlMigrationSpec,
     } = props
@@ -96,11 +93,7 @@ const partialGrapherConfigs = ${serializeJSONForHTML(
 const urlMigrationSpec = ${
         urlMigrationSpec ? JSON.stringify(urlMigrationSpec) : "undefined"
     };
-const catalogPathToIndicatorIdMap = ${serializeJSONForHTML(
-        catalogPathToIndicatorIdMap,
-        EMBEDDED_EXPLORER_CATALOG_PATH_TO_INDICATOR_ID_MAP
-    )};
-window.Explorer.renderSingleExplorerOnExplorerPage(explorerProgram, grapherConfigs, partialGrapherConfigs, urlMigrationSpec, catalogPathToIndicatorIdMap);`
+window.Explorer.renderSingleExplorerOnExplorerPage(explorerProgram, grapherConfigs, partialGrapherConfigs, urlMigrationSpec);`
 
     return (
         <html>