feat(explorers): allow referencing indicators by catalog path

baker/ExplorerBaker.tsx

@@ -6,6 +6,106 @@ import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.
 import { explorerRedirectTable } from "../explorerAdminServer/ExplorerRedirects.js"
 import { renderExplorerPage } from "./siteRenderers.js"
 import * as db from "../db/db.js"
+import { getVariableIdsByCatalogPath } from "../db/model/Variable.js"
+import { ExplorerGrammar } from "../explorer/ExplorerGrammar.js"
+import {
+    CoreTable,
+    ErrorValueTypes,
+    isNotErrorValueOrEmptyCell,
+} from "@ourworldindata/core-table"
+import { ColumnGrammar } from "../explorer/ColumnGrammar.js"
+import { ColumnTypeNames } from "@ourworldindata/types"
+
+export const transformExplorerProgramToResolveCatalogPaths = async (
+    program: ExplorerProgram,
+    knex: db.KnexReadonlyTransaction
+): Promise<{
+    program: ExplorerProgram
+    unresolvedCatalogPaths?: Set<string>
+}> => {
+    const { decisionMatrix } = program
+    const { requiredCatalogPaths } = decisionMatrix
+
+    if (requiredCatalogPaths.size === 0) return { program }
+
+    const catalogPathToIndicatorIdMap = await getVariableIdsByCatalogPath(
+        [...requiredCatalogPaths],
+        knex
+    )
+    const unresolvedCatalogPaths = new Set(
+        [...requiredCatalogPaths].filter(
+            (path) => !catalogPathToIndicatorIdMap.get(path)
+        )
+    )
+
+    const colSlugsToUpdate = decisionMatrix.allColumnsWithIndicatorIds.map(
+        (col) => col.slug
+    )
+    // In the decision matrix table, replace any catalog paths with their corresponding indicator ids
+    // If a catalog path is not found, it will be left as is
+    const newDecisionMatrixTable =
+        decisionMatrix.tableWithOriginalColumnNames.replaceCells(
+            colSlugsToUpdate,
+            (val) => {
+                if (typeof val === "string") {
+                    const vals = val.split(" ")
+                    const updatedVals = vals.map(
+                        (val) =>
+                            catalogPathToIndicatorIdMap.get(val)?.toString() ??
+                            val
+                    )
+                    return updatedVals.join(" ")
+                }
+                return val
+            }
+        )
+    const grapherBlockLine = program.getRowMatchingWords(
+        ExplorerGrammar.graphers.keyword
+    )
+    const newProgram = program.updateBlock(
+        grapherBlockLine,
+        newDecisionMatrixTable.toMatrix()
+    )
+
+    program.columnDefsByTableSlug.forEach((columnDefs, tableSlug) => {
+        const lineNoInProgram = newProgram.getRowMatchingWords(
+            ExplorerGrammar.columns.keyword,
+            tableSlug
+        )
+        const columnDefTable = new CoreTable(
+            newProgram.getBlock(lineNoInProgram)
+        )
+        const newColumnDefsTable = columnDefTable.combineColumns(
+            [
+                ColumnGrammar.variableId.keyword,
+                ColumnGrammar.catalogPath.keyword,
+            ],
+            {
+                slug: ColumnGrammar.variableId.keyword,
+                type: ColumnTypeNames.Numeric,
+            },
+            (row) => {
+                const variableId = row[ColumnGrammar.variableId.keyword]
+                if (isNotErrorValueOrEmptyCell(variableId)) return variableId
+
+                const catalogPath = row[ColumnGrammar.catalogPath.keyword]
+                if (
+                    isNotErrorValueOrEmptyCell(catalogPath) &&
+                    typeof catalogPath === "string"
+                ) {
+                    return (
+                        catalogPathToIndicatorIdMap.get(catalogPath) ??
+                        ErrorValueTypes.NoMatchingVariableId
+                    )
+                }
+                return ErrorValueTypes.NoMatchingVariableId
+            }
+        )
+        newProgram.updateBlock(lineNoInProgram, newColumnDefsTable.toMatrix())
+    })
+
+    return { program: newProgram, unresolvedCatalogPaths }
+}
 
 export const bakeAllPublishedExplorers = async (
     outputFolder: string,

baker/siteRenderers.tsx

@@ -91,6 +91,8 @@
     getAndLoadGdocBySlug,
     getAndLoadGdocById,
 } from "../db/model/Gdoc/GdocFactory.js"
+import { getVariableIdsByCatalogPath } from "../db/model/Variable.js"
+import { transformExplorerProgramToResolveCatalogPaths } from "./ExplorerBaker.js"
 
 export const renderToHtmlPage = (element: any) =>
     `<!doctype html>${ReactDOMServer.renderToStaticMarkup(element)}`
@@ -696,7 +698,22 @@
     knex: KnexReadonlyTransaction,
     urlMigrationSpec?: ExplorerPageUrlMigrationSpec
 ) => {
-    const { requiredGrapherIds, requiredVariableIds } = program.decisionMatrix
+    const transformResult = await transformExplorerProgramToResolveCatalogPaths(
+        program,
+        knex
+    )
+    const { program: transformedProgram, unresolvedCatalogPaths } =
+        transformResult
+    if (unresolvedCatalogPaths?.size) {
+        void logErrorAndMaybeSendToBugsnag(
+            new JsonError(
+                `${unresolvedCatalogPaths.size} catalog paths cannot be found for explorer ${transformedProgram.slug}: ${[...unresolvedCatalogPaths].join(", ")}.`
+            )
+        )
+    }
+
+    const { requiredGrapherIds, requiredVariableIds } =
+        transformedProgram.decisionMatrix
 
     type ChartRow = { id: number; config: string }
     let grapherConfigRows: ChartRow[] = []
@@ -726,7 +743,7 @@
         if (missingIds.length > 0) {
             void logErrorAndMaybeSendToBugsnag(
                 new JsonError(
-                    `Referenced variable IDs do not exist in the database for explorer ${program.slug}: ${missingIds.join(", ")}.`
+                    `Referenced variable IDs do not exist in the database for explorer ${transformedProgram.slug}: ${missingIds.join(", ")}.`
                 )
             )
         }
@@ -758,10 +775,13 @@
             return mergePartialGrapherConfigs(etlConfig, adminConfig)
         })
 
-    const wpContent = program.wpBlockId
+    const wpContent = transformedProgram.wpBlockId
         ? await renderReusableBlock(
-              await getBlockContentFromSnapshot(knex, program.wpBlockId),
-              program.wpBlockId,
+              await getBlockContentFromSnapshot(
+                  knex,
+                  transformedProgram.wpBlockId
+              ),
+              transformedProgram.wpBlockId,
               knex
           )
         : undefined
@@ -772,7 +792,7 @@
             <ExplorerPage
                 grapherConfigs={grapherConfigs}
                 partialGrapherConfigs={partialGrapherConfigs}
-                program={program}
+                program={transformedProgram}
                 wpContent={wpContent}
                 baseUrl={BAKED_BASE_URL}
                 urlMigrationSpec={urlMigrationSpec}

db/model/Variable.ts

@@ -19,6 +19,7 @@ import {
     DimensionProperty,
     GrapherInterface,
     DbRawVariable,
+    VariablesTableName,
 } from "@ourworldindata/types"
 import { knexRaw } from "../db.js"
 
@@ -578,3 +579,23 @@ export interface VariableResultView {
     table: string
     shortName: string
 }
+
+export const getVariableIdsByCatalogPath = async (
+    catalogPaths: string[],
+    knex: db.KnexReadonlyTransaction
+): Promise<Map<string, number | null>> => {
+    const rows: Pick<DbRawVariable, "id" | "catalogPath">[] = await knex
+        .select("id", "catalogPath")
+        .from(VariablesTableName)
+        .whereIn("catalogPath", catalogPaths)
+
+    const rowsByPath = _.keyBy(rows, "catalogPath")
+
+    // `rowsByPath` only contains the rows that were found, so we need to create
+    // a map where all keys from `catalogPaths` are present, and set the value to
+    // undefined if no row was found for that catalog path.
+    return new Map(
+        // Sort for good measure and determinism.
+        catalogPaths.sort().map((path) => [path, rowsByPath[path]?.id ?? null])
+    )
+}

explorer/ColumnGrammar.ts

@@ -8,6 +8,7 @@ import { ToleranceStrategy } from "@ourworldindata/utils"
 import {
     BooleanCellDef,
     EnumCellDef,
+    EtlPathCellDef,
     Grammar,
     IntegerCellDef,
     NumericCellDef,
@@ -22,6 +23,11 @@ export const ColumnGrammar: Grammar = {
         keyword: "variableId",
         description: "Numerical variable ID",
     },
+    catalogPath: {
+        ...EtlPathCellDef,
+        keyword: "catalogPath",
+        description: "Catalog path to the etl indicator",
+    },
     slug: {
         ...SlugDeclarationCellDef,
         keyword: "slug",

explorer/Explorer.tsx

@@ -41,6 +41,7 @@ import {
     keyBy,
     keyMap,
     omitUndefinedValues,
+    parseIntOrUndefined,
     PromiseCache,
     PromiseSwitcher,
     SerializedGridProgram,
@@ -501,8 +502,8 @@ export class Explorer
 
         const yVariableIdsList = yVariableIds
             .split(" ")
-            .map((item) => parseInt(item, 10))
-            .filter((item) => !isNaN(item))
+            .map(parseIntOrUndefined)
+            .filter((item) => item !== undefined)
 
         const partialGrapherConfig =
             this.partialGrapherConfigsByVariableId.get(yVariableIdsList[0]) ??

explorer/ExplorerDecisionMatrix.ts

@@ -5,6 +5,7 @@ import {
     identity,
     trimObject,
     uniq,
+    parseIntOrUndefined,
 } from "@ourworldindata/utils"
 import { ColumnTypeNames } from "@ourworldindata/types"
 import {
@@ -141,6 +142,45 @@ export class DecisionMatrix {
         )
     }
 
+    get allColumnsWithIndicatorIds() {
+        const indicatorColKeywords = [
+            GrapherGrammar.yVariableIds.keyword,
+            GrapherGrammar.xVariableId.keyword,
+            GrapherGrammar.colorVariableId.keyword,
+            GrapherGrammar.sizeVariableId.keyword,
+        ]
+        return this.table
+            .getColumns(indicatorColKeywords)
+            .filter((col) => !col.isMissing)
+    }
+
+    get requiredCatalogPaths(): Set<string> {
+        const allIndicators = this.allColumnsWithIndicatorIds
+            .flatMap((col) => col.uniqValues)
+            .flatMap((value) => value.split(" "))
+            .filter((value) => value !== "")
+
+        // Assume it's a catalog path if it doesn't look like a number
+        const catalogPaths = allIndicators.filter(
+            (indicator) => parseIntOrUndefined(indicator) === undefined
+        )
+
+        return new Set(catalogPaths)
+    }
+
+    // This is, basically, the inverse of `dropColumnTypes`.
+    get tableWithOriginalColumnNames() {
+        return this.table.renameColumns(
+            Object.fromEntries(
+                [...this.choiceNameToControlTypeMap.entries()].map(
+                    ([choiceName, controlType]) => {
+                        return [choiceName, `${choiceName} ${controlType}`]
+                    }
+                )
+            )
+        )
+    }
+
     choiceNameToControlTypeMap: Map<ChoiceName, ExplorerControlType>
     hash: string