feat: resolve indicator ids at bake time

baker/ExplorerBaker.tsx

@@ -8,8 +8,13 @@ import { renderExplorerPage } from "./siteRenderers.js"
 import * as db from "../db/db.js"
 import { getVariableIdsByCatalogPath } from "../db/model/Variable.js"
 import { ExplorerGrammar } from "../explorer/ExplorerGrammar.js"
-import { CoreTable } from "@ourworldindata/core-table"
+import {
+    CoreTable,
+    ErrorValueTypes,
+    isNotErrorValueOrEmptyCell,
+} from "@ourworldindata/core-table"
 import { ColumnGrammar } from "../explorer/ColumnGrammar.js"
+import { ColumnTypeNames } from "@ourworldindata/types"
 
 export const transformExplorerProgramToResolveCatalogPaths = async (
     program: ExplorerProgram,
@@ -70,14 +75,30 @@ export const transformExplorerProgramToResolveCatalogPaths = async (
         const columnDefTable = new CoreTable(
             newProgram.getBlock(lineNoInProgram)
         )
-        const newColumnDefsTable = columnDefTable.replaceCells(
-            [ColumnGrammar.variableId.keyword],
-            (val) => {
-                if (typeof val === "string")
+        const newColumnDefsTable = columnDefTable.combineColumns(
+            [
+                ColumnGrammar.variableId.keyword,
+                ColumnGrammar.catalogPath.keyword,
+            ],
+            {
+                slug: ColumnGrammar.variableId.keyword,
+                type: ColumnTypeNames.Numeric,
+            },
+            (row) => {
+                const variableId = row[ColumnGrammar.variableId.keyword]
+                if (isNotErrorValueOrEmptyCell(variableId)) return variableId
+
+                const catalogPath = row[ColumnGrammar.catalogPath.keyword]
+                if (
+                    isNotErrorValueOrEmptyCell(catalogPath) &&
+                    typeof catalogPath === "string"
+                ) {
                     return (
-                        catalogPathToIndicatorIdMap.get(val)?.toString() ?? val
+                        catalogPathToIndicatorIdMap.get(catalogPath) ??
+                        ErrorValueTypes.NoMatchingVariableId
                     )
-                return val
+                }
+                return ErrorValueTypes.NoMatchingVariableId
             }
         )
         newProgram.updateBlock(lineNoInProgram, newColumnDefsTable.toMatrix())

explorer/ColumnGrammar.ts

@@ -8,8 +8,8 @@ import { ToleranceStrategy } from "@ourworldindata/utils"
 import {
     BooleanCellDef,
     EnumCellDef,
+    EtlPathCellDef,
     Grammar,
-    IndicatorIdOrEtlPathCellDef,
     IntegerCellDef,
     NumericCellDef,
     SlugDeclarationCellDef,
@@ -19,9 +19,14 @@ import {
 
 export const ColumnGrammar: Grammar = {
     variableId: {
-        ...IndicatorIdOrEtlPathCellDef,
+        ...IntegerCellDef,
         keyword: "variableId",
-        description: "Numerical variable ID, or ETL path",
+        description: "Numerical variable ID",
+    },
+    catalogPath: {
+        ...EtlPathCellDef,
+        keyword: "catalogPath",
+        description: "Catalog path to the etl indicator",
     },
     slug: {
         ...SlugDeclarationCellDef,

explorer/Explorer.tsx

@@ -84,7 +84,6 @@ import Bugsnag from "@bugsnag/js"
 export interface ExplorerProps extends SerializedGridProgram {
     grapherConfigs?: GrapherInterface[]
     partialGrapherConfigs?: GrapherInterface[]
-    catalogPathToIndicatorIdMap?: Record<string, number | null>
     queryStr?: string
     isEmbeddedInAnOwidPage?: boolean
     isInStandalonePage?: boolean
@@ -131,14 +130,12 @@ export class Explorer
         program: ExplorerProps,
         grapherConfigs: GrapherInterface[],
         partialGrapherConfigs: GrapherInterface[],
-        urlMigrationSpec?: ExplorerPageUrlMigrationSpec,
-        catalogPathToIndicatorIdMap?: Record<string, number | null>
+        urlMigrationSpec?: ExplorerPageUrlMigrationSpec
     ) {
         const props: ExplorerProps = {
             ...program,
             grapherConfigs,
             partialGrapherConfigs,
-            catalogPathToIndicatorIdMap,
             isEmbeddedInAnOwidPage: false,
             isInStandalonePage: true,
         }
@@ -422,13 +419,7 @@ export class Explorer
         const { columnDefsWithoutTableSlug } = this.explorerProgram
         return keyBy(
             columnDefsWithoutTableSlug,
-            (def: OwidColumnDef) =>
-                def.owidVariableId ??
-                (def.catalogPath &&
-                    this.resolveMaybeCatalogPathToIndicatorId(
-                        def.catalogPath
-                    )) ??
-                def.slug
+            (def: OwidColumnDef) => def.owidVariableId ?? def.slug
         )
     }
 
@@ -495,23 +486,6 @@ export class Explorer
         grapher.downloadData()
     }
 
-    // The input to this function may be a catalog path (e.g. grapher/demography/2023-03-31/population/population#population)
-    // or an integer indicator id.
-    resolveMaybeCatalogPathToIndicatorId = (
-        catalogPathOrIndicatorId: string
-    ): number | undefined => {
-        const possiblyIndicatorId = parseIntOrUndefined(
-            catalogPathOrIndicatorId
-        )
-        if (possiblyIndicatorId !== undefined) return possiblyIndicatorId
-        else
-            return (
-                this.props.catalogPathToIndicatorIdMap?.[
-                    catalogPathOrIndicatorId
-                ] ?? undefined
-            )
-    }
-
     @action.bound private async updateGrapherFromExplorerUsingVariableIds() {
         const grapher = this.grapher
         if (!grapher) return
@@ -528,7 +502,7 @@ export class Explorer
 
         const yVariableIdsList = yVariableIds
             .split(" ")
-            .map(this.resolveMaybeCatalogPathToIndicatorId)
+            .map(parseIntOrUndefined)
             .filter((item) => item !== undefined)
 
         const partialGrapherConfig =
@@ -560,31 +534,22 @@ export class Explorer
             })
         })
         if (xVariableId) {
-            const resolvedXVariableId =
-                this.resolveMaybeCatalogPathToIndicatorId(xVariableId)
-            if (resolvedXVariableId)
-                dimensions.push({
-                    variableId: resolvedXVariableId,
-                    property: DimensionProperty.x,
-                })
+            dimensions.push({
+                variableId: xVariableId,
+                property: DimensionProperty.x,
+            })
         }
         if (colorVariableId) {
-            const resolvedColorVariableId =
-                this.resolveMaybeCatalogPathToIndicatorId(colorVariableId)
-            if (resolvedColorVariableId)
-                dimensions.push({
-                    variableId: resolvedColorVariableId,
-                    property: DimensionProperty.color,
-                })
+            dimensions.push({
+                variableId: colorVariableId,
+                property: DimensionProperty.color,
+            })
         }
         if (sizeVariableId) {
-            const resolvedSizeVariableId =
-                this.resolveMaybeCatalogPathToIndicatorId(sizeVariableId)
-            if (resolvedSizeVariableId)
-                dimensions.push({
-                    variableId: resolvedSizeVariableId,
-                    property: DimensionProperty.size,
-                })
+            dimensions.push({
+                variableId: sizeVariableId,
+                property: DimensionProperty.size,
+            })
         }
 
         // Slugs that are used to create a chart refer to columns derived from variables

explorer/ExplorerProgram.ts

@@ -54,9 +54,9 @@ interface ExplorerGrapherInterface extends GrapherInterface {
     grapherId?: number
     tableSlug?: string
     yVariableIds?: string
-    xVariableId?: string
-    colorVariableId?: string
-    sizeVariableId?: string
+    xVariableId?: number
+    colorVariableId?: number
+    sizeVariableId?: number
     yScaleToggle?: boolean
     yAxisMin?: number
     facetYDomain?: FacetAxisDomain
@@ -504,13 +504,14 @@ const parseColumnDefs = (block: string[][]): OwidColumnDef[] => {
             name: "owidVariableId",
             type: ColumnTypeNames.Integer,
         })
-        .replaceCells(
-            ["owidVariableId"],
-            (cell) =>
-                (typeof cell === "string"
-                    ? parseIntOrUndefined(cell)
-                    : undefined) ?? ErrorValueTypes.FilteredValue
-        )
+        .replaceCells(["owidVariableId"], (cell) => {
+            if (typeof cell === "number") return cell
+            if (typeof cell === "string")
+                return (
+                    parseIntOrUndefined(cell) ?? ErrorValueTypes.FilteredValue
+                )
+            return ErrorValueTypes.FilteredValue
+        })
 
         // Extract non-integer catalog paths from owidVariableIdOrCatalogPath
         .duplicateColumn("owidVariableIdOrCatalogPath", {

gridLang/GridLangConstants.ts

@@ -182,6 +182,14 @@ export const SlugsDeclarationCellDef: CellDef = {
     requirementsDescription: `Can only contain the characters a-zA-Z0-9-_ `,
 }
 
+export const EtlPathCellDef: CellDef = {
+    keyword: "IndicatorIdOrEtlPath",
+    cssClass: "EtlPath",
+    description: "Path to an ETL indicator.",
+    regex: /^$|^[\w\d_/\-]+(#[\w\d_/\-]+)?$/,
+    requirementsDescription: `Can only contain the characters a-zA-Z0-9-_/#`,
+}
+
 export const IndicatorIdOrEtlPathCellDef: CellDef = {
     keyword: "",
     cssClass: "IndicatorIdOrEtlPath",

packages/@ourworldindata/core-table/src/ErrorValues.ts

@@ -28,6 +28,7 @@ class FilteredValue extends ErrorValue {}
 class NoValueForInterpolation extends ErrorValue {}
 class InvalidQuarterValue extends ErrorValue {}
 class InvalidNegativeValue extends ErrorValue {}
+class NoMatchingVariableId extends ErrorValue {}
 
 // todo: if we don't export this, get an error in Transforms. should be fixable, see: https://github.com/microsoft/TypeScript/issues/5711
 export class MissingValuePlaceholder extends ErrorValue {}
@@ -53,6 +54,7 @@ export const ErrorValueTypes = {
     NoValueForInterpolation: new NoValueForInterpolation(),
     InvalidQuarterValue: new InvalidQuarterValue(),
     InvalidNegativeValue: new InvalidNegativeValue(),
+    NoMatchingVariableId: new NoMatchingVariableId(),
 }
 
 // https://github.com/robertmassaioli/ts-is-present