🔨 (grapher) let charts inherit defaults

adminSiteClient/ChartEditorPage.tsx

@@ -20,6 +20,7 @@ import {
     groupBy,
     extractDetailsFromSyntax,
     getIndexableKeys,
+    diffGrapherConfigs,
 } from "@ourworldindata/utils"
 import {
     Topic,
@@ -104,7 +105,7 @@ export class ChartEditorPage
     extends React.Component<{
         grapherId?: number
         newGrapherIndex?: number
-        grapherConfig?: any
+        grapherConfig?: GrapherInterface
     }>
     implements ChartEditorManager
 {
@@ -124,7 +125,8 @@ export class ChartEditorPage
 
     @observable simulateVisionDeficiency?: VisionDeficiency
 
-    fetchedGrapherConfig?: any
+    fetchedGrapherConfig?: GrapherInterface
+    baseGrapherConfig?: GrapherInterface
 
     async fetchGrapher(): Promise<void> {
         const { grapherId } = this.props
@@ -143,7 +145,13 @@ export class ChartEditorPage
     }
 
     @action.bound private updateGrapher(): void {
-        const config = this.fetchedGrapherConfig ?? this.props.grapherConfig
+        let config = this.fetchedGrapherConfig ?? this.props.grapherConfig
+
+        // if there is a base layer, update the patch instead of the full config
+        if (config && this.baseGrapherConfig) {
+            config = diffGrapherConfigs(config, this.baseGrapherConfig)
+        }
+
         const grapherConfig = {
             ...config,
             // binds the grapher instance to this.grapher
@@ -165,6 +173,16 @@ export class ChartEditorPage
         this._isDbSet = true
     }
 
+    async fetchBaseGrapher(): Promise<void> {
+        const { admin } = this.context
+        const { grapherId } = this.props
+        if (grapherId !== undefined) {
+            this.baseGrapherConfig = await admin.getJSON(
+                `/api/charts/${grapherId}.base.json`
+            )
+        }
+    }
+
     async fetchData(): Promise<void> {
         const { admin } = this.context
 
@@ -362,6 +380,7 @@ export class ChartEditorPage
 
     @action.bound refresh(): void {
         void this.fetchGrapher()
+        void this.fetchBaseGrapher()
         void this.fetchDetails()
         void this.fetchData()
         void this.fetchLogs()

adminSiteClient/GrapherConfigGridEditor.tsx

@@ -348,6 +348,7 @@ export class GrapherConfigGridEditor extends React.Component<GrapherConfigGridEd
             selectedRowContent.id
         )
 
+        // TODO(inheritance): use mergeGrapherConfigs instead
         const mergedConfig = merge(grapherConfig, finalConfigLayer)
         this.loadGrapherJson(mergedConfig)
     }

adminSiteServer/apiRouter.ts

@@ -45,6 +45,8 @@ import {
     Json,
     checkIsGdocPostExcludingFragments,
     checkIsPlainObjectWithGuard,
+    mergeGrapherConfigs,
+    diffGrapherConfigs,
 } from "@ourworldindata/utils"
 import { applyPatch } from "../adminShared/patchHelper.js"
 import {
@@ -85,6 +87,7 @@ import {
     DbRawChartConfig,
 } from "@ourworldindata/types"
 import {
+    defaultGrapherConfig,
     getVariableDataRoute,
     getVariableMetadataRoute,
 } from "@ourworldindata/grapher"
@@ -278,6 +281,110 @@ const getBinaryUUID = async (
     return id
 }
 
+const saveNewChart = async (
+    knex: db.KnexReadWriteTransaction,
+    { config, user }: { config: GrapherInterface; user: DbPlainUser }
+): Promise<GrapherInterface> => {
+    // if the schema version is missing, assume it's the latest
+    if (!config["$schema"]) {
+        config["$schema"] = defaultGrapherConfig["$schema"]
+    }
+
+    // compute patch and full configs
+    const baseConfig = getBaseLayerConfig()
+    const patchConfig = diffGrapherConfigs(config, baseConfig)
+    const fullConfig = mergeGrapherConfigs(baseConfig, patchConfig)
+
+    // insert patch & full configs into the chart_configs table
+    const configId = await getBinaryUUID(knex)
+    await db.knexRaw(
+        knex,
+        `-- sql
+            INSERT INTO chart_configs (id, patch, full)
+            VALUES (?, ?, ?)
+        `,
+        [configId, JSON.stringify(patchConfig), JSON.stringify(fullConfig)]
+    )
+
+    // add a new chart to the charts table
+    const result = await db.knexRawInsert(
+        knex,
+        `-- sql
+            INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId)
+            VALUES (?, ?, ?)
+        `,
+        [configId, new Date(), user.id]
+    )
+
+    // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
+    const chartId = result.insertId
+    patchConfig.id = chartId
+    fullConfig.id = chartId
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE chart_configs cc
+            JOIN charts c ON c.configId = cc.id
+            SET
+                cc.patch=JSON_SET(cc.patch, '$.id', ?),
+                cc.full=JSON_SET(cc.full, '$.id', ?)
+            WHERE c.id = ?
+        `,
+        [chartId, chartId, chartId]
+    )
+
+    return patchConfig
+}
+
+const updateExistingChart = async (
+    knex: db.KnexReadWriteTransaction,
+    {
+        config,
+        user,
+        chartId,
+    }: { config: GrapherInterface; user: DbPlainUser; chartId: number }
+): Promise<GrapherInterface> => {
+    // make sure that the id of the incoming config matches the chart id
+    config.id = chartId
+
+    // if the schema version is missing, assume it's the latest
+    if (!config["$schema"]) {
+        config["$schema"] = defaultGrapherConfig["$schema"]
+    }
+
+    // compute patch and full configs
+    const baseConfig = getBaseLayerConfig()
+    const patchConfig = diffGrapherConfigs(config, baseConfig)
+    const fullConfig = mergeGrapherConfigs(baseConfig, patchConfig)
+
+    // update configs
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE chart_configs cc
+            JOIN charts c ON c.configId = cc.id
+            SET
+                cc.patch=?,
+                cc.full=?
+            WHERE c.id = ?
+        `,
+        [JSON.stringify(patchConfig), JSON.stringify(fullConfig), chartId]
+    )
+
+    // update charts row
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE charts
+            SET lastEditedAt=?, lastEditedByUserId=?
+            WHERE id = ?
+        `,
+        [new Date(), user.id, chartId]
+    )
+
+    return patchConfig
+}
+
 const saveGrapher = async (
     knex: db.KnexReadWriteTransaction,
     user: DbPlainUser,
@@ -358,64 +465,17 @@ const saveGrapher = async (
     else newConfig.version = 1
 
     // Execute the actual database update or creation
-    const now = new Date()
-    let chartId = existingConfig && existingConfig.id
-    const newJsonConfig = JSON.stringify(newConfig)
+    let chartId: number
     if (existingConfig) {
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE chart_configs cc
-                JOIN charts c ON c.configId = cc.id
-                SET
-                    cc.patch=?,
-                    cc.full=?
-                WHERE c.id = ?
-            `,
-            [newJsonConfig, newJsonConfig, chartId]
-        )
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE charts
-                SET lastEditedAt=?, lastEditedByUserId=?
-                WHERE id = ?
-            `,
-            [now, user.id, chartId]
-        )
+        chartId = existingConfig.id!
+        newConfig = await updateExistingChart(knex, {
+            config: newConfig,
+            user,
+            chartId,
+        })
     } else {
-        const configId = await getBinaryUUID(knex)
-        await db.knexRaw(
-            knex,
-            `-- sql
-                INSERT INTO chart_configs (id, patch, full)
-                VALUES (?, ?, ?)
-            `,
-            [configId, newJsonConfig, newJsonConfig]
-        )
-        const result = await db.knexRawInsert(
-            knex,
-            `-- sql
-                INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId)
-                VALUES (?, ?, ?)
-            `,
-            [configId, now, user.id]
-        )
-        chartId = result.insertId
-        // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
-        newConfig.id = chartId
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE chart_configs cc
-                JOIN charts c ON c.configId = cc.id
-                SET
-                    cc.patch=JSON_SET(cc.patch, '$.id', ?),
-                    cc.full=JSON_SET(cc.full, '$.id', ?)
-                WHERE c.id = ?
-            `,
-            [chartId, chartId, chartId]
-        )
+        newConfig = await saveNewChart(knex, { config: newConfig, user })
+        chartId = newConfig.id!
     }
 
     // Record this change in version history
@@ -470,7 +530,7 @@ const saveGrapher = async (
         await db.knexRaw(
             knex,
             `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `,
-            [now, user.id, chartId]
+            [new Date(), user.id, chartId]
         )
         await triggerStaticBuild(user, `Publishing chart ${newConfig.slug}`)
     } else if (
@@ -572,6 +632,16 @@ getRouteWithROTransaction(
     async (req, res, trx) => expectChartById(trx, req.params.chartId)
 )
 
+function getBaseLayerConfig(): GrapherInterface {
+    return defaultGrapherConfig
+}
+
+getRouteWithROTransaction(
+    apiRouter,
+    "/charts/:chartId.base.json",
+    async (req, res, trx) => getBaseLayerConfig()
+)
+
 getRouteWithROTransaction(
     apiRouter,
     "/editorData/namespaces.json",

baker/GrapherBaker.tsx

@@ -157,6 +157,7 @@ export async function renderDataPageV2(
     // this is true for preview pages for datapages on the indicator level but false
     // if we are on Grapher pages. Once we have a good way in the grapher admin for how
     // to use indicator level defaults, we should reconsider how this works here.
+    // TODO(inheritance): use mergeGrapherConfigs instead
     const grapher = useIndicatorGrapherConfigs
         ? mergePartialGrapherConfigs(grapherConfigForVariable, pageGrapher)
         : pageGrapher ?? {}

baker/siteRenderers.tsx

@@ -791,6 +791,7 @@ export const renderExplorerPage = async (
                       config: row.grapherConfigETL as string,
                   })
                 : {}
+            // TODO(inheritance): use mergeGrapherConfigs instead
             return mergePartialGrapherConfigs(etlConfig, adminConfig)
         })
 

db/migration/1720600092980-MakeChartsInheritDefaults.ts

@@ -0,0 +1,59 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils"
+import { defaultGrapherConfig } from "@ourworldindata/grapher"
+
+export class MakeChartsInheritDefaults1720600092980
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const charts = (await queryRunner.query(
+            `-- sql
+                SELECT id, patch as config FROM chart_configs
+            `
+        )) as { id: string; config: string }[]
+
+        for (const chart of charts) {
+            const originalConfig = JSON.parse(chart.config)
+
+            // if the schema version is missing, assume it's the latest
+            if (!originalConfig["$schema"]) {
+                originalConfig["$schema"] = defaultGrapherConfig["$schema"]
+            }
+
+            const patchConfig = diffGrapherConfigs(
+                originalConfig,
+                defaultGrapherConfig
+            )
+            const fullConfig = mergeGrapherConfigs(
+                defaultGrapherConfig,
+                patchConfig
+            )
+
+            await queryRunner.query(
+                `-- sql
+                    UPDATE chart_configs
+                    SET
+                        patch = ?,
+                        full = ?
+                    WHERE id = ?
+                `,
+                [
+                    JSON.stringify(patchConfig),
+                    JSON.stringify(fullConfig),
+                    chart.id,
+                ]
+            )
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // we can't recover the original configs,
+        // but the patched one is the next best thing
+        await queryRunner.query(
+            `-- sql
+                UPDATE chart_configs
+                SET full = patch
+            `
+        )
+    }
+}

db/model/Variable.ts

@@ -45,6 +45,7 @@ export async function getMergedGrapherConfigForVariable(
     const grapherConfigETL = row.grapherConfigETL
         ? JSON.parse(row.grapherConfigETL)
         : undefined
+    // TODO(inheritance): use mergeGrapherConfigs instead
     return _.merge({}, grapherConfigAdmin, grapherConfigETL)
 }