🔨 (grapher) let charts inherit defaults

adminSiteClient/GrapherConfigGridEditor.tsx

@@ -348,6 +348,7 @@ export class GrapherConfigGridEditor extends React.Component<GrapherConfigGridEd
             selectedRowContent.id
         )
 
+        // TODO(inheritance): use mergeGrapherConfigs instead
         const mergedConfig = merge(grapherConfig, finalConfigLayer)
         this.loadGrapherJson(mergedConfig)
     }

adminSiteServer/apiRouter.ts

@@ -45,6 +45,8 @@ import {
     Json,
     checkIsGdocPostExcludingFragments,
     checkIsPlainObjectWithGuard,
+    mergeGrapherConfigs,
+    diffGrapherConfigs,
 } from "@ourworldindata/utils"
 import { applyPatch } from "../adminShared/patchHelper.js"
 import {
@@ -85,6 +87,7 @@ import {
     DbRawChartConfig,
 } from "@ourworldindata/types"
 import {
+    defaultGrapherConfig,
     getVariableDataRoute,
     getVariableMetadataRoute,
 } from "@ourworldindata/grapher"
@@ -278,6 +281,112 @@ const getBinaryUUID = async (
     return id
 }
 
+const saveNewChart = async (
+    knex: db.KnexReadWriteTransaction,
+    { config, user }: { config: GrapherInterface; user: DbPlainUser }
+): Promise<GrapherInterface> => {
+    // if the schema version is missing, assume it's the latest
+    if (!config["$schema"]) {
+        config["$schema"] = defaultGrapherConfig["$schema"]
+    }
+
+    // store minimal config as patch
+    const patchConfig = diffGrapherConfigs(config, defaultGrapherConfig)
+
+    // all charts inherit from the default config
+    const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, patchConfig)
+
+    // insert patch & full configs into the chart_configs table
+    const configId = await getBinaryUUID(knex)
+    await db.knexRaw(
+        knex,
+        `-- sql
+            INSERT INTO chart_configs (id, patch, full)
+            VALUES (?, ?, ?)
+        `,
+        [configId, JSON.stringify(patchConfig), JSON.stringify(fullConfig)]
+    )
+
+    // add a new chart to the charts table
+    const result = await db.knexRawInsert(
+        knex,
+        `-- sql
+            INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId)
+            VALUES (?, ?, ?)
+        `,
+        [configId, new Date(), user.id]
+    )
+
+    // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
+    const chartId = result.insertId
+    patchConfig.id = chartId
+    fullConfig.id = chartId
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE chart_configs cc
+            JOIN charts c ON c.configId = cc.id
+            SET
+                cc.patch=JSON_SET(cc.patch, '$.id', ?),
+                cc.full=JSON_SET(cc.full, '$.id', ?)
+            WHERE c.id = ?
+        `,
+        [chartId, chartId, chartId]
+    )
+
+    return patchConfig
+}
+
+const updateExistingChart = async (
+    knex: db.KnexReadWriteTransaction,
+    {
+        config,
+        user,
+        chartId,
+    }: { config: GrapherInterface; user: DbPlainUser; chartId: number }
+): Promise<GrapherInterface> => {
+    // make sure that the id of the incoming config matches the chart id
+    config.id = chartId
+
+    // if the schema version is missing, assume it's the latest
+    if (!config["$schema"]) {
+        config["$schema"] = defaultGrapherConfig["$schema"]
+    }
+
+    // store minimal config as patch
+    const patchConfig = diffGrapherConfigs(config, defaultGrapherConfig)
+
+    // all charts inherit from the default config
+    const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, patchConfig)
+
+    // update configs
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE chart_configs cc
+            JOIN charts c ON c.configId = cc.id
+            SET
+                cc.patch=?,
+                cc.full=?
+            WHERE c.id = ?
+        `,
+        [JSON.stringify(patchConfig), JSON.stringify(fullConfig), chartId]
+    )
+
+    // update charts row
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE charts
+            SET lastEditedAt=?, lastEditedByUserId=?
+            WHERE id = ?
+        `,
+        [new Date(), user.id, chartId]
+    )
+
+    return patchConfig
+}
+
 const saveGrapher = async (
     knex: db.KnexReadWriteTransaction,
     user: DbPlainUser,
@@ -358,64 +467,17 @@ const saveGrapher = async (
     else newConfig.version = 1
 
     // Execute the actual database update or creation
-    const now = new Date()
-    let chartId = existingConfig && existingConfig.id
-    const newJsonConfig = JSON.stringify(newConfig)
+    let chartId: number
     if (existingConfig) {
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE chart_configs cc
-                JOIN charts c ON c.configId = cc.id
-                SET
-                    cc.patch=?,
-                    cc.full=?
-                WHERE c.id = ?
-            `,
-            [newJsonConfig, newJsonConfig, chartId]
-        )
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE charts
-                SET lastEditedAt=?, lastEditedByUserId=?
-                WHERE id = ?
-            `,
-            [now, user.id, chartId]
-        )
+        chartId = existingConfig.id!
+        newConfig = await updateExistingChart(knex, {
+            config: newConfig,
+            user,
+            chartId,
+        })
     } else {
-        const configId = await getBinaryUUID(knex)
-        await db.knexRaw(
-            knex,
-            `-- sql
-                INSERT INTO chart_configs (id, patch, full)
-                VALUES (?, ?, ?)
-            `,
-            [configId, newJsonConfig, newJsonConfig]
-        )
-        const result = await db.knexRawInsert(
-            knex,
-            `-- sql
-                INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId)
-                VALUES (?, ?, ?)
-            `,
-            [configId, now, user.id]
-        )
-        chartId = result.insertId
-        // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
-        newConfig.id = chartId
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE chart_configs cc
-                JOIN charts c ON c.configId = cc.id
-                SET
-                    cc.patch=JSON_SET(cc.patch, '$.id', ?),
-                    cc.full=JSON_SET(cc.full, '$.id', ?)
-                WHERE c.id = ?
-            `,
-            [chartId, chartId, chartId]
-        )
+        newConfig = await saveNewChart(knex, { config: newConfig, user })
+        chartId = newConfig.id!
     }
 
     // Record this change in version history
@@ -470,7 +532,7 @@ const saveGrapher = async (
         await db.knexRaw(
             knex,
             `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `,
-            [now, user.id, chartId]
+            [new Date(), user.id, chartId]
         )
         await triggerStaticBuild(user, `Publishing chart ${newConfig.slug}`)
     } else if (

baker/GrapherBaker.tsx

@@ -157,6 +157,7 @@ export async function renderDataPageV2(
     // this is true for preview pages for datapages on the indicator level but false
     // if we are on Grapher pages. Once we have a good way in the grapher admin for how
     // to use indicator level defaults, we should reconsider how this works here.
+    // TODO(inheritance): use mergeGrapherConfigs instead
     const grapher = useIndicatorGrapherConfigs
         ? mergePartialGrapherConfigs(grapherConfigForVariable, pageGrapher)
         : pageGrapher ?? {}

baker/siteRenderers.tsx

@@ -791,6 +791,7 @@ export const renderExplorerPage = async (
                       config: row.grapherConfigETL as string,
                   })
                 : {}
+            // TODO(inheritance): use mergeGrapherConfigs instead
             return mergePartialGrapherConfigs(etlConfig, adminConfig)
         })
 

db/migration/1720600092980-MakeChartsInheritDefaults.ts

@@ -0,0 +1,59 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils"
+import { defaultGrapherConfig } from "@ourworldindata/grapher"
+
+export class MakeChartsInheritDefaults1720600092980
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const charts = (await queryRunner.query(
+            `-- sql
+                SELECT id, patch as config FROM chart_configs
+            `
+        )) as { id: string; config: string }[]
+
+        for (const chart of charts) {
+            const originalConfig = JSON.parse(chart.config)
+
+            // if the schema version is missing, assume it's the latest
+            if (!originalConfig["$schema"]) {
+                originalConfig["$schema"] = defaultGrapherConfig["$schema"]
+            }
+
+            const patchConfig = diffGrapherConfigs(
+                originalConfig,
+                defaultGrapherConfig
+            )
+            const fullConfig = mergeGrapherConfigs(
+                defaultGrapherConfig,
+                patchConfig
+            )
+
+            await queryRunner.query(
+                `-- sql
+                    UPDATE chart_configs
+                    SET
+                        patch = ?,
+                        full = ?
+                    WHERE id = ?
+                `,
+                [
+                    JSON.stringify(patchConfig),
+                    JSON.stringify(fullConfig),
+                    chart.id,
+                ]
+            )
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // we can't recover the original configs,
+        // but the patched one is the next best thing
+        await queryRunner.query(
+            `-- sql
+                UPDATE chart_configs
+                SET full = patch
+            `
+        )
+    }
+}

db/model/Variable.ts

@@ -45,6 +45,7 @@ export async function getMergedGrapherConfigForVariable(
     const grapherConfigETL = row.grapherConfigETL
         ? JSON.parse(row.grapherConfigETL)
         : undefined
+    // TODO(inheritance): use mergeGrapherConfigs instead
     return _.merge({}, grapherConfigAdmin, grapherConfigETL)
 }
 

devTools/schema/generate-default-object-from-schema.ts

@@ -44,15 +44,31 @@ async function main(parsedArgs: parseArgs.ParsedArgs) {
 
     let schema = fs.readJSONSync(schemaFilename)
     const defs = schema.$defs || {}
-    const defaultObject = generateDefaultObjectFromSchema(schema, defs)
-    process.stdout.write(JSON.stringify(defaultObject, undefined, 2))
+    const defaultConfig = generateDefaultObjectFromSchema(schema, defs)
+    const defaultConfigJSON = JSON.stringify(defaultConfig, undefined, 2)
+
+    // save as ts file if requested
+    if (parsedArgs["save-ts"]) {
+        const out = parsedArgs["save-ts"]
+        const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
+
+// GENERATED BY devTools/schema/generate-default-object-from-schema.ts
+
+import { GrapherInterface } from "@ourworldindata/types"
+
+export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface`
+        fs.outputFileSync(out, content)
+    }
+
+    // write json to stdout
+    process.stdout.write(defaultConfigJSON)
 }
 
 function help() {
     console.log(`generate-default-object-from-schema.ts - utility to generate an object with all default values that are given in a JSON schema
 
 Usage:
-  generate-default-object-from-schema.js <schema.json>`)
+  generate-default-object-from-schema.js --save-ts <out.ts> <schema.json>`)
 }
 
 const parsedArgs = parseArgs(process.argv.slice(2))

explorer/GrapherGrammar.ts

@@ -224,8 +224,7 @@ export const GrapherGrammar: Grammar = {
     hideRelativeToggle: {
         ...BooleanCellDef,
         keyword: "hideRelativeToggle",
-        description:
-            "Whether to hide the relative mode UI toggle. Default depends on the chart type.",
+        description: "Whether to hide the relative mode UI toggle",
     },
     timelineMinTime: {
         ...IntegerCellDef,

packages/@ourworldindata/grapher/src/index.ts

@@ -79,3 +79,4 @@ export {
     type SlideShowManager,
     SlideShowController,
 } from "./slideshowController/SlideShowController"
+export { defaultGrapherConfig } from "./schema/defaultGrapherConfig"

packages/@ourworldindata/grapher/src/schema/.gitignore

@@ -0,0 +1 @@
+grapher-schema.*.json

packages/@ourworldindata/grapher/src/schema/README.md

@@ -6,6 +6,8 @@ The schema is versioned. There is one yaml file with a version number. For nonbr
 edit the yaml file as is. A github action will then generate a .latest.yaml and two json files (one .latest.json and one with the version number.json)
 and upload them to S3 so they can be accessed publicly.
 
+If you add a new property with a default value or if you update the default value of an existing property, make sure to regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below). You should also write a migration to update the `chart_configs.full` column in the database.
+
 Breaking changes should be done by renaming the schema file to an increased version number. Make sure to also rename the authorative url
 inside the schema file (the "$id" field at the top level) to point to the new version number json. Then write the migrations from the last to
 the current version of the schema, including the migration of pointing to the URL of the new schema version. Also update `DEFAULT_GRAPHER_CONFIG_SCHEMA` in `GrapherConstants.ts` to point to the new schema version number url.
@@ -16,3 +18,14 @@ Checklist for breaking changes:
 -   Rename the authorative url inside the schema file to point to the new version number json
 -   Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version
 -   Update `DEFAULT_GRAPHER_CONFIG_SCHEMA` in `GrapherConstants.ts` to point to the new schema version number url
+-   Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below)
+
+To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run:
+
+```bash
+# generate json from the yaml schema
+nu -c 'open packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.yaml | to json' > packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.json
+
+# generate the default object from the schema
+node itsJustJavascript/devTools/schema/generate-default-object-from-schema.js packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.json --save-ts packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts
+```