🔨 let charts inherit from a root config

adminSiteClient/GrapherConfigGridEditor.tsx

@@ -7,6 +7,7 @@ import {
     getWindowUrl,
     setWindowUrl,
     excludeNull,
+    mergeGrapherConfigs,
 } from "@ourworldindata/utils"
 import { GrapherConfigPatch } from "../adminShared/AdminSessionTypes.js"
 import {
@@ -348,7 +349,10 @@ export class GrapherConfigGridEditor extends React.Component<GrapherConfigGridEd
             selectedRowContent.id
         )
 
-        const mergedConfig = merge(grapherConfig, finalConfigLayer)
+        const mergedConfig = mergeGrapherConfigs(
+            grapherConfig,
+            finalConfigLayer
+        )
         this.loadGrapherJson(mergedConfig)
     }
 

adminSiteServer/apiRouter.ts

@@ -46,6 +46,7 @@ import {
     Json,
     checkIsGdocPostExcludingFragments,
     checkIsPlainObjectWithGuard,
+    mergeGrapherConfigs,
 } from "@ourworldindata/utils"
 import { applyPatch } from "../adminShared/patchHelper.js"
 import {
@@ -86,6 +87,7 @@ import {
     DbRawChartConfig,
 } from "@ourworldindata/types"
 import {
+    defaultGrapherConfig,
     getVariableDataRoute,
     getVariableMetadataRoute,
 } from "@ourworldindata/grapher"
@@ -284,6 +286,94 @@ const getBinaryUUID = async (
     return id
 }
 
+const saveNewChart = async (
+    knex: db.KnexReadWriteTransaction,
+    { config, user }: { config: GrapherInterface; user: DbPlainUser }
+): Promise<GrapherInterface> => {
+    // all charts inherit from the default config
+    const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, config)
+
+    // insert patch & full config into the chart_configs table
+    const configId = await getBinaryUUID(knex)
+    await db.knexRaw(
+        knex,
+        `-- sql
+            INSERT INTO chart_configs (id, patch, full)
+            VALUES (?, ?, ?)
+        `,
+        [configId, JSON.stringify(config), JSON.stringify(fullConfig)]
+    )
+
+    // add a new chart to the charts table
+    const result = await db.knexRawInsert(
+        knex,
+        `-- sql
+            INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId)
+            VALUES (?, ?, ?)
+        `,
+        [configId, new Date(), user.id]
+    )
+
+    // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
+    const chartId = result.insertId
+    config.id = chartId
+    fullConfig.id = chartId
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE chart_configs cc
+            JOIN charts c ON c.configId = cc.id
+            SET cc.patch=?, cc.full=?
+            WHERE c.id = ?
+        `,
+        [JSON.stringify(config), JSON.stringify(fullConfig), chartId]
+    )
+
+    return config
+}
+
+const updateExistingChart = async (
+    knex: db.KnexReadWriteTransaction,
+    {
+        config,
+        user,
+        chartId,
+    }: { config: GrapherInterface; user: DbPlainUser; chartId: number }
+): Promise<GrapherInterface> => {
+    // make sure that the id of the incoming config matches the chart id
+    config.id = chartId
+
+    // all charts inherit from the default config
+    const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, config)
+
+    // update configs
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE chart_configs cc
+            JOIN charts c ON c.configId = cc.id
+            SET
+                cc.patch=?,
+                cc.full=?
+            WHERE c.id = ?
+        `,
+        [JSON.stringify(config), JSON.stringify(fullConfig), chartId]
+    )
+
+    // update charts row
+    await db.knexRaw(
+        knex,
+        `-- sql
+            UPDATE charts
+            SET lastEditedAt=?, lastEditedByUserId=?
+            WHERE id = ?
+        `,
+        [new Date(), user.id, chartId]
+    )
+
+    return config
+}
+
 const saveGrapher = async (
     knex: db.KnexReadWriteTransaction,
     user: DbPlainUser,
@@ -364,62 +454,17 @@ const saveGrapher = async (
     else newConfig.version = 1
 
     // Execute the actual database update or creation
-    const now = new Date()
-    let chartId = existingConfig && existingConfig.id
-    const newJsonConfig = JSON.stringify(newConfig)
+    let chartId: number
     if (existingConfig) {
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE chart_configs cc
-                JOIN charts c ON c.configId = cc.id
-                SET
-                    cc.patch=?,
-                    cc.full=?
-                WHERE c.id = ?
-            `,
-            [newJsonConfig, newJsonConfig, chartId]
-        )
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE charts
-                SET lastEditedAt=?, lastEditedByUserId=?
-                WHERE id = ?
-            `,
-            [now, user.id, chartId]
-        )
+        chartId = existingConfig.id!
+        newConfig = await updateExistingChart(knex, {
+            config: newConfig,
+            user,
+            chartId,
+        })
     } else {
-        const configId = await getBinaryUUID(knex)
-        await db.knexRaw(
-            knex,
-            `-- sql
-                INSERT INTO chart_configs (id, patch, full)
-                VALUES (?, ?, ?)
-            `,
-            [configId, newJsonConfig, newJsonConfig]
-        )
-        const result = await db.knexRawInsert(
-            knex,
-            `-- sql
-                INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId)
-                VALUES (?, ?, ?)
-            `,
-            [configId, now, user.id]
-        )
-        chartId = result.insertId
-        // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true
-        newConfig.id = chartId
-        await db.knexRaw(
-            knex,
-            `-- sql
-                UPDATE chart_configs cc
-                JOIN charts c ON c.configId = cc.id
-                SET cc.patch=?, cc.full=?
-                WHERE c.id = ?
-            `,
-            [JSON.stringify(newConfig), JSON.stringify(newConfig), chartId]
-        )
+        newConfig = await saveNewChart(knex, { config: newConfig, user })
+        chartId = newConfig.id!
     }
 
     // Record this change in version history
@@ -474,7 +519,7 @@ const saveGrapher = async (
         await db.knexRaw(
             knex,
             `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `,
-            [now, user.id, chartId]
+            [new Date(), user.id, chartId]
         )
         await triggerStaticBuild(user, `Publishing chart ${newConfig.slug}`)
     } else if (

baker/GrapherBaker.tsx

@@ -9,7 +9,7 @@ import {
     deserializeJSONFromHTML,
     uniq,
     keyBy,
-    mergePartialGrapherConfigs,
+    mergeGrapherConfigs,
     compact,
     partition,
 } from "@ourworldindata/utils"
@@ -158,7 +158,7 @@ export async function renderDataPageV2(
     // if we are on Grapher pages. Once we have a good way in the grapher admin for how
     // to use indicator level defaults, we should reconsider how this works here.
     const grapher = useIndicatorGrapherConfigs
-        ? mergePartialGrapherConfigs(grapherConfigForVariable, pageGrapher)
+        ? mergeGrapherConfigs(grapherConfigForVariable ?? {}, pageGrapher ?? {})
         : pageGrapher ?? {}
 
     const faqDocs = compact(

baker/siteRenderers.tsx

@@ -43,7 +43,7 @@ import {
     JsonError,
     Url,
     IndexPost,
-    mergePartialGrapherConfigs,
+    mergeGrapherConfigs,
     OwidGdocType,
     OwidGdoc,
     OwidGdocDataInsightInterface,
@@ -791,7 +791,7 @@ export const renderExplorerPage = async (
                       config: row.grapherConfigETL as string,
                   })
                 : {}
-            return mergePartialGrapherConfigs(etlConfig, adminConfig)
+            return mergeGrapherConfigs(etlConfig, adminConfig)
         })
 
     const wpContent = transformedProgram.wpBlockId

db/migration/1720600092980-MakeChartsInheritDefaults.ts

@@ -0,0 +1,42 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+import { mergeGrapherConfigs } from "@ourworldindata/utils"
+import { defaultGrapherConfig } from "@ourworldindata/grapher"
+
+export class MakeChartsInheritDefaults1720600092980
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const charts = (await queryRunner.query(
+            `-- sql
+                SELECT id, uuid, patch as config FROM chart_configs
+            `
+        )) as { id: string; uuid: string; config: string }[]
+
+        for (const chart of charts) {
+            const originalConfig = JSON.parse(chart.config)
+
+            const fullConfig = mergeGrapherConfigs(
+                defaultGrapherConfig,
+                originalConfig
+            )
+
+            await queryRunner.query(
+                `-- sql
+                    UPDATE chart_configs
+                    SET full = ?
+                    WHERE id = ?
+                `,
+                [JSON.stringify(fullConfig), chart.id]
+            )
+        }
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `-- sql
+                UPDATE chart_configs
+                SET full = patch
+            `
+        )
+    }
+}

db/model/Variable.ts

@@ -1,7 +1,11 @@
 import _ from "lodash"
 import { Writable } from "stream"
 import * as db from "../db.js"
-import { retryPromise, isEmpty } from "@ourworldindata/utils"
+import {
+    retryPromise,
+    isEmpty,
+    mergeGrapherConfigs,
+} from "@ourworldindata/utils"
 import {
     getVariableDataRoute,
     getVariableMetadataRoute,
@@ -45,7 +49,7 @@ export async function getMergedGrapherConfigForVariable(
     const grapherConfigETL = row.grapherConfigETL
         ? JSON.parse(row.grapherConfigETL)
         : undefined
-    return _.merge({}, grapherConfigAdmin, grapherConfigETL)
+    return mergeGrapherConfigs(grapherConfigETL, grapherConfigAdmin)
 }
 
 // TODO: these are domain functions and should live somewhere else

devTools/schema/generate-default-object-from-schema.ts

@@ -44,15 +44,31 @@ async function main(parsedArgs: parseArgs.ParsedArgs) {
 
     let schema = fs.readJSONSync(schemaFilename)
     const defs = schema.$defs || {}
-    const defaultObject = generateDefaultObjectFromSchema(schema, defs)
-    process.stdout.write(JSON.stringify(defaultObject, undefined, 2))
+    const defaultConfig = generateDefaultObjectFromSchema(schema, defs)
+    const defaultConfigJSON = JSON.stringify(defaultConfig, undefined, 2)
+
+    // save as js file if requested
+    if (parsedArgs["save-ts"]) {
+        const out = parsedArgs["save-ts"]
+        const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
+
+// GENERATED BY devTools/schema/generate-default-object-from-schema.ts
+
+import { GrapherInterface } from "@ourworldindata/types"
+
+export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface`
+        fs.outputFileSync(out, content)
+    }
+
+    // write json to stdout
+    process.stdout.write(defaultConfigJSON)
 }
 
 function help() {
     console.log(`generate-default-object-from-schema.ts - utility to generate an object with all default values that are given in a JSON schema
 
 Usage:
-  generate-default-object-from-schema.js <schema.json>`)
+  generate-default-object-from-schema.js --save-ts <out.js> <schema.json>`)
 }
 
 const parsedArgs = parseArgs(process.argv.slice(2))

explorer/GrapherGrammar.ts

@@ -224,8 +224,7 @@ export const GrapherGrammar: Grammar = {
     hideRelativeToggle: {
         ...BooleanCellDef,
         keyword: "hideRelativeToggle",
-        description:
-            "Whether to hide the relative mode UI toggle. Default depends on the chart type.",
+        description: "Whether to hide the relative mode UI toggle",
     },
     timelineMinTime: {
         ...IntegerCellDef,

packages/@ourworldindata/grapher/src/index.ts

@@ -79,3 +79,4 @@ export {
     type SlideShowManager,
     SlideShowController,
 } from "./slideshowController/SlideShowController"
+export { defaultGrapherConfig } from "./schema/defaultGrapherConfig"

packages/@ourworldindata/grapher/src/schema/.gitignore

@@ -0,0 +1 @@
+grapher-schema.*.json

packages/@ourworldindata/grapher/src/schema/README.md

@@ -6,13 +6,26 @@ The schema is versioned. There is one yaml file with a version number. For nonbr
 edit the yaml file as is. A github action will then generate a .latest.yaml and two json files (one .latest.json and one with the version number.json)
 and upload them to S3 so they can be accessed publicly.
 
+If you update a default value in the schema, make sure to regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below).
+
 Breaking changes should be done by renaming the schema file to an increased version number. Make sure to also rename the authorative url
 inside the schema file (the "$id" field at the top level) to point to the new version number json. Then write the migrations from the last to
-the current version of the schema, including the migration of pointing to the URL of the new schema version. Also update `DEFAULT_GRAPHER_CONFIG_SCHEMA` in `GrapherConstants.ts` to point to the new schema version number url.
+the current version of the schema, including the migration of pointing to the URL of the new schema version. Also update `DEFAULT_GRAPHER_CONFIG_SCHEMA` to point to the new schema version number url.
 
 Checklist for breaking changes:
 
 -   Rename the schema file to an increased version number
 -   Rename the authorative url inside the schema file to point to the new version number json
 -   Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version
 -   Update `DEFAULT_GRAPHER_CONFIG_SCHEMA` in `GrapherConstants.ts` to point to the new schema version number url
+-   Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below)
+
+To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run:
+
+```bash
+# generate json from the yaml schema
+nu -c 'open packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.yaml | to json' > packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.json
+
+# generate the default object from the schema
+ node itsJustJavascript/devTools/schema/generate-default-object-from-schema.js packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.json--save-ts packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts
+```