🎉 (grapher) make charts inherit indicator-level settings

adminSiteServer/apiRouter.ts

@@ -85,6 +85,7 @@ import {
     DbInsertUser,
     FlatTagGraph,
     DbRawChartConfig,
+    parseChartConfig,
 } from "@ourworldindata/types"
 import {
     defaultGrapherConfig,
@@ -1290,6 +1291,121 @@ getRouteWithROTransaction(
     }
 )
 
+postRouteWithRWTransaction(
+    apiRouter,
+    "/variables/:variableId/grapherConfigETL/new",
+    async (req, res, trx) => {
+        const variableId = expectInt(req.params.variableId)
+        // const user = res.locals.user
+        const config = req.body
+
+        // if no schema is given, assume it's the latest
+        if (!config.$schema) {
+            config.$schema = defaultGrapherConfig.$schema
+        }
+
+        // check if the given dimensions are correct
+        if (config.dimensions && config.dimensions.length >= 1) {
+            // make sure there is only a single entry
+            config.dimensions = config.dimensions.slice(0, 1)
+            // make sure the variable id matches
+            config.dimensions[0].variableId = variableId
+        }
+
+        // fill dimensions if not given to make the config plottable
+        if (!config.dimensions || config.dimensions.length === 0) {
+            config.dimensions = [{ property: DimensionProperty.y, variableId }]
+        }
+
+        // ETL configs inherit from the default
+        const patchConfigETL = diffGrapherConfigs(config, defaultGrapherConfig)
+        const fullConfigETL = mergeGrapherConfigs(
+            defaultGrapherConfig,
+            patchConfigETL
+        )
+
+        // insert chart config into the database
+        const configId = await getBinaryUUID(trx)
+        await db.knexRaw(
+            trx,
+            `-- sql
+                INSERT INTO chart_configs (id, patch, full)
+                VALUES (?, ?, ?)
+            `,
+            [
+                configId,
+                JSON.stringify(patchConfigETL),
+                JSON.stringify(fullConfigETL),
+            ]
+        )
+
+        // make a reference to the config from the variables table
+        await db.knexRaw(
+            trx,
+            `-- sql
+                UPDATE variables
+                SET grapherConfigIdETL = ?
+                WHERE id = ?
+            `,
+            [configId, variableId]
+        )
+
+        // grab the admin-authored indicator chart if there is one
+        let patchConfigAdmin: GrapherInterface = {}
+        const row = await db.knexRawFirst<{
+            adminConfig: DbRawChartConfig["full"] // TODO: type
+        }>(
+            trx,
+            `-- sql
+                SELECT cc.patch as adminConfig
+                FROM chart_configs cc
+                JOIN variables v ON cc.id = v.grapherConfigIdAdmin
+                WHERE v.id = ?
+            `,
+            [variableId]
+        )
+        if (row) {
+            patchConfigAdmin = parseChartConfig(row.adminConfig)
+        }
+
+        // find all charts that inherit from the indicator
+        const children = await db.knexRaw<{
+            chartId: number
+            patchConfig: string
+        }>(
+            trx,
+            `-- sql
+                SELECT c.id as chartId, cc.patch as patchConfig
+                FROM inheriting_charts ic
+                    JOIN charts c ON c.id = ic.chartId
+                    JOIN chart_configs cc ON cc.id = c.configId
+                WHERE ic.variableId = ?
+            `,
+            [variableId]
+        )
+
+        for (const child of children) {
+            const patchConfigChild = JSON.parse(child.patchConfig)
+            const fullConfigChild = mergeGrapherConfigs(
+                defaultGrapherConfig,
+                patchConfigETL,
+                patchConfigAdmin,
+                patchConfigChild
+            )
+            await db.knexRaw(
+                trx,
+                `-- sql
+                    UPDATE chart_configs cc
+                    JOIN charts c ON c.configId = cc.id
+                    SET cc.full = ?
+                    WHERE c.id = ?
+                `,
+                [JSON.stringify(fullConfigChild), child.chartId]
+            )
+        }
+    }
+)
+
 getRouteWithROTransaction(
     apiRouter,
     "/datasets.json",

db/migration/1721134584504-MoveIndicatorChartsToTheChartsConfigTable.ts

@@ -0,0 +1,190 @@
+import { defaultGrapherConfig } from "@ourworldindata/grapher"
+import { DimensionProperty, GrapherInterface } from "@ourworldindata/types"
+import {
+    diffGrapherConfigs,
+    mergeGrapherConfigs,
+    omit,
+} from "@ourworldindata/utils"
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class MoveIndicatorChartsToTheChartsConfigTable1721134584504
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            ALTER TABLE variables
+                ADD COLUMN grapherConfigIdAdmin binary(16) UNIQUE AFTER sort,
+                ADD COLUMN grapherConfigIdETL binary(16) UNIQUE AFTER grapherConfigIdAdmin,
+                ADD CONSTRAINT fk_variables_grapherConfigIdAdmin
+                FOREIGN KEY (grapherConfigIdAdmin)
+                    REFERENCES chart_configs (id)
+                    ON DELETE RESTRICT
+                    ON UPDATE RESTRICT,
+                ADD CONSTRAINT fk_variables_grapherConfigIdETL
+                    FOREIGN KEY (grapherConfigIdETL)
+                    REFERENCES chart_configs (id)
+                    ON DELETE RESTRICT
+                    ON UPDATE RESTRICT
+        `)
+
+        // note that we copy the ETL-authored configs to the chart_configs table,
+        // but drop the admin-authored configs
+
+        const variables = await queryRunner.query(`-- sql
+            SELECT id, grapherConfigETL
+            FROM variables
+            WHERE grapherConfigETL IS NOT NULL
+        `)
+
+        for (const { id: variableId, grapherConfigETL } of variables) {
+            let config: GrapherInterface = JSON.parse(grapherConfigETL)
+
+            // if the config has no schema, assume it's the default version
+            if (!config.$schema) {
+                config.$schema = defaultGrapherConfig.$schema
+            }
+
+            // check if the given dimensions are correct
+            if (config.dimensions && config.dimensions.length >= 1) {
+                // make sure there is only a single entry
+                config.dimensions = config.dimensions.slice(0, 1)
+                // make sure the variable id matches
+                config.dimensions[0].variableId = variableId
+            }
+
+            // fill dimensions if not given to make the config plottable
+            if (!config.dimensions || config.dimensions.length === 0) {
+                config.dimensions = [
+                    { property: DimensionProperty.y, variableId },
+                ]
+            }
+
+            // we have v3 configs in the database (the current version is v4);
+            // turn these into v4 configs by removing the `data` property
+            // which was the breaking change that lead to v4
+            // (we don't have v2 or v1 configs in the database, so we don't need to handle those)
+            if (
+                config.$schema ===
+                "https://files.ourworldindata.org/schemas/grapher-schema.003.json"
+            ) {
+                config = omit(config, "data")
+                config.$schema = defaultGrapherConfig.$schema
+            }
+
+            // ETL configs inherit from the default config
+            const patchConfig = diffGrapherConfigs(config, defaultGrapherConfig)
+            const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, config)
+
+            // insert config into the chart_configs table
+            const configId = await getBinaryUUID(queryRunner)
+            await queryRunner.query(
+                `-- sql
+                    INSERT INTO chart_configs (id, patch, full)
+                    VALUES (?, ?, ?)
+                `,
+                [
+                    configId,
+                    JSON.stringify(patchConfig),
+                    JSON.stringify(fullConfig),
+                ]
+            )
+
+            // update reference in the variables table
+            await queryRunner.query(
+                `-- sql
+                    UPDATE variables
+                    SET grapherConfigIdETL = ?
+                    WHERE id = ?
+                `,
+                [configId, variableId]
+            )
+        }
+
+        // drop `grapherConfigAdmin` and `grapherConfigETL` columns
+        await queryRunner.query(`-- sql
+            ALTER TABLE variables
+                DROP COLUMN grapherConfigAdmin,
+                DROP COLUMN grapherConfigETL
+        `)
+
+        await queryRunner.query(`-- sql
+          CREATE VIEW inheritance_indicators_x_charts AS (
+            WITH y_dimensions AS (
+              SELECT
+                *
+              FROM
+                chart_dimensions
+              WHERE
+                property = 'y'
+            ),
+            single_y_indicator_charts As (
+              SELECT
+                c.id as chartId,
+                cc.patch as patchConfig,
+                max(yd.variableId) as variableId
+              FROM
+                charts c
+                JOIN chart_configs cc ON cc.id = c.configId
+                JOIN y_dimensions yd ON c.id = yd.chartId
+              WHERE
+                cc.full ->> '$.type' != 'ScatterPlot'
+              GROUP BY
+                c.id
+              HAVING
+                COUNT(distinct yd.variableId) = 1
+            )
+            SELECT
+              variableId,
+              chartId
+            FROM
+              single_y_indicator_charts
+            ORDER BY
+              variableId
+          )
+        `)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // add back the `grapherConfigAdmin` and `grapherConfigETL` columns
+        await queryRunner.query(`-- sql
+          ALTER TABLE variables
+              ADD COLUMN grapherConfigAdmin json AFTER sort,
+              ADD COLUMN grapherConfigETL json AFTER grapherConfigAdmin
+        `)
+
+        // copy configs from the chart_configs table to the variables table
+        await queryRunner.query(`-- sql
+            UPDATE variables v
+            JOIN chart_configs cc ON v.grapherConfigIdETL = cc.id
+            SET v.grapherConfigETL = cc.patch
+        `)
+
+        // remove constraints on the `grapherConfigIdAdmin` and `grapherConfigIdETL` columns
+        await queryRunner.query(`-- sql
+            ALTER TABLE variables
+                DROP CONSTRAINT fk_variables_grapherConfigIdAdmin,
+                DROP CONSTRAINT fk_variables_grapherConfigIdETL
+        `)
+
+        // drop rows from the chart_configs table
+        await queryRunner.query(`-- sql
+            DELETE FROM chart_configs
+            WHERE id IN (
+                SELECT grapherConfigIdETL FROM variables
+                WHERE grapherConfigIdETL IS NOT NULL
+            )
+        `)
+
+        // remove the `grapherConfigIdAdmin` and `grapherConfigIdETL` columns
+        await queryRunner.query(`-- sql
+            ALTER TABLE variables
+                DROP COLUMN grapherConfigIdAdmin,
+                DROP COLUMN grapherConfigIdETL
+        `)
+    }
+}
+
+const getBinaryUUID = async (queryRunner: QueryRunner): Promise<Buffer> => {
+    const rows = await queryRunner.query(`SELECT UUID_TO_BIN(UUID(), 1) AS id`)
+    return rows[0].id
+}

packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml

@@ -185,7 +185,7 @@ $defs:
                 description: The minimum bracket of the first bin
         additionalProperties: false
 required:
-    - title
+    - $schema
     - version
     - dimensions
 type: object

packages/@ourworldindata/types/src/dbTypes/InheritingCharts.ts

@@ -0,0 +1,8 @@
+export const InheritingChartsTableName = "inheriting_charts"
+
+export interface DbInsertInheritingChart {
+    variableId: number
+    chartId: number
+}
+
+export type DbPlainInheritingChart = Required<DbInsertInheritingChart>