diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 5850f677750..9362de400ef 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -85,6 +85,7 @@ import { DbInsertUser, FlatTagGraph, DbRawChartConfig, + parseChartConfig, } from "@ourworldindata/types" import { defaultGrapherConfig, @@ -1290,6 +1291,121 @@ getRouteWithROTransaction( } ) +postRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL/new", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + // const user = res.locals.user + const config = req.body + + // if no schema is given, assume it's the latest + if (!config.$schema) { + config.$schema = defaultGrapherConfig.$schema + } + + // check if the given dimensions are correct + if (config.dimensions && config.dimensions.length >= 1) { + // make sure there is only a single entry + config.dimensions = config.dimensions.slice(0, 1) + // make sure the variable id matches + config.dimensions[0].variableId = variableId + } + + // fill dimensions if not given to make the config plottable + if (!config.dimensions || config.dimensions.length === 0) { + config.dimensions = [{ property: DimensionProperty.y, variableId }] + } + + // ETL configs inherit from the default + const patchConfigETL = diffGrapherConfigs(config, defaultGrapherConfig) + const fullConfigETL = mergeGrapherConfigs( + defaultGrapherConfig, + patchConfigETL + ) + + // insert chart config into the database + const configId = await getBinaryUUID(trx) + await db.knexRaw( + trx, + `-- sql + INSERT INTO chart_configs (id, patch, full) + VALUES (?, ?, ?) + `, + [ + configId, + JSON.stringify(patchConfigETL), + JSON.stringify(fullConfigETL), + ] + ) + + // make a reference to the config from the variables table + await db.knexRaw( + trx, + `-- sql + UPDATE variables + SET grapherConfigIdETL = ? + WHERE id = ? + `, + [configId, variableId] + ) + + // grab the admin-authored indicator chart if there is one + let patchConfigAdmin: GrapherInterface = {} + const row = await db.knexRawFirst<{ + adminConfig: DbRawChartConfig["full"] // TODO: type + }>( + trx, + `-- sql + SELECT cc.patch as adminConfig + FROM chart_configs cc + JOIN variables v ON cc.id = v.grapherConfigIdAdmin + WHERE v.id = ? + `, + [variableId] + ) + if (row) { + patchConfigAdmin = parseChartConfig(row.adminConfig) + } + + // find all charts that inherit from the indicator + const children = await db.knexRaw<{ + chartId: number + patchConfig: string + }>( + trx, + `-- sql + SELECT c.id as chartId, cc.patch as patchConfig + FROM inheriting_charts ic + JOIN charts c ON c.id = ic.chartId + JOIN chart_configs cc ON cc.id = c.configId + WHERE ic.variableId = ? + `, + [variableId] + ) + + for (const child of children) { + const patchConfigChild = JSON.parse(child.patchConfig) + const fullConfigChild = mergeGrapherConfigs( + defaultGrapherConfig, + patchConfigETL, + patchConfigAdmin, + patchConfigChild + ) + await db.knexRaw( + trx, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET cc.full = ? + WHERE c.id = ? + `, + [JSON.stringify(fullConfigChild), child.chartId] + ) + } + } +) + getRouteWithROTransaction( apiRouter, "/datasets.json", diff --git a/db/migration/1721134584504-MoveIndicatorChartsToTheChartsConfigTable.ts b/db/migration/1721134584504-MoveIndicatorChartsToTheChartsConfigTable.ts new file mode 100644 index 00000000000..02d4ded66a0 --- /dev/null +++ b/db/migration/1721134584504-MoveIndicatorChartsToTheChartsConfigTable.ts @@ -0,0 +1,190 @@ +import { defaultGrapherConfig } from "@ourworldindata/grapher" +import { DimensionProperty, GrapherInterface } from "@ourworldindata/types" +import { + diffGrapherConfigs, + mergeGrapherConfigs, + omit, +} from "@ourworldindata/utils" +import { MigrationInterface, QueryRunner } from "typeorm" + +export class MoveIndicatorChartsToTheChartsConfigTable1721134584504 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`-- sql + ALTER TABLE variables + ADD COLUMN grapherConfigIdAdmin binary(16) UNIQUE AFTER sort, + ADD COLUMN grapherConfigIdETL binary(16) UNIQUE AFTER grapherConfigIdAdmin, + ADD CONSTRAINT fk_variables_grapherConfigIdAdmin + FOREIGN KEY (grapherConfigIdAdmin) + REFERENCES chart_configs (id) + ON DELETE RESTRICT + ON UPDATE RESTRICT, + ADD CONSTRAINT fk_variables_grapherConfigIdETL + FOREIGN KEY (grapherConfigIdETL) + REFERENCES chart_configs (id) + ON DELETE RESTRICT + ON UPDATE RESTRICT + `) + + // note that we copy the ETL-authored configs to the chart_configs table, + // but drop the admin-authored configs + + const variables = await queryRunner.query(`-- sql + SELECT id, grapherConfigETL + FROM variables + WHERE grapherConfigETL IS NOT NULL + `) + + for (const { id: variableId, grapherConfigETL } of variables) { + let config: GrapherInterface = JSON.parse(grapherConfigETL) + + // if the config has no schema, assume it's the default version + if (!config.$schema) { + config.$schema = defaultGrapherConfig.$schema + } + + // check if the given dimensions are correct + if (config.dimensions && config.dimensions.length >= 1) { + // make sure there is only a single entry + config.dimensions = config.dimensions.slice(0, 1) + // make sure the variable id matches + config.dimensions[0].variableId = variableId + } + + // fill dimensions if not given to make the config plottable + if (!config.dimensions || config.dimensions.length === 0) { + config.dimensions = [ + { property: DimensionProperty.y, variableId }, + ] + } + + // we have v3 configs in the database (the current version is v4); + // turn these into v4 configs by removing the `data` property + // which was the breaking change that lead to v4 + // (we don't have v2 or v1 configs in the database, so we don't need to handle those) + if ( + config.$schema === + "https://files.ourworldindata.org/schemas/grapher-schema.003.json" + ) { + config = omit(config, "data") + config.$schema = defaultGrapherConfig.$schema + } + + // ETL configs inherit from the default config + const patchConfig = diffGrapherConfigs(config, defaultGrapherConfig) + const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, config) + + // insert config into the chart_configs table + const configId = await getBinaryUUID(queryRunner) + await queryRunner.query( + `-- sql + INSERT INTO chart_configs (id, patch, full) + VALUES (?, ?, ?) + `, + [ + configId, + JSON.stringify(patchConfig), + JSON.stringify(fullConfig), + ] + ) + + // update reference in the variables table + await queryRunner.query( + `-- sql + UPDATE variables + SET grapherConfigIdETL = ? + WHERE id = ? + `, + [configId, variableId] + ) + } + + // drop `grapherConfigAdmin` and `grapherConfigETL` columns + await queryRunner.query(`-- sql + ALTER TABLE variables + DROP COLUMN grapherConfigAdmin, + DROP COLUMN grapherConfigETL + `) + + await queryRunner.query(`-- sql + CREATE VIEW inheritance_indicators_x_charts AS ( + WITH y_dimensions AS ( + SELECT + * + FROM + chart_dimensions + WHERE + property = 'y' + ), + single_y_indicator_charts As ( + SELECT + c.id as chartId, + cc.patch as patchConfig, + max(yd.variableId) as variableId + FROM + charts c + JOIN chart_configs cc ON cc.id = c.configId + JOIN y_dimensions yd ON c.id = yd.chartId + WHERE + cc.full ->> '$.type' != 'ScatterPlot' + GROUP BY + c.id + HAVING + COUNT(distinct yd.variableId) = 1 + ) + SELECT + variableId, + chartId + FROM + single_y_indicator_charts + ORDER BY + variableId + ) + `) + } + + public async down(queryRunner: QueryRunner): Promise { + // add back the `grapherConfigAdmin` and `grapherConfigETL` columns + await queryRunner.query(`-- sql + ALTER TABLE variables + ADD COLUMN grapherConfigAdmin json AFTER sort, + ADD COLUMN grapherConfigETL json AFTER grapherConfigAdmin + `) + + // copy configs from the chart_configs table to the variables table + await queryRunner.query(`-- sql + UPDATE variables v + JOIN chart_configs cc ON v.grapherConfigIdETL = cc.id + SET v.grapherConfigETL = cc.patch + `) + + // remove constraints on the `grapherConfigIdAdmin` and `grapherConfigIdETL` columns + await queryRunner.query(`-- sql + ALTER TABLE variables + DROP CONSTRAINT fk_variables_grapherConfigIdAdmin, + DROP CONSTRAINT fk_variables_grapherConfigIdETL + `) + + // drop rows from the chart_configs table + await queryRunner.query(`-- sql + DELETE FROM chart_configs + WHERE id IN ( + SELECT grapherConfigIdETL FROM variables + WHERE grapherConfigIdETL IS NOT NULL + ) + `) + + // remove the `grapherConfigIdAdmin` and `grapherConfigIdETL` columns + await queryRunner.query(`-- sql + ALTER TABLE variables + DROP COLUMN grapherConfigIdAdmin, + DROP COLUMN grapherConfigIdETL + `) + } +} + +const getBinaryUUID = async (queryRunner: QueryRunner): Promise => { + const rows = await queryRunner.query(`SELECT UUID_TO_BIN(UUID(), 1) AS id`) + return rows[0].id +} diff --git a/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml b/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml index 1d51b63eff6..cc716cc2435 100644 --- a/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml +++ b/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml @@ -185,7 +185,7 @@ $defs: description: The minimum bracket of the first bin additionalProperties: false required: - - title + - $schema - version - dimensions type: object diff --git a/packages/@ourworldindata/types/src/dbTypes/InheritingCharts.ts b/packages/@ourworldindata/types/src/dbTypes/InheritingCharts.ts new file mode 100644 index 00000000000..3a956f68cc7 --- /dev/null +++ b/packages/@ourworldindata/types/src/dbTypes/InheritingCharts.ts @@ -0,0 +1,8 @@ +export const InheritingChartsTableName = "inheriting_charts" + +export interface DbInsertInheritingChart { + variableId: number + chartId: number +} + +export type DbPlainInheritingChart = Required