From 8c3b4a45664940fcc69df2ff30a1090331efc991 Mon Sep 17 00:00:00 2001 From: Sophia Mersmann Date: Mon, 7 Oct 2024 16:08:34 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20automatically=20migrate=20outdated?= =?UTF-8?q?=20configs=20/=20TAS-623=20(#3967)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ automatically migrate outdated configs * ✅ add db tests * 📜 (grapher schema) update documentation * 🔨 move config migration into saveGrapher * 🔨 use migrate function in db migration --- adminSiteServer/apiRouter.ts | 96 ++++++++++++------- adminSiteServer/app.test.tsx | 57 +++++++++-- ...1-MigrateOutdatedConfigsToLatestVersion.ts | 52 ++++++++++ db/model/Variable.ts | 5 - .../generate-default-object-from-schema.ts | 18 ++++ packages/@ourworldindata/grapher/src/index.ts | 1 + .../grapher/src/schema/README.md | 2 + .../src/schema/defaultGrapherConfig.ts | 3 + .../grapher/src/schema/migrations/helpers.ts | 58 +++++++++++ .../src/schema/migrations/migrate.test.ts | 74 ++++++++++++++ .../grapher/src/schema/migrations/migrate.ts | 52 ++++++++++ .../src/schema/migrations/migrations.ts | 74 ++++++++++++++ 12 files changed, 443 insertions(+), 49 deletions(-) create mode 100644 db/migration/1726588731621-MigrateOutdatedConfigsToLatestVersion.ts create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/migrate.test.ts create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts create mode 100644 packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index a379345a7dd..3332fe34c90 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -108,6 +108,7 @@ import { import { uuidv7 } from "uuidv7" import { defaultGrapherConfig, + migrateGrapherConfigToLatestVersion, getVariableDataRoute, getVariableMetadataRoute, } from "@ourworldindata/grapher" @@ -501,7 +502,7 @@ const saveGrapher = async ( referencedVariablesMightChange = true, }: { user: DbPlainUser - newConfig: GrapherInterface // Note that it is valid for newConfig to be of an older schema version which means that GrapherInterface as a type is slightly misleading + newConfig: GrapherInterface existingConfig?: GrapherInterface // if undefined, keep inheritance as is. // if true or false, enable or disable inheritance @@ -511,6 +512,9 @@ const saveGrapher = async ( referencedVariablesMightChange?: boolean } ) => { + // Try to migrate the new config to the latest version + newConfig = migrateGrapherConfigToLatestVersion(newConfig) + // Slugs need some special logic to ensure public urls remain consistent whenever possible async function isSlugUsedInRedirect() { const rows = await db.knexRaw( @@ -587,22 +591,6 @@ const saveGrapher = async ( newConfig.version += 1 else newConfig.version = 1 - // if the schema version is missing, assume it's the latest - if (newConfig.$schema === undefined) { - newConfig.$schema = defaultGrapherConfig.$schema - } else if ( - newConfig.$schema === - "https://files.ourworldindata.org/schemas/grapher-schema.004.json" - ) { - // TODO: find a more principled way to do schema upgrades - - // grapher-schema.004 -> grapher-schema.005 removed the obsolete hideLinesOutsideTolerance field - const configForMigration = newConfig as any - delete configForMigration.hideLinesOutsideTolerance - configForMigration.$schema = defaultGrapherConfig.$schema - newConfig = configForMigration - } - // add the isPublished field if is missing if (newConfig.isPublished === undefined) { newConfig.isPublished = false @@ -1042,13 +1030,17 @@ postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { shouldInherit = req.query.inheritance === "enable" } - const { chartId } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - shouldInherit, - }) + try { + const { chartId } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + shouldInherit, + }) - return { success: true, chartId: chartId } + return { success: true, chartId: chartId } + } catch (err) { + return { success: false, error: String(err) } + } }) postRouteWithRWTransaction( @@ -1074,19 +1066,29 @@ putRouteWithRWTransaction( const existingConfig = await expectChartById(trx, req.params.chartId) - const { chartId, savedPatch } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - existingConfig, - shouldInherit, - }) + try { + const { chartId, savedPatch } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + existingConfig, + shouldInherit, + }) - const logs = await getLogsByChartId(trx, existingConfig.id as number) - return { - success: true, - chartId, - savedPatch, - newLog: logs[0], + const logs = await getLogsByChartId( + trx, + existingConfig.id as number + ) + return { + success: true, + chartId, + savedPatch, + newLog: logs[0], + } + } catch (err) { + return { + success: false, + error: String(err), + } } } ) @@ -1617,13 +1619,23 @@ putRouteWithRWTransaction( async (req, res, trx) => { const variableId = expectInt(req.params.variableId) + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), + } + } + const variable = await getGrapherConfigsForVariable(trx, variableId) if (!variable) { throw new JsonError(`Variable with id ${variableId} not found`, 500) } const { savedPatch, updatedCharts } = - await updateGrapherConfigETLOfVariable(trx, variable, req.body) + await updateGrapherConfigETLOfVariable(trx, variable, validConfig) // trigger build if any published chart has been updated if (updatedCharts.some((chart) => chart.isPublished)) { @@ -1712,13 +1724,23 @@ putRouteWithRWTransaction( async (req, res, trx) => { const variableId = expectInt(req.params.variableId) + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), + } + } + const variable = await getGrapherConfigsForVariable(trx, variableId) if (!variable) { throw new JsonError(`Variable with id ${variableId} not found`, 500) } const { savedPatch, updatedCharts } = - await updateGrapherConfigAdminOfVariable(trx, variable, req.body) + await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) // trigger build if any published chart has been updated if (updatedCharts.some((chart) => chart.isPublished)) { diff --git a/adminSiteServer/app.test.tsx b/adminSiteServer/app.test.tsx index 8acb2fb4c94..26b3cdca948 100644 --- a/adminSiteServer/app.test.tsx +++ b/adminSiteServer/app.test.tsx @@ -53,7 +53,6 @@ import { DatasetsTableName, VariablesTableName, } from "@ourworldindata/types" -import { defaultGrapherConfig } from "@ourworldindata/grapher" import path from "path" import fs from "fs" import { omitUndefinedValues } from "@ourworldindata/utils" @@ -191,6 +190,8 @@ async function makeRequestAgainstAdminApi( describe("OwidAdminApp", () => { const testChartConfig = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", slug: "test-chart", title: "Test chart", type: "LineChart", @@ -252,7 +253,7 @@ describe("OwidAdminApp", () => { ) expect(fullConfig).toHaveProperty( "$schema", - defaultGrapherConfig.$schema + "https://files.ourworldindata.org/schemas/grapher-schema.005.json" ) expect(fullConfig).toHaveProperty("id", chartId) // must match the db id expect(fullConfig).toHaveProperty("version", 1) // automatically added @@ -267,7 +268,8 @@ describe("OwidAdminApp", () => { `/charts/${chartId}.patchConfig.json` ) expect(patchConfig).toEqual({ - $schema: defaultGrapherConfig["$schema"], + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", id: chartId, version: 1, isPublished: false, @@ -321,12 +323,16 @@ describe("OwidAdminApp: indicator-level chart configs", () => { display: '{ "unit": "kg", "shortUnit": "kg" }', } const testVariableConfigETL = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", hasMapTab: true, note: "Indicator note", selectedEntityNames: ["France", "Italy", "Spain"], hideRelativeToggle: false, } const testVariableConfigAdmin = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", title: "Admin title", subtitle: "Admin subtitle", } @@ -337,10 +343,14 @@ describe("OwidAdminApp: indicator-level chart configs", () => { id: otherVariableId, } const otherTestVariableConfig = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", note: "Other indicator note", } const testChartConfig = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", slug: "test-chart", title: "Test chart", type: "Marimekko", @@ -382,12 +392,11 @@ describe("OwidAdminApp: indicator-level chart configs", () => { // for ETL configs, patch and full configs should be the same expect(patchConfigETL).toEqual(fullConfigETL) - // check that $schema and dimensions field were added to the config + // check that the dimensions field were added to the config const processedTestVariableConfigETL = { ...testVariableConfigETL, // automatically added - $schema: defaultGrapherConfig.$schema, dimensions: [ { property: "y", @@ -503,7 +512,8 @@ describe("OwidAdminApp: indicator-level chart configs", () => { `/charts/${chartId}.patchConfig.json` ) expect(patchConfig).toEqual({ - $schema: defaultGrapherConfig["$schema"], + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", id: chartId, version: 1, isPublished: false, @@ -558,7 +568,8 @@ describe("OwidAdminApp: indicator-level chart configs", () => { `/charts/${chartId}.patchConfig.json` ) expect(patchConfigAfterDelete).toEqual({ - $schema: defaultGrapherConfig["$schema"], + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.005.json", id: chartId, version: 1, isPublished: false, @@ -765,4 +776,36 @@ describe("OwidAdminApp: indicator-level chart configs", () => { expect(configAfterUpdate).not.toBeNull() expect(chartAfterUpdate).toEqual(configAfterUpdate) }) + + it("should return an error if the schema is missing", async () => { + const invalidConfig = { + title: "Title", + // note that the $schema field is missing + } + const json = await makeRequestAgainstAdminApi( + { + method: "PUT", + path: `/variables/${variableId}/grapherConfigETL`, + body: JSON.stringify(invalidConfig), + }, + { verifySuccess: false } + ) + expect(json.success).toBe(false) + }) + + it("should return an error if the schema is invalid", async () => { + const invalidConfig = { + $schema: "invalid", // note that the $schema field is invalid + title: "Title", + } + const json = await makeRequestAgainstAdminApi( + { + method: "PUT", + path: `/variables/${variableId}/grapherConfigETL`, + body: JSON.stringify(invalidConfig), + }, + { verifySuccess: false } + ) + expect(json.success).toBe(false) + }) }) diff --git a/db/migration/1726588731621-MigrateOutdatedConfigsToLatestVersion.ts b/db/migration/1726588731621-MigrateOutdatedConfigsToLatestVersion.ts new file mode 100644 index 00000000000..5df7228815c --- /dev/null +++ b/db/migration/1726588731621-MigrateOutdatedConfigsToLatestVersion.ts @@ -0,0 +1,52 @@ +import { migrateGrapherConfigToLatestVersion } from "@ourworldindata/grapher" +import { GrapherInterface } from "@ourworldindata/types" +import { MigrationInterface, QueryRunner } from "typeorm" + +export class MigrateOutdatedConfigsToLatestVersion1726588731621 + implements MigrationInterface +{ + private migrateConfig(config: Record): GrapherInterface { + try { + return migrateGrapherConfigToLatestVersion(config) + } catch { + // if the migration function throws, then the $schema field + // is either missing or invalid. when that happens, we assume + // a schema v1, and try again + config.$schema = + "https://files.ourworldindata.org/schemas/grapher-schema.001.json" + return migrateGrapherConfigToLatestVersion(config) + } + } + + public async up(queryRunner: QueryRunner): Promise { + const outdatedConfigs = await queryRunner.query( + `-- sql + SELECT id, patch, full + FROM chart_configs + WHERE + patch ->> '$.$schema' != 'https://files.ourworldindata.org/schemas/grapher-schema.005.json' + OR full ->> '$.$schema' != 'https://files.ourworldindata.org/schemas/grapher-schema.005.json' + ` + ) + + for (const { id, patch, full } of outdatedConfigs) { + const updatedPatch = this.migrateConfig(JSON.parse(patch)) + const updatedFull = this.migrateConfig(JSON.parse(full)) + + await queryRunner.query( + `-- sql + UPDATE chart_configs + SET patch = ?, full = ? + WHERE id = ? + `, + [JSON.stringify(updatedPatch), JSON.stringify(updatedFull), id] + ) + } + } + + public async down(): Promise { + throw new Error( + "Can't revert migration MigrateOutdatedConfigsToLatestVersion1726588731621" + ) + } +} diff --git a/db/model/Variable.ts b/db/model/Variable.ts index 5397c65feda..ebc232fbc3e 100644 --- a/db/model/Variable.ts +++ b/db/model/Variable.ts @@ -189,11 +189,6 @@ function makeConfigValidForIndicator({ }): GrapherInterface { const updatedConfig = { ...config } - // if no schema is given, assume it's the latest - if (!updatedConfig.$schema) { - updatedConfig.$schema = defaultGrapherConfig.$schema - } - // validate the given y-dimensions const defaultDimension = { property: DimensionProperty.y, variableId } const [yDimensions, otherDimensions] = _.partition( diff --git a/devTools/schema/generate-default-object-from-schema.ts b/devTools/schema/generate-default-object-from-schema.ts index 3ef1f66ca00..b4f553a5744 100644 --- a/devTools/schema/generate-default-object-from-schema.ts +++ b/devTools/schema/generate-default-object-from-schema.ts @@ -2,6 +2,15 @@ import parseArgs from "minimist" import fs from "fs-extra" +import { range } from "lodash" + +const schemaVersionRegex = + /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?\d{3})\.json/m +const getSchemaVersion = (config: Record): string => + config.$schema?.match(schemaVersionRegex)?.groups?.version ?? "000" + +const toArrayString = (arr: string[]) => + `[${arr.map((v) => `"${v}"`).join(", ")}]` function generateDefaultObjectFromSchema( schema: Record, @@ -49,6 +58,12 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { // save as ts file if requested if (parsedArgs["save-ts"]) { + const latestVersion = getSchemaVersion(defaultConfig) + const outdatedVersionsAsInts = range(1, parseInt(latestVersion)) + const outdatedVersions = outdatedVersionsAsInts.map((versionNumber) => + versionNumber.toString().padStart(3, "0") + ) + const out = parsedArgs["save-ts"] const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY @@ -56,6 +71,9 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { import { GrapherInterface } from "@ourworldindata/types" +export const latestSchemaVersion = "${latestVersion}" as const +export const outdatedSchemaVersions = ${toArrayString(outdatedVersions)} as const + export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface` fs.outputFileSync(out, content) } diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index 9e9d08b9902..203bb60db3a 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -80,3 +80,4 @@ export { SlideShowController, } from "./slideshowController/SlideShowController" export { defaultGrapherConfig } from "./schema/defaultGrapherConfig" +export { migrateGrapherConfigToLatestVersion } from "./schema/migrations/migrate" diff --git a/packages/@ourworldindata/grapher/src/schema/README.md b/packages/@ourworldindata/grapher/src/schema/README.md index cad2a6eb595..2d6ae518365 100644 --- a/packages/@ourworldindata/grapher/src/schema/README.md +++ b/packages/@ourworldindata/grapher/src/schema/README.md @@ -19,6 +19,8 @@ Checklist for breaking changes: - Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version - Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below) - Write a migration to update the `chart_configs.full` column in the database for all stand-alone charts +- Write a migration to update configs in code (see `migrations/migrations.ts`) +- Update the hardcoded default schema version in ETL To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run: diff --git a/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts b/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts index 01493285bff..6497d08f13c 100644 --- a/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts +++ b/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts @@ -4,6 +4,9 @@ import { GrapherInterface } from "@ourworldindata/types" +export const latestSchemaVersion = "005" as const +export const outdatedSchemaVersions = ["001", "002", "003", "004"] as const + export const defaultGrapherConfig = { $schema: "https://files.ourworldindata.org/schemas/grapher-schema.005.json", map: { diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts b/packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts new file mode 100644 index 00000000000..456da3aa68a --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/helpers.ts @@ -0,0 +1,58 @@ +import { + latestSchemaVersion, + outdatedSchemaVersions, +} from "../defaultGrapherConfig" + +const allSchemaVersions = [...outdatedSchemaVersions, latestSchemaVersion] + +type LatestSchemaVersion = typeof latestSchemaVersion +type OutdatedSchemaVersion = (typeof outdatedSchemaVersions)[number] +type SchemaVersion = OutdatedSchemaVersion | LatestSchemaVersion + +type Schema = + `https://files.ourworldindata.org/schemas/grapher-schema.${SchemaVersion}.json` + +// we can't type configs that don't adhere to the latest schema as we don't know what they look like +export type AnyConfig = Record +export type AnyConfigWithValidSchema = AnyConfig & { + $schema: Schema +} + +const schemaVersionRegex = + /https:\/\/files\.ourworldindata\.org\/schemas\/grapher-schema\.(?\d{3})\.json/m + +const isValidSchemaVersion = (version: string): version is SchemaVersion => + allSchemaVersions.includes(version as any) + +export function getSchemaVersion( + config: AnyConfigWithValidSchema +): SchemaVersion +export function getSchemaVersion(config: AnyConfig): SchemaVersion | null +export function getSchemaVersion( + config: AnyConfig | AnyConfigWithValidSchema +): SchemaVersion | null { + const version = config.$schema?.match(schemaVersionRegex)?.groups?.version + if (!version || !isValidSchemaVersion(version)) return null + return version +} + +export function createSchemaForVersion(version: SchemaVersion): Schema { + return `https://files.ourworldindata.org/schemas/grapher-schema.${version}.json` +} + +export const isLatestVersion = (version: SchemaVersion) => + version === latestSchemaVersion + +export const isOutdatedVersion = (version: SchemaVersion) => + outdatedSchemaVersions.includes(version as any) + +export const hasValidSchema = ( + config: AnyConfig +): config is AnyConfigWithValidSchema => getSchemaVersion(config) !== null + +export const hasOutdatedSchema = ( + config: AnyConfigWithValidSchema +): boolean => { + const version = getSchemaVersion(config) + return isOutdatedVersion(version) +} diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/migrate.test.ts b/packages/@ourworldindata/grapher/src/schema/migrations/migrate.test.ts new file mode 100644 index 00000000000..9f6704c0e98 --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/migrate.test.ts @@ -0,0 +1,74 @@ +#! /usr/bin/env jest + +import { defaultGrapherConfig } from "../defaultGrapherConfig" +import { migrateGrapherConfigToLatestVersion } from "./migrate" + +it("returns a valid config as is", () => { + const validConfig = { + $schema: defaultGrapherConfig.$schema, + title: "Test", + } + expect(migrateGrapherConfigToLatestVersion(validConfig)).toEqual( + validConfig + ) +}) + +it("throws if the schema field is missing", () => { + expect(() => migrateGrapherConfigToLatestVersion({})).toThrow() +}) + +it("throws if the schema field is invalid", () => { + expect(() => + migrateGrapherConfigToLatestVersion({ + $schema: "invalid", + }) + ).toThrow() +}) + +it("runs multiple migrations if necessary", () => { + const outdatedConfig = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.003.json", + data: { availableEntities: [] }, // removed in v4 + hideLinesOutsideTolerance: true, // removed in v5 + } + const validConfig = migrateGrapherConfigToLatestVersion(outdatedConfig) + expect(validConfig).not.toHaveProperty("data") + expect(validConfig).not.toHaveProperty("hideLinesOutsideTolerance") +}) + +it("doesn't mutate the given config", () => { + const outdatedConfig = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.004.json", + hideLinesOutsideTolerance: true, + } + const validConfig = migrateGrapherConfigToLatestVersion(outdatedConfig) + expect(validConfig).not.toHaveProperty("hideLinesOutsideTolerance") + expect(outdatedConfig).toEqual({ + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.004.json", + hideLinesOutsideTolerance: true, + }) +}) + +// Bit of a funky test that ensures the migration function terminates. +// Not really necessary since the tests above also fail when they encounter +// an infinite loop, but this one provides context on what's going wrong and gives +// guidance on what to do about it. +it("terminates", () => { + const outdatedConfig = { + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.001.json", + } + + try { + migrateGrapherConfigToLatestVersion(outdatedConfig) + } catch (error) { + if (error instanceof RangeError) { + expect("should terminate, but doesn't").toBe( + "check if the config's $schema field is updated to the next version in every migration function" + ) + } + } +}) diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts b/packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts new file mode 100644 index 00000000000..dccd7a892a5 --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/migrate.ts @@ -0,0 +1,52 @@ +import { GrapherInterface } from "@ourworldindata/types" +import { cloneDeep } from "@ourworldindata/utils" + +import { defaultGrapherConfig } from "../defaultGrapherConfig" +import { + getSchemaVersion, + hasValidSchema, + isLatestVersion, + hasOutdatedSchema, + type AnyConfig, + type AnyConfigWithValidSchema, +} from "./helpers" +import { runMigration } from "./migrations" + +const recursivelyApplyMigrations = ( + config: AnyConfigWithValidSchema +): AnyConfigWithValidSchema => { + const version = getSchemaVersion(config) + if (isLatestVersion(version)) return config + return recursivelyApplyMigrations(runMigration(config)) +} + +const migrate = (config: AnyConfigWithValidSchema): GrapherInterface => + recursivelyApplyMigrations(config) as GrapherInterface + +/** + * Attempts to migrate a config to the latest schema version. + * + * An outdated config is migrated to the latest version by applying a series of + * predefined migrations. We rely on the schema version to determine if a config + * is outdated. + * + * Note that the given config is not actually validated against the schema! + */ +export const migrateGrapherConfigToLatestVersion = ( + config: AnyConfig +): GrapherInterface => { + // the config adheres to the latest schema + if (config.$schema === defaultGrapherConfig.$schema) return config + + // if the schema version is outdated, migrate it + if (hasValidSchema(config) && hasOutdatedSchema(config)) { + return migrate(cloneDeep(config)) + } + + // throw if the schema is invalid or missing + if (config.$schema === undefined) { + throw new Error("Schema missing") + } else { + throw new Error(`Invalid schema: ${config.$schema}`) + } +} diff --git a/packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts b/packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts new file mode 100644 index 00000000000..7a4fefa30f0 --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/migrations/migrations.ts @@ -0,0 +1,74 @@ +// Schema migrations +// +// Every breaking change to the schema should be accompanied by a migration. +// +// To add a migration, follow these steps: +// - Create a new function `migrateXXXToXXX` that migrates a config from one version to the next. +// Make sure to update the $schema field to the next version in your migration function, +// or else the recursively defined migrate function will not terminate! +// - Add a new case to the match statement in `runMigration` that calls the new migration function. + +import { match } from "ts-pattern" +import { + type AnyConfigWithValidSchema, + createSchemaForVersion, + getSchemaVersion, + isLatestVersion, +} from "./helpers" + +// see https://github.com/owid/owid-grapher/commit/26f2a0d1790c71bdda7e12f284ca552945d2f6ef +const migrateFrom001To002 = ( + config: AnyConfigWithValidSchema +): AnyConfigWithValidSchema => { + delete config.selectedData + config.$schema = createSchemaForVersion("002") + return config +} + +// see https://github.com/owid/owid-grapher/commit/4525ad81fb7064709ffab83677a8b0354b324dfb +const migrateFrom002To003 = ( + config: AnyConfigWithValidSchema +): AnyConfigWithValidSchema => { + if (config.hideTitleAnnotation) { + config.hideTitleAnnotations = { + entity: true, + time: true, + change: true, + } + } + delete config.hideTitleAnnotation + + config.$schema = createSchemaForVersion("003") + return config +} + +// see https://github.com/owid/owid-grapher/commit/1776721253cf61d7f1e24ebadeaf7a7ca2f43ced +const migrateFrom003To004 = ( + config: AnyConfigWithValidSchema +): AnyConfigWithValidSchema => { + delete config.data + config.$schema = createSchemaForVersion("004") + return config +} + +// see https://github.com/owid/owid-grapher/commit/1d67de3174764a413bc5055fbdf34efb2b49e079 +const migrateFrom004To005 = ( + config: AnyConfigWithValidSchema +): AnyConfigWithValidSchema => { + delete config.hideLinesOutsideTolerance + config.$schema = createSchemaForVersion("005") + return config +} + +export const runMigration = ( + config: AnyConfigWithValidSchema +): AnyConfigWithValidSchema => { + const version = getSchemaVersion(config) + if (isLatestVersion(version)) return config + return match(version) + .with("001", () => migrateFrom001To002(config)) + .with("002", () => migrateFrom002To003(config)) + .with("003", () => migrateFrom003To004(config)) + .with("004", () => migrateFrom004To005(config)) + .exhaustive() +}