-
-
Notifications
You must be signed in to change notification settings - Fork 229
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: create helpers to save chart configs to DB and R2 simultane…
…ously (#4189) * refactor: create helper file for common chart config writes * refactor: adapt multidim to use helper methods * refactor: use helpers in `apiRouter` * refactor: use better types in multidim * enhance: address review comments
- Loading branch information
1 parent
53e4191
commit 844a20b
Showing
3 changed files
with
151 additions
and
170 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| import { | ||
| Base64String, | ||
| ChartConfigsTableName, | ||
| DbInsertChartConfig, | ||
| DbRawChartConfig, | ||
| GrapherInterface, | ||
| R2GrapherConfigDirectory, | ||
| serializeChartConfig, | ||
| } from "@ourworldindata/types" | ||
| import { uuidv7 } from "uuidv7" | ||
| import * as db from "../db/db.js" | ||
| import { | ||
| saveGrapherConfigToR2, | ||
| saveGrapherConfigToR2ByUUID, | ||
| } from "./chartConfigR2Helpers.js" | ||
|
|
||
| /** | ||
| * One particular detail of of MySQL's JSON support is that MySQL _normalizes_ JSON when storing it. | ||
| * This means that the JSON string representation of a JSON object stored in MySQL is not equivalent | ||
| * to the input of an INSERT statement: it may have different whitespace and key order. | ||
| * This is a problem when we compute MD5 hashes of JSON objects using computed MySQL columns - in | ||
| * order to get the correct hash, we need to first store the JSON object in MySQL and then retrieve | ||
| * it and its hash again from MySQL immediately afterwards, such that we can store the exact same | ||
| * JSON representation and hash in R2 also. | ||
| * The below is a helper function that does just this. | ||
| * - @marcelgerber, 2024-11-20 | ||
| */ | ||
|
|
||
| export const retrieveChartConfigFromDbAndSaveToR2 = async ( | ||
| knex: db.KnexReadonlyTransaction, | ||
| chartConfigId: Base64String, | ||
| r2Path?: { directory: R2GrapherConfigDirectory; filename: string } | ||
| ) => { | ||
| // We need to get the full config and the md5 hash from the database instead of | ||
| // computing our own md5 hash because MySQL normalizes JSON and our | ||
| // client computed md5 would be different from the ones computed by and stored in R2 | ||
| const fullConfigMd5: Pick<DbRawChartConfig, "full" | "fullMd5"> = | ||
| await knex(ChartConfigsTableName) | ||
| .select("full", "fullMd5") | ||
| .where({ id: chartConfigId }) | ||
| .first() | ||
|
|
||
| if (!fullConfigMd5) | ||
| throw new Error( | ||
| `Chart config not found in the database! id=${chartConfigId}` | ||
| ) | ||
|
|
||
| if (!r2Path) { | ||
| await saveGrapherConfigToR2ByUUID( | ||
| chartConfigId, | ||
| fullConfigMd5.full, | ||
| fullConfigMd5.fullMd5 as Base64String | ||
| ) | ||
| } else { | ||
| await saveGrapherConfigToR2( | ||
| fullConfigMd5.full, | ||
| r2Path.directory, | ||
| r2Path.filename, | ||
| fullConfigMd5.fullMd5 as Base64String | ||
| ) | ||
| } | ||
|
|
||
| return { | ||
| chartConfigId, | ||
| fullConfig: fullConfigMd5.full, | ||
| fullConfigMd5: fullConfigMd5.fullMd5, | ||
| } | ||
| } | ||
|
|
||
| export const updateChartConfigInDbAndR2 = async ( | ||
| knex: db.KnexReadWriteTransaction, | ||
| chartConfigId: Base64String, | ||
| patchConfig: GrapherInterface, | ||
| fullConfig: GrapherInterface | ||
| ) => { | ||
| await knex<DbInsertChartConfig>(ChartConfigsTableName) | ||
| .update({ | ||
| patch: serializeChartConfig(patchConfig), | ||
| full: serializeChartConfig(fullConfig), | ||
| updatedAt: new Date(), // It's not updated automatically in the DB. | ||
| }) | ||
| .where({ id: chartConfigId }) | ||
|
|
||
| return retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId) | ||
| } | ||
|
|
||
| export const saveNewChartConfigInDbAndR2 = async ( | ||
| knex: db.KnexReadWriteTransaction, | ||
| chartConfigId: Base64String | undefined, | ||
| patchConfig: GrapherInterface, | ||
| fullConfig: GrapherInterface | ||
| ) => { | ||
| chartConfigId = chartConfigId ?? (uuidv7() as Base64String) | ||
|
|
||
| await knex<DbInsertChartConfig>(ChartConfigsTableName).insert({ | ||
| id: chartConfigId, | ||
| patch: serializeChartConfig(patchConfig), | ||
| full: serializeChartConfig(fullConfig), | ||
| }) | ||
|
|
||
| return retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId) | ||
| } |
Oops, something went wrong.