diff --git a/.env.devcontainer b/.env.devcontainer index 7bc29cc6500..dc8a8bd8d54 100644 --- a/.env.devcontainer +++ b/.env.devcontainer @@ -16,8 +16,8 @@ GDOCS_CLIENT_ID='' GDOCS_BASIC_ARTICLE_TEMPLATE_URL='' GDOCS_SHARED_DRIVE_ID='' -IMAGE_HOSTING_R2_ENDPOINT='' +R2_ENDPOINT='' IMAGE_HOSTING_R2_CDN_URL='' IMAGE_HOSTING_R2_BUCKET_PATH='' -IMAGE_HOSTING_R2_ACCESS_KEY_ID='' -IMAGE_HOSTING_R2_SECRET_ACCESS_KEY='' +R2_ACCESS_KEY_ID='' +R2_SECRET_ACCESS_KEY='' diff --git a/.env.example-full b/.env.example-full index 8407d05b69b..d0cb5063adf 100644 --- a/.env.example-full +++ b/.env.example-full @@ -22,11 +22,17 @@ GDOCS_BASIC_ARTICLE_TEMPLATE_URL= GDOCS_SHARED_DRIVE_ID= GDOCS_DONATE_FAQS_DOCUMENT_ID= # optional -IMAGE_HOSTING_R2_ENDPOINT= # optional +R2_ENDPOINT= # optional IMAGE_HOSTING_R2_CDN_URL= IMAGE_HOSTING_R2_BUCKET_PATH= -IMAGE_HOSTING_R2_ACCESS_KEY_ID= # optional -IMAGE_HOSTING_R2_SECRET_ACCESS_KEY= # optional +R2_ACCESS_KEY_ID= # optional +R2_SECRET_ACCESS_KEY= # optional +# These two GRAPHER_CONFIG_ settings are used to store grapher configs in an R2 bucket. +# The cloudflare workers for thumbnail rendering etc use these settings to fetch the grapher configs. +# This means that for most local dev it is not necessary to set these. +GRAPHER_CONFIG_R2_BUCKET= # optional - for local dev set it to "owid-grapher-configs-staging" +GRAPHER_CONFIG_R2_BUCKET_PATH= # optional - for local dev set it to "devs/YOURNAME" + OPENAI_API_KEY= diff --git a/.github/workflows/check-default-grapher-config.yml b/.github/workflows/check-default-grapher-config.yml new file mode 100644 index 00000000000..c1c56fdae95 --- /dev/null +++ b/.github/workflows/check-default-grapher-config.yml @@ -0,0 +1,53 @@ +name: Check default grapher config +on: + push: + branches: + - "**" + - "!master" + paths: + - "packages/@ourworldindata/grapher/src/schema/**" + +jobs: + commit-default-config: + runs-on: ubuntu-latest + + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - uses: ./.github/actions/setup-node-yarn-deps + - uses: ./.github/actions/build-tsc + + - uses: hustcer/setup-nu@v3 + with: + version: "0.80" # Don't use 0.80 here, as it was a float number and will be convert to 0.8, you can use v0.80/0.80.0 or '0.80' + + # Turn all yaml files in the schema directory into json (should only be one) + - name: Convert yaml schema to json + run: | + (ls packages/@ourworldindata/grapher/src/schema/*.yaml + | each {|yaml| + open $yaml.name + | to json + | save -f ($yaml.name + | path parse + | upsert extension "json" + | path join) }) + shell: nu {0} + + # Construct default config objects for all grapher schemas in the schema directory (should only be one) + - name: Generate default grapher config + run: | + (ls packages/@ourworldindata/grapher/src/schema/grapher-schema.*.json + | each {|json| + node itsJustJavascript/devTools/schema/generate-default-object-from-schema.js $json.name --save-ts packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts }) + shell: nu {0} + + - name: Run prettier + run: yarn fixPrettierAll + + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "🤖 update default grapher config" diff --git a/.gitignore b/.gitignore index 388475e6261..526e12ee4b2 100755 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ dist/ .nx/workspace-data .dev.vars **/tsup.config.bundled*.mjs +cfstorage diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index d13047ff5e0..d79c9dee9fa 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -14,6 +14,10 @@ import { ChartRedirect, DimensionProperty, Json, + GrapherInterface, + diffGrapherConfigs, + isEqual, + omit, } from "@ourworldindata/utils" import { computed, observable, runInAction, when } from "mobx" import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js" @@ -97,6 +101,7 @@ export interface ChartEditorManager { admin: Admin grapher: Grapher database: EditorDatabase + parentGrapherConfig: GrapherInterface logs: Log[] references: References | undefined redirects: ChartRedirect[] @@ -124,7 +129,7 @@ export class ChartEditor { @observable.ref errorMessage?: { title: string; content: string } @observable.ref previewMode: "mobile" | "desktop" @observable.ref showStaticPreview = false - @observable.ref savedGrapherJson: string = "" + @observable.ref savedPatchConfig: GrapherInterface = {} // This gets set when we save a new chart for the first time // so the page knows to update the url @@ -137,13 +142,26 @@ export class ChartEditor { ? "mobile" : "desktop" when( - () => this.grapher.isReady, - () => (this.savedGrapherJson = JSON.stringify(this.grapher.object)) + () => this.grapher.hasData && this.grapher.isReady, + () => (this.savedPatchConfig = this.patchConfig) ) } + @computed get fullConfig(): GrapherInterface { + return this.grapher.object + } + + @computed get patchConfig(): GrapherInterface { + const { parentGrapherConfig } = this.manager + if (!parentGrapherConfig) return this.fullConfig + return diffGrapherConfigs(this.fullConfig, parentGrapherConfig) + } + @computed get isModified(): boolean { - return JSON.stringify(this.grapher.object) !== this.savedGrapherJson + return !isEqual( + omit(this.patchConfig, "version"), + omit(this.savedPatchConfig, "version") + ) } @computed get grapher() { @@ -238,12 +256,12 @@ export class ChartEditor { if (isNewGrapher) { this.newChartId = json.chartId this.grapher.id = json.chartId - this.savedGrapherJson = JSON.stringify(this.grapher.object) + this.savedPatchConfig = json.savedPatch } else { runInAction(() => { grapher.version += 1 this.logs.unshift(json.newLog) - this.savedGrapherJson = JSON.stringify(currentGrapherObject) + this.savedPatchConfig = json.savedPatch }) } } else onError?.() diff --git a/adminSiteClient/ChartEditorPage.tsx b/adminSiteClient/ChartEditorPage.tsx index e1d1a49fc82..b16eeac4229 100644 --- a/adminSiteClient/ChartEditorPage.tsx +++ b/adminSiteClient/ChartEditorPage.tsx @@ -28,7 +28,7 @@ import { ChartRedirect, DimensionProperty, } from "@ourworldindata/types" -import { Grapher } from "@ourworldindata/grapher" +import { defaultGrapherConfig, Grapher } from "@ourworldindata/grapher" import { Admin } from "./Admin.js" import { ChartEditor, @@ -104,7 +104,7 @@ export class ChartEditorPage extends React.Component<{ grapherId?: number newGrapherIndex?: number - grapherConfig?: any + grapherConfig?: GrapherInterface }> implements ChartEditorManager { @@ -124,7 +124,9 @@ export class ChartEditorPage @observable simulateVisionDeficiency?: VisionDeficiency - fetchedGrapherConfig?: any + fetchedGrapherConfig?: GrapherInterface + // for now, every chart's parent config is the default layer + parentGrapherConfig = defaultGrapherConfig async fetchGrapher(): Promise { const { grapherId } = this.props diff --git a/adminSiteClient/EditorHistoryTab.tsx b/adminSiteClient/EditorHistoryTab.tsx index 0fa832b4acf..4c0e8baf5f8 100644 --- a/adminSiteClient/EditorHistoryTab.tsx +++ b/adminSiteClient/EditorHistoryTab.tsx @@ -129,7 +129,7 @@ export class EditorHistoryTab extends React.Component<{ editor: ChartEditor }> { @action.bound copyYamlToClipboard() { // Use the Clipboard API to copy the config into the users clipboard const chartConfigObject = { - ...this.props.editor.grapher.object, + ...this.props.editor.patchConfig, } delete chartConfigObject.id delete chartConfigObject.dimensions @@ -149,7 +149,7 @@ export class EditorHistoryTab extends React.Component<{ editor: ChartEditor }> { // Avoid modifying the original JSON object // Due to mobx memoizing computed values, the JSON can be mutated. const chartConfigObject = { - ...this.props.editor.grapher.object, + ...this.props.editor.patchConfig, } return (
@@ -157,7 +157,12 @@ export class EditorHistoryTab extends React.Component<{ editor: ChartEditor }> { diff --git a/adminSiteClient/GrapherConfigGridEditor.tsx b/adminSiteClient/GrapherConfigGridEditor.tsx index b26aa5edfde..06ebd7ab5c0 100644 --- a/adminSiteClient/GrapherConfigGridEditor.tsx +++ b/adminSiteClient/GrapherConfigGridEditor.tsx @@ -348,6 +348,7 @@ export class GrapherConfigGridEditor extends React.Component => { + // if the schema version is missing, assume it's the latest + if (!config["$schema"]) { + config["$schema"] = defaultGrapherConfig["$schema"] + } + + // compute patch and full configs + const parentConfig = defaultGrapherConfig + const patchConfig = diffGrapherConfigs(config, parentConfig) + const fullConfig = mergeGrapherConfigs(parentConfig, patchConfig) + const fullConfigStringified = JSON.stringify(fullConfig) + + // compute a sha-1 hash of the full config + const fullConfigMd5 = await getMd5HashBase64(fullConfigStringified) + + // insert patch & full configs into the chart_configs table + const chartConfigId = uuidv7() + await db.knexRaw( + knex, + `-- sql + INSERT INTO chart_configs (id, patch, full, fullMd5) + VALUES (?, ?, ?, ?) + `, + [ + chartConfigId, + JSON.stringify(patchConfig), + fullConfigStringified, + fullConfigMd5, + ] + ) + + // add a new chart to the charts table + const result = await db.knexRawInsert( + knex, + `-- sql + INSERT INTO charts (configId, lastEditedAt, lastEditedByUserId) + VALUES (?, ?, ?) + `, + [chartConfigId, new Date(), user.id] + ) + + // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true + const chartId = result.insertId + patchConfig.id = chartId + fullConfig.id = chartId + await db.knexRaw( + knex, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch=JSON_SET(cc.patch, '$.id', ?), + cc.full=JSON_SET(cc.full, '$.id', ?) + WHERE c.id = ? + `, + [chartId, chartId, chartId] + ) + + await saveGrapherConfigToR2ByUUID(chartConfigId, fullConfigStringified) + + return { patchConfig, fullConfig } +} + +const updateExistingChart = async ( + knex: db.KnexReadWriteTransaction, + { + config, + user, + chartId, + }: { config: GrapherInterface; user: DbPlainUser; chartId: number } +): Promise<{ patchConfig: GrapherInterface; fullConfig: GrapherInterface }> => { + // make sure that the id of the incoming config matches the chart id + config.id = chartId + + // if the schema version is missing, assume it's the latest + if (!config["$schema"]) { + config["$schema"] = defaultGrapherConfig["$schema"] + } + + // compute patch and full configs + const parentConfig = defaultGrapherConfig + const patchConfig = diffGrapherConfigs(config, parentConfig) + const fullConfig = mergeGrapherConfigs(parentConfig, patchConfig) + const fullConfigStringified = JSON.stringify(fullConfig) + + const fullConfigMd5 = await getMd5HashBase64(fullConfigStringified) + + const chartConfigId = await db.knexRawFirst>( + knex, + `SELECT configId FROM charts WHERE id = ?`, + [chartId] + ) + + if (!chartConfigId) + throw new JsonError(`No chart config found for id ${chartId}`, 404) + + // update configs + await db.knexRaw( + knex, + `-- sql + UPDATE chart_configs + SET + patch=?, + full=?, + fullMd5=? + WHERE id = ? + `, + [ + JSON.stringify(patchConfig), + fullConfigStringified, + fullConfigMd5, + chartConfigId.configId, + ] + ) + + // update charts row + await db.knexRaw( + knex, + `-- sql + UPDATE charts + SET lastEditedAt=?, lastEditedByUserId=? + WHERE id = ? + `, + [new Date(), user.id, chartId] + ) + + await saveGrapherConfigToR2ByUUID( + chartConfigId.configId, + fullConfigStringified + ) + + return { patchConfig, fullConfig } +} + const saveGrapher = async ( knex: db.KnexReadWriteTransaction, user: DbPlainUser, @@ -288,9 +438,17 @@ const saveGrapher = async ( } async function isSlugUsedInOtherGrapher() { - const rows = await db.knexRaw>( + const rows = await db.knexRaw>( knex, - `SELECT id FROM charts WHERE id != ? AND config->>"$.isPublished" = "true" AND JSON_EXTRACT(config, "$.slug") = ?`, + `-- sql + SELECT c.id + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE + c.id != ? + AND cc.full ->> "$.isPublished" = "true" + AND cc.slug = ? + `, // -1 is a placeholder ID that will never exist; but we cannot use NULL because // in that case we would always get back an empty resultset [existingConfig ? existingConfig.id : -1, newConfig.slug] @@ -326,6 +484,11 @@ const saveGrapher = async ( `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, [existingConfig.id, existingConfig.slug] ) + // When we rename grapher configs, make sure to delete the old one (the new one will be saved below) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${existingConfig.slug}.json` + ) } } @@ -339,32 +502,30 @@ const saveGrapher = async ( else newConfig.version = 1 // Execute the actual database update or creation - const now = new Date() - let chartId = existingConfig && existingConfig.id - const newJsonConfig = JSON.stringify(newConfig) - if (existingConfig) - await db.knexRaw( - knex, - `UPDATE charts SET config=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=? WHERE id = ?`, - [newJsonConfig, now, now, user.id, chartId] - ) - else { - const result = await db.knexRawInsert( - knex, - `INSERT INTO charts (config, createdAt, updatedAt, lastEditedAt, lastEditedByUserId) VALUES (?, ?, ?, ?, ?)`, - [newJsonConfig, now, now, now, user.id] - ) - chartId = result.insertId - // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true - newConfig.id = chartId - await db.knexRaw(knex, `UPDATE charts SET config=? WHERE id = ?`, [ - JSON.stringify(newConfig), + let chartId: number + let patchConfig: GrapherInterface + let fullConfig: GrapherInterface + if (existingConfig) { + chartId = existingConfig.id! + const configs = await updateExistingChart(knex, { + config: newConfig, + user, chartId, - ]) + }) + patchConfig = configs.patchConfig + fullConfig = configs.fullConfig + } else { + const configs = await saveNewChart(knex, { + config: newConfig, + user, + }) + chartId = newConfig.id! + patchConfig = configs.patchConfig + fullConfig = configs.fullConfig } + newConfig = patchConfig // Record this change in version history - const chartRevisionLog = { chartId: chartId as number, userId: user.id, @@ -407,6 +568,17 @@ const saveGrapher = async ( newDimensions.map((d) => d.variableId) ) + if (newConfig.isPublished) { + const configStringified = JSON.stringify(fullConfig) + const configMd5 = await getMd5HashBase64(configStringified) + await saveGrapherConfigToR2( + configStringified, + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${newConfig.slug}.json`, + configMd5 + ) + } + if ( newConfig.isPublished && (!existingConfig || !existingConfig.isPublished) @@ -415,7 +587,7 @@ const saveGrapher = async ( await db.knexRaw( knex, `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `, - [now, user.id, chartId] + [new Date(), user.id, chartId] ) await triggerStaticBuild(user, `Publishing chart ${newConfig.slug}`) } else if ( @@ -429,11 +601,18 @@ const saveGrapher = async ( `DELETE FROM chart_slug_redirects WHERE chart_id = ?`, [existingConfig.id] ) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${existingConfig.slug}.json` + ) await triggerStaticBuild(user, `Unpublishing chart ${newConfig.slug}`) } else if (newConfig.isPublished) await triggerStaticBuild(user, `Updating chart ${newConfig.slug}`) - return chartId + return { + chartId, + savedPatch: newConfig, + } } getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { @@ -441,11 +620,12 @@ getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { const charts = await db.knexRaw( trx, `-- sql - SELECT ${oldChartFieldList} FROM charts - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - ORDER BY charts.lastEditedAt DESC LIMIT ? - `, + SELECT ${oldChartFieldList} FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + ORDER BY charts.lastEditedAt DESC LIMIT ? + `, [limit] ) @@ -461,36 +641,37 @@ getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { const charts = await db.knexRaw( trx, `-- sql - SELECT - charts.id, - charts.config->>"$.version" AS version, - CONCAT("${BAKED_BASE_URL}/grapher/", charts.config->>"$.slug") AS url, - CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl, - charts.config->>"$.slug" AS slug, - charts.config->>"$.title" AS title, - charts.config->>"$.subtitle" AS subtitle, - charts.config->>"$.sourceDesc" AS sourceDesc, - charts.config->>"$.note" AS note, - charts.config->>"$.type" AS type, - charts.config->>"$.internalNotes" AS internalNotes, - charts.config->>"$.variantName" AS variantName, - charts.config->>"$.isPublished" AS isPublished, - charts.config->>"$.tab" AS tab, - JSON_EXTRACT(charts.config, "$.hasChartTab") = true AS hasChartTab, - JSON_EXTRACT(charts.config, "$.hasMapTab") = true AS hasMapTab, - charts.config->>"$.originUrl" AS originUrl, - charts.lastEditedAt, - charts.lastEditedByUserId, - lastEditedByUser.fullName AS lastEditedBy, - charts.publishedAt, - charts.publishedByUserId, - publishedByUser.fullName AS publishedBy - FROM charts - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - ORDER BY charts.lastEditedAt DESC - LIMIT ? - `, + SELECT + charts.id, + chart_configs.full->>"$.version" AS version, + CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url, + CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl, + chart_configs.full->>"$.slug" AS slug, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.subtitle" AS subtitle, + chart_configs.full->>"$.sourceDesc" AS sourceDesc, + chart_configs.full->>"$.note" AS note, + chart_configs.full->>"$.type" AS type, + chart_configs.full->>"$.internalNotes" AS internalNotes, + chart_configs.full->>"$.variantName" AS variantName, + chart_configs.full->>"$.isPublished" AS isPublished, + chart_configs.full->>"$.tab" AS tab, + JSON_EXTRACT(chart_configs.full, "$.hasChartTab") = true AS hasChartTab, + JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab, + chart_configs.full->>"$.originUrl" AS originUrl, + charts.lastEditedAt, + charts.lastEditedByUserId, + lastEditedByUser.fullName AS lastEditedBy, + charts.publishedAt, + charts.publishedByUserId, + publishedByUser.fullName AS publishedBy + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + ORDER BY charts.lastEditedAt DESC + LIMIT ? + `, [limit] ) // note: retrieving references is VERY slow. @@ -708,7 +889,7 @@ apiRouter.get( ) postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { - const chartId = await saveGrapher(trx, res.locals.user, req.body) + const { chartId } = await saveGrapher(trx, res.locals.user, req.body) return { success: true, chartId: chartId } }) @@ -731,10 +912,20 @@ putRouteWithRWTransaction( async (req, res, trx) => { const existingConfig = await expectChartById(trx, req.params.chartId) - await saveGrapher(trx, res.locals.user, req.body, existingConfig) + const { chartId, savedPatch } = await saveGrapher( + trx, + res.locals.user, + req.body, + existingConfig + ) const logs = await getLogsByChartId(trx, existingConfig.id as number) - return { success: true, chartId: existingConfig.id, newLog: logs[0] } + return { + success: true, + chartId, + savedPatch, + newLog: logs[0], + } } ) @@ -759,7 +950,20 @@ deleteRouteWithRWTransaction( `DELETE FROM chart_slug_redirects WHERE chart_id=?`, [chart.id] ) - await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) + + const row = await db.knexRawFirst>( + trx, + `SELECT configId FROM charts WHERE id = ?`, + [chart.id] + ) + if (!row) + throw new JsonError(`No chart config found for id ${chart.id}`, 404) + if (row) { + await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) + await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ + row.configId, + ]) + } if (chart.isPublished) await triggerStaticBuild( @@ -767,6 +971,13 @@ deleteRouteWithRWTransaction( `Deleting chart ${chart.slug}` ) + await deleteGrapherConfigFromR2ByUUID(row.configId) + if (chart.isPublished) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${chart.slug}.json` + ) + return { success: true } } ) @@ -871,7 +1082,7 @@ getRouteWithROTransaction( trx ): Promise> => { const context: OperationContext = { - grapherConfigFieldName: "config", + grapherConfigFieldName: "chart_configs.full", whitelistedColumnNamesAndTypes: chartBulkUpdateAllowedColumnNamesAndTypes, } @@ -889,21 +1100,25 @@ getRouteWithROTransaction( const whereClause = filterSExpr?.toSql() ?? "true" const resultsWithStringGrapherConfigs = await db.knexRaw( trx, - `SELECT charts.id as id, - charts.config as config, - charts.createdAt as createdAt, - charts.updatedAt as updatedAt, - charts.lastEditedAt as lastEditedAt, - charts.publishedAt as publishedAt, - lastEditedByUser.fullName as lastEditedByUser, - publishedByUser.fullName as publishedByUser -FROM charts -LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId -LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId -WHERE ${whereClause} -ORDER BY charts.id DESC -LIMIT 50 -OFFSET ${offset.toString()}` + `-- sql + SELECT + charts.id as id, + chart_configs.full as config, + charts.createdAt as createdAt, + charts.updatedAt as updatedAt, + charts.lastEditedAt as lastEditedAt, + charts.publishedAt as publishedAt, + lastEditedByUser.fullName as lastEditedByUser, + publishedByUser.fullName as publishedByUser + FROM charts + LEFT JOIN chart_configs ON chart_configs.id = charts.configId + LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId + WHERE ${whereClause} + ORDER BY charts.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` ) const results = resultsWithStringGrapherConfigs.map((row: any) => ({ @@ -912,9 +1127,12 @@ OFFSET ${offset.toString()}` })) const resultCount = await db.knexRaw<{ count: number }>( trx, - `SELECT count(*) as count -FROM charts -WHERE ${whereClause}` + `-- sql + SELECT count(*) as count + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + WHERE ${whereClause} + ` ) return { rows: results, numTotalRows: resultCount[0].count } } @@ -928,10 +1146,17 @@ patchRouteWithRWTransaction( const chartIds = new Set(patchesList.map((patch) => patch.id)) const configsAndIds = await db.knexRaw< - Pick - >(trx, `SELECT id, config FROM charts where id IN (?)`, [ - [...chartIds.values()], - ]) + Pick & { config: DbRawChartConfig["full"] } + >( + trx, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE c.id IN (?) + `, + [[...chartIds.values()]] + ) const configMap = new Map( configsAndIds.map((item: any) => [ item.id, @@ -1099,13 +1324,14 @@ getRouteWithROTransaction( const charts = await db.knexRaw( trx, `-- sql - SELECT ${oldChartFieldList} - FROM charts - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - JOIN chart_dimensions cd ON cd.chartId = charts.id - WHERE cd.variableId = ? - GROUP BY charts.id + SELECT ${oldChartFieldList} + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + JOIN chart_dimensions cd ON cd.chartId = charts.id + WHERE cd.variableId = ? + GROUP BY charts.id `, [variableId] ) @@ -1325,15 +1551,16 @@ getRouteWithROTransaction( const charts = await db.knexRaw( trx, `-- sql - SELECT ${oldChartFieldList} - FROM charts - JOIN chart_dimensions AS cd ON cd.chartId = charts.id - JOIN variables AS v ON cd.variableId = v.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE v.datasetId = ? - GROUP BY charts.id - `, + SELECT ${oldChartFieldList} + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN chart_dimensions AS cd ON cd.chartId = charts.id + JOIN variables AS v ON cd.variableId = v.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE v.datasetId = ? + GROUP BY charts.id + `, [datasetId] ) @@ -1506,16 +1733,18 @@ postRouteWithRWTransaction( if (req.body.republish) { await db.knexRaw( trx, - ` - UPDATE charts - SET config = JSON_SET(config, "$.version", config->"$.version" + 1) - WHERE id IN ( - SELECT DISTINCT chart_dimensions.chartId - FROM chart_dimensions - JOIN variables ON variables.id = chart_dimensions.variableId - WHERE variables.datasetId = ? - ) - `, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), + cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) + WHERE c.id IN ( + SELECT DISTINCT chart_dimensions.chartId + FROM chart_dimensions + JOIN variables ON variables.id = chart_dimensions.variableId + WHERE variables.datasetId = ? + )`, [datasetId] ) } @@ -1537,9 +1766,16 @@ getRouteWithROTransaction( redirects: await db.knexRaw( trx, `-- sql - SELECT r.id, r.slug, r.chart_id as chartId, JSON_UNQUOTE(JSON_EXTRACT(charts.config, "$.slug")) AS chartSlug - FROM chart_slug_redirects AS r JOIN charts ON charts.id = r.chart_id - ORDER BY r.id DESC` + SELECT + r.id, + r.slug, + r.chart_id as chartId, + chart_configs.slug AS chartSlug + FROM chart_slug_redirects AS r + JOIN charts ON charts.id = r.chart_id + JOIN chart_configs ON chart_configs.id = charts.configId + ORDER BY r.id DESC + ` ), }) ) @@ -1714,14 +1950,15 @@ getRouteWithROTransaction( const charts = await db.knexRaw( trx, `-- sql - SELECT ${oldChartFieldList} FROM charts - LEFT JOIN chart_tags ct ON ct.chartId=charts.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"} - GROUP BY charts.id - ORDER BY charts.updatedAt DESC - `, + SELECT ${oldChartFieldList} FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + LEFT JOIN chart_tags ct ON ct.chartId=charts.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"} + GROUP BY charts.id + ORDER BY charts.updatedAt DESC + `, uncategorized ? [] : [tagId] ) tag.charts = charts diff --git a/adminSiteServer/chartConfigR2Helpers.ts b/adminSiteServer/chartConfigR2Helpers.ts new file mode 100644 index 00000000000..e2122e24e22 --- /dev/null +++ b/adminSiteServer/chartConfigR2Helpers.ts @@ -0,0 +1,162 @@ +import { + GRAPHER_CONFIG_R2_BUCKET, + GRAPHER_CONFIG_R2_BUCKET_PATH, + R2_ACCESS_KEY_ID, + R2_ENDPOINT, + R2_REGION, + R2_SECRET_ACCESS_KEY, +} from "../settings/serverSettings.js" +import { + DeleteObjectCommand, + DeleteObjectCommandInput, + PutObjectCommand, + PutObjectCommandInput, + S3Client, +} from "@aws-sdk/client-s3" +import { Base64String, JsonError } from "@ourworldindata/utils" +import { R2GrapherConfigDirectory } from "@ourworldindata/types" +import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" +import { createHash } from "crypto" + +export function getMd5HashBase64(data: string): Base64String { + // I would have liked to create a function in utils that can compute a varienty of hashes + // in both the browser, CF workers and node but unfortunately this isn't easily possible + // for md5 - so here we just special case for md5, node and base64 encoding for now. + return createHash("md5") + .update(data, "utf-8") + .digest("base64") as Base64String +} + +let s3Client: S3Client | undefined = undefined + +export async function saveGrapherConfigToR2ByUUID( + id: string, + chartConfigStringified: string +) { + const configMd5 = await getMd5HashBase64(chartConfigStringified) + + await saveGrapherConfigToR2( + chartConfigStringified, + R2GrapherConfigDirectory.byUUID, + `${id}.json`, + configMd5 + ) +} + +export async function deleteGrapherConfigFromR2ByUUID(id: string) { + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.byUUID, + `${id}.json` + ) +} + +export async function saveGrapherConfigToR2( + config_stringified: string, + directory: R2GrapherConfigDirectory, + filename: string, + configMd5: Base64String +) { + if ( + GRAPHER_CONFIG_R2_BUCKET === undefined || + GRAPHER_CONFIG_R2_BUCKET_PATH === undefined + ) { + console.info( + "R2 bucket not configured, not storing grapher config to R2" + ) + return + } + try { + if (!s3Client) { + s3Client = new S3Client({ + endpoint: R2_ENDPOINT, + forcePathStyle: false, + region: R2_REGION, + credentials: { + accessKeyId: R2_ACCESS_KEY_ID, + secretAccessKey: R2_SECRET_ACCESS_KEY, + }, + }) + } + + if (!GRAPHER_CONFIG_R2_BUCKET || !GRAPHER_CONFIG_R2_BUCKET_PATH) { + throw new Error("R2 bucket not configured") + } + + const bucket = GRAPHER_CONFIG_R2_BUCKET + const path = [GRAPHER_CONFIG_R2_BUCKET_PATH, directory, filename].join( + "/" + ) + + const MIMEType = "application/json" + + const params: PutObjectCommandInput = { + Bucket: bucket, + Key: path, + Body: config_stringified, + ContentType: MIMEType, + ContentMD5: configMd5, + } + + await s3Client.send(new PutObjectCommand(params)) + console.log( + `Successfully uploaded object: ${params.Bucket}/${params.Key}` + ) + } catch (err) { + await logErrorAndMaybeSendToBugsnag(err) + throw new JsonError( + `Failed to save the grapher config to R2. Inner error: ${err}` + ) + } +} + +export async function deleteGrapherConfigFromR2( + directory: R2GrapherConfigDirectory, + filename: string +) { + if ( + GRAPHER_CONFIG_R2_BUCKET === undefined || + GRAPHER_CONFIG_R2_BUCKET_PATH === undefined + ) { + console.info( + "R2 bucket not configured, not deleting grapher config to R2" + ) + return + } + try { + if (!s3Client) { + s3Client = new S3Client({ + endpoint: R2_ENDPOINT, + forcePathStyle: false, + region: R2_REGION, + credentials: { + accessKeyId: R2_ACCESS_KEY_ID, + secretAccessKey: R2_SECRET_ACCESS_KEY, + }, + }) + } + + if (!GRAPHER_CONFIG_R2_BUCKET || !GRAPHER_CONFIG_R2_BUCKET_PATH) { + throw new Error("R2 bucket not configured") + } + + const bucket = GRAPHER_CONFIG_R2_BUCKET + const path = [GRAPHER_CONFIG_R2_BUCKET_PATH, directory, filename].join( + "/" + ) + + const params: DeleteObjectCommandInput = { + Bucket: bucket, + Key: path, + } + + await s3Client.send(new DeleteObjectCommand(params)) + console.log( + `Successfully deleted object: ${params.Bucket}/${params.Key}` + ) + } catch (err) { + await logErrorAndMaybeSendToBugsnag(err) + throw new JsonError( + `Failed to delete the grapher config to R2 at ${directory}/${filename}. Inner error: ${err}` + ) + } +} diff --git a/adminSiteServer/testPageRouter.tsx b/adminSiteServer/testPageRouter.tsx index c9a3bd06c29..7db55fc9f88 100644 --- a/adminSiteServer/testPageRouter.tsx +++ b/adminSiteServer/testPageRouter.tsx @@ -24,13 +24,13 @@ import { import { grapherToSVG } from "../baker/GrapherImageBaker.js" import { ChartTypeName, - ChartsTableName, ColorSchemeName, - DbRawChart, + DbRawChartConfig, + DbPlainChart, EntitySelectionMode, GrapherTabOption, StackMode, - parseChartsRow, + parseChartConfig, } from "@ourworldindata/types" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" @@ -140,27 +140,28 @@ async function propsFromQueryParams( let query = knex .table("charts") - .whereRaw("publishedAt IS NOT NULL") - .orderBy("id", "DESC") + .join({ cc: "chart_configs" }, "charts.configId", "cc.id") + .whereRaw("charts.publishedAt IS NOT NULL") + .orderBy("charts.id", "DESC") console.error(query.toSQL()) let tab = params.tab if (params.type) { if (params.type === ChartTypeName.WorldMap) { - query = query.andWhereRaw(`config->>"$.hasMapTab" = "true"`) + query = query.andWhereRaw(`cc.full->>"$.hasMapTab" = "true"`) tab = tab || GrapherTabOption.map } else { if (params.type === "LineChart") { query = query.andWhereRaw( `( - config->"$.type" = "LineChart" - OR config->"$.type" IS NULL - ) AND COALESCE(config->>"$.hasChartTab", "true") = "true"` + cc.full->"$.type" = "LineChart" + OR cc.full->"$.type" IS NULL + ) AND COALESCE(cc.full->>"$.hasChartTab", "true") = "true"` ) } else { query = query.andWhereRaw( - `config->"$.type" = :type AND COALESCE(config->>"$.hasChartTab", "true") = "true"`, + `cc.full->"$.type" = :type AND COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`, { type: params.type } ) } @@ -170,27 +171,27 @@ async function propsFromQueryParams( if (params.logLinear) { query = query.andWhereRaw( - `config->>'$.yAxis.canChangeScaleType' = "true" OR config->>'$.xAxis.canChangeScaleType' = "true"` + `cc.full->>'$.yAxis.canChangeScaleType' = "true" OR cc.full->>'$.xAxis.canChangeScaleType' = "true"` ) tab = GrapherTabOption.chart } if (params.comparisonLines) { query = query.andWhereRaw( - `config->'$.comparisonLines[0].yEquals' != ''` + `cc.full->'$.comparisonLines[0].yEquals' != ''` ) tab = GrapherTabOption.chart } if (params.stackMode) { - query = query.andWhereRaw(`config->'$.stackMode' = :stackMode`, { + query = query.andWhereRaw(`cc.full->'$.stackMode' = :stackMode`, { stackMode: params.stackMode, }) tab = GrapherTabOption.chart } if (params.relativeToggle) { - query = query.andWhereRaw(`config->>'$.hideRelativeToggle' = "false"`) + query = query.andWhereRaw(`cc.full->>'$.hideRelativeToggle' = "false"`) tab = GrapherTabOption.chart } @@ -199,7 +200,7 @@ async function propsFromQueryParams( // have a visible categorial legend, and can leave out some that have one. // But in practice it seems to work reasonably well. query = query.andWhereRaw( - `json_length(config->'$.map.colorScale.customCategoryColors') > 1` + `json_length(cc.full->'$.map.colorScale.customCategoryColors') > 1` ) tab = GrapherTabOption.map } @@ -225,13 +226,13 @@ async function propsFromQueryParams( const mode = params.addCountryMode if (mode === EntitySelectionMode.MultipleEntities) { query = query.andWhereRaw( - `config->'$.addCountryMode' IS NULL OR config->'$.addCountryMode' = :mode`, + `cc.full->'$.addCountryMode' IS NULL OR cc.full->'$.addCountryMode' = :mode`, { mode: EntitySelectionMode.MultipleEntities, } ) } else { - query = query.andWhereRaw(`config->'$.addCountryMode' = :mode`, { + query = query.andWhereRaw(`cc.full->'$.addCountryMode' = :mode`, { mode, }) } @@ -242,10 +243,10 @@ async function propsFromQueryParams( } if (tab === GrapherTabOption.map) { - query = query.andWhereRaw(`config->>"$.hasMapTab" = "true"`) + query = query.andWhereRaw(`cc.full->>"$.hasMapTab" = "true"`) } else if (tab === GrapherTabOption.chart) { query = query.andWhereRaw( - `COALESCE(config->>"$.hasChartTab", "true") = "true"` + `COALESCE(cc.full->>"$.hasChartTab", "true") = "true"` ) } @@ -283,7 +284,7 @@ async function propsFromQueryParams( const chartsQuery = query .clone() - .select("id", "slug") + .select(knex.raw("charts.id, cc.slug")) .limit(perPage) .offset(perPage * (page - 1)) @@ -473,13 +474,26 @@ getPlainRouteWithROTransaction( "/embeds/:id", async (req, res, trx) => { const id = req.params.id - const chartRaw: DbRawChart = await trx - .table(ChartsTableName) - .where({ id: id }) - .first() - const chartEnriched = parseChartsRow(chartRaw) - const viewProps = await getViewPropsFromQueryParams(req.query) - if (chartEnriched) { + const chartRaw = await db.knexRawFirst< + Pick & { config: DbRawChartConfig["full"] } + >( + trx, + `--sql + select ca.id, cc.full as config + from charts ca + join chart_configs cc + on ca.configId = cc.id + where ca.id = ? + `, + [id] + ) + + if (chartRaw) { + const chartEnriched = { + ...chartRaw, + config: parseChartConfig(chartRaw.config), + } + const viewProps = await getViewPropsFromQueryParams(req.query) const charts = [ { id: chartEnriched.id, @@ -727,9 +741,15 @@ getPlainRouteWithROTransaction( testPageRouter, "/previews", async (req, res, trx) => { - const rows = await db.knexRaw( + const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>( trx, - `SELECT config FROM charts LIMIT 200` + `--sql + SELECT cc.full as config + FROM charts ca + JOIN chart_configs cc + ON ca.configId = cc.id + LIMIT 200 + ` ) const charts = rows.map((row: any) => JSON.parse(row.config)) @@ -741,9 +761,15 @@ getPlainRouteWithROTransaction( testPageRouter, "/embedVariants", async (req, res, trx) => { - const rows = await db.knexRaw( + const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>( trx, - `SELECT config FROM charts WHERE id=64` + `--sql + SELECT cc.full as config + FROM charts ca + JOIN chart_configs cc + ON ca.configId = cc.id + WHERE ca.id=64 + ` ) const charts = rows.map((row: any) => JSON.parse(row.config)) const viewProps = getViewPropsFromQueryParams(req.query) diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index b4e78699c43..701b254dbd0 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -42,6 +42,8 @@ import { FaqDictionary, ImageMetadata, OwidGdocBaseInterface, + DbPlainChart, + DbRawChartConfig, } from "@ourworldindata/types" import ProgressBar from "progress" import { @@ -154,6 +156,7 @@ export async function renderDataPageV2( // this is true for preview pages for datapages on the indicator level but false // if we are on Grapher pages. Once we have a good way in the grapher admin for how // to use indicator level defaults, we should reconsider how this works here. + // TODO(inheritance): use mergeGrapherConfigs instead const grapher = useIndicatorGrapherConfigs ? mergePartialGrapherConfigs(grapherConfigForVariable, pageGrapher) : pageGrapher ?? {} @@ -467,16 +470,24 @@ export const bakeSingleGrapherChart = async ( export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers = // TODO: this transaction is only RW because somewhere inside it we fetch images async (bakedSiteDir: string, knex: db.KnexReadWriteTransaction) => { - const chartsToBake: { id: number; config: string; slug: string }[] = - await knexRaw( - knex, - `-- sql - SELECT - id, config, config->>'$.slug' as slug - FROM charts WHERE JSON_EXTRACT(config, "$.isPublished")=true - ORDER BY JSON_EXTRACT(config, "$.slug") ASC + const chartsToBake = await knexRaw< + Pick & { + config: DbRawChartConfig["full"] + slug: string + } + >( + knex, + `-- sql + SELECT + c.id, + cc.full as config, + cc.slug + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE JSON_EXTRACT(cc.full, "$.isPublished")=true + ORDER BY cc.slug ASC ` - ) + ) const newSlugs = chartsToBake.map((row) => row.slug) await fs.mkdirp(bakedSiteDir + "/grapher") diff --git a/baker/GrapherBakingUtils.ts b/baker/GrapherBakingUtils.ts index f44c67083c0..b7f80eee83c 100644 --- a/baker/GrapherBakingUtils.ts +++ b/baker/GrapherBakingUtils.ts @@ -82,7 +82,12 @@ export const bakeGrapherUrls = async ( const rows = await db.knexRaw<{ version: number }>( knex, - `SELECT charts.config->>"$.version" AS version FROM charts WHERE charts.id=?`, + `-- sql + SELECT cc.full->>"$.version" AS version + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE c.id=? + `, [chartId] ) if (!rows.length) { diff --git a/baker/GrapherImageBaker.tsx b/baker/GrapherImageBaker.tsx index 173ae4449df..4ecbea8d799 100644 --- a/baker/GrapherImageBaker.tsx +++ b/baker/GrapherImageBaker.tsx @@ -1,7 +1,8 @@ import { DbPlainChartSlugRedirect, - DbRawChart, + DbPlainChart, GrapherInterface, + DbRawChartConfig, } from "@ourworldindata/types" import { Grapher, GrapherProgrammaticInterface } from "@ourworldindata/grapher" import { MultipleOwidVariableDataDimensionsMap } from "@ourworldindata/utils" @@ -79,9 +80,16 @@ export async function getPublishedGraphersBySlug( const graphersById: Map = new Map() // Select all graphers that are published - const sql = `SELECT id, config FROM charts WHERE config->>"$.isPublished" = "true"` - - const query = db.knexRaw>(knex, sql) + const sql = `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.full ->> "$.isPublished" = 'true' + ` + + const query = db.knexRaw< + Pick & { config: DbRawChartConfig["full"] } + >(knex, sql) for (const row of await query) { const grapher = JSON.parse(row.config) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index ed49183ad42..467f8f89fb6 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -720,19 +720,21 @@ export class SiteBaker { knex, `-- sql SELECT - config ->> '$.slug' as slug, - config ->> '$.subtitle' as subtitle, - config ->> '$.note' as note + cc.slug, + cc.full ->> '$.subtitle' as subtitle, + cc.full ->> '$.note' as note FROM - charts + charts c + JOIN + chart_configs cc ON c.configId = cc.id WHERE - JSON_EXTRACT(config, "$.isPublished") = true + JSON_EXTRACT(cc.full, "$.isPublished") = true AND ( - JSON_EXTRACT(config, "$.subtitle") LIKE "%#dod:%" - OR JSON_EXTRACT(config, "$.note") LIKE "%#dod:%" + JSON_EXTRACT(cc.full, "$.subtitle") LIKE "%#dod:%" + OR JSON_EXTRACT(cc.full, "$.note") LIKE "%#dod:%" ) ORDER BY - JSON_EXTRACT(config, "$.slug") ASC + cc.slug ASC ` ) diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts index 8ed80f03cb5..188c42d13fa 100644 --- a/baker/algolia/indexChartsToAlgolia.ts +++ b/baker/algolia/indexChartsToAlgolia.ts @@ -123,19 +123,20 @@ const getChartsRecords = async ( `-- sql WITH indexable_charts_with_entity_names AS ( SELECT c.id, - config ->> "$.slug" AS slug, - config ->> "$.title" AS title, - config ->> "$.variantName" AS variantName, - config ->> "$.subtitle" AS subtitle, - JSON_LENGTH(config ->> "$.dimensions") AS numDimensions, + cc.slug, + cc.full ->> "$.title" AS title, + cc.full ->> "$.variantName" AS variantName, + cc.full ->> "$.subtitle" AS subtitle, + JSON_LENGTH(cc.full ->> "$.dimensions") AS numDimensions, c.publishedAt, c.updatedAt, JSON_ARRAYAGG(e.name) AS entityNames FROM charts c + LEFT JOIN chart_configs cc ON c.configId = cc.id LEFT JOIN charts_x_entities ce ON c.id = ce.chartId LEFT JOIN entities e ON ce.entityId = e.id - WHERE config ->> "$.isPublished" = 'true' - AND isIndexable IS TRUE + WHERE cc.full ->> "$.isPublished" = 'true' + AND c.isIndexable IS TRUE GROUP BY c.id ) SELECT c.id, diff --git a/baker/countryProfiles.tsx b/baker/countryProfiles.tsx index 74f996aa068..3c774d1aed1 100644 --- a/baker/countryProfiles.tsx +++ b/baker/countryProfiles.tsx @@ -7,6 +7,8 @@ import { DbEnrichedVariable, VariablesTableName, parseVariablesRow, + DbRawChartConfig, + parseChartConfig, } from "@ourworldindata/types" import * as lodash from "lodash" import { @@ -45,13 +47,20 @@ const countryIndicatorGraphers = async ( trx: db.KnexReadonlyTransaction ): Promise => bakeCache(countryIndicatorGraphers, async () => { - const graphers = ( - await trx - .table("charts") - .whereRaw( - "publishedAt is not null and config->>'$.isPublished' = 'true' and isIndexable is true" - ) - ).map((c: any) => JSON.parse(c.config)) as GrapherInterface[] + const configs = await db.knexRaw<{ config: DbRawChartConfig["full"] }>( + trx, + `-- sql + SELECT cc.full as config + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE + c.publishedAt is not null + AND cc.full->>'$.isPublished' = 'true' + AND c.isIndexable is true + ` + ) + + const graphers = configs.map((c: any) => parseChartConfig(c.config)) return graphers.filter(checkShouldShowIndicator) }) diff --git a/baker/redirects.ts b/baker/redirects.ts index 12fed31e444..1c45a9d7697 100644 --- a/baker/redirects.ts +++ b/baker/redirects.ts @@ -97,9 +97,11 @@ export const getGrapherRedirectsMap = async ( }>( knex, `-- sql - SELECT chart_slug_redirects.slug as oldSlug, charts.config ->> "$.slug" as newSlug - FROM chart_slug_redirects INNER JOIN charts ON charts.id=chart_id - ` + SELECT chart_slug_redirects.slug as oldSlug, chart_configs.slug as newSlug + FROM chart_slug_redirects + INNER JOIN charts ON charts.id=chart_id + INNER JOIN chart_configs ON chart_configs.id=charts.configId + ` )) as Array<{ oldSlug: string; newSlug: string }> return new Map( diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 547360a807d..3a4ec6635bf 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -46,7 +46,12 @@ import { OwidGdocDataInsightInterface, } from "@ourworldindata/utils" import { extractFormattingOptions } from "../serverUtils/wordpressUtils.js" -import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" +import { + DbPlainChart, + DbRawChartConfig, + FormattingOptions, + GrapherInterface, +} from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" import { @@ -104,15 +109,16 @@ export const renderChartsPage = async ( knex, `-- sql SELECT - id, - config->>"$.slug" AS slug, - config->>"$.title" AS title, - config->>"$.variantName" AS variantName - FROM charts + c.id, + cc.slug, + cc.full->>"$.title" AS title, + cc.full->>"$.variantName" AS variantName + FROM charts c + JOIN chart_configs cc ON c.configId=cc.id WHERE - isIndexable IS TRUE - AND publishedAt IS NOT NULL - AND config->>"$.isPublished" = "true" + c.isIndexable IS TRUE + AND c.publishedAt IS NOT NULL + AND cc.full->>"$.isPublished" = "true" ` ) @@ -734,9 +740,16 @@ export const renderExplorerPage = async ( type ChartRow = { id: number; config: string } let grapherConfigRows: ChartRow[] = [] if (requiredGrapherIds.length) - grapherConfigRows = await knexRaw( + grapherConfigRows = await knexRaw< + Pick & { config: DbRawChartConfig["full"] } + >( knex, - `SELECT id, config FROM charts WHERE id IN (?)`, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON c.configId=cc.id + WHERE c.id IN (?) + `, [requiredGrapherIds] ) @@ -788,6 +801,7 @@ export const renderExplorerPage = async ( config: row.grapherConfigETL as string, }) : {} + // TODO(inheritance): use mergeGrapherConfigs instead return mergePartialGrapherConfigs(etlConfig, adminConfig) }) diff --git a/baker/sitemap.ts b/baker/sitemap.ts index 8ddcd6e39c8..d41ac7688bd 100644 --- a/baker/sitemap.ts +++ b/baker/sitemap.ts @@ -7,7 +7,7 @@ import { dayjs, countries, queryParamsToStr, - ChartsTableName, + DbPlainChart, } from "@ourworldindata/utils" import * as db from "../db/db.js" import urljoin from "url-join" @@ -82,13 +82,18 @@ export const makeSitemap = async ( publishedDataInsights.length ) - const charts = (await knex - .table(ChartsTableName) - .select(knex.raw(`updatedAt, config->>"$.slug" AS slug`)) - .whereRaw('config->"$.isPublished" = true')) as { - updatedAt: Date - slug: string - }[] + const charts = await db.knexRaw< + Pick & { slug: string } + >( + knex, + `-- sql + SELECT c.updatedAt, cc.slug + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE + cc.full->"$.isPublished" = true + ` + ) const explorers = await explorerAdminServer.getAllPublishedExplorers() diff --git a/baker/updateChartEntities.ts b/baker/updateChartEntities.ts index 412b80ec361..6589978f6d9 100644 --- a/baker/updateChartEntities.ts +++ b/baker/updateChartEntities.ts @@ -6,13 +6,13 @@ import { Grapher } from "@ourworldindata/grapher" import { - ChartsTableName, ChartsXEntitiesTableName, - DbRawChart, + DbPlainChart, GrapherInterface, GrapherTabOption, MultipleOwidVariableDataDimensionsMap, OwidVariableDataMetadataDimensions, + DbRawChartConfig, } from "@ourworldindata/types" import * as db from "../db/db.js" import pMap from "p-map" @@ -41,13 +41,15 @@ const preFetchCommonVariables = async ( const commonVariables = (await db.knexRaw( trx, `-- sql - SELECT variableId, COUNT(variableId) AS useCount - FROM chart_dimensions cd - JOIN charts c ON cd.chartId = c.id - WHERE config ->> "$.isPublished" = "true" - GROUP BY variableId - ORDER BY COUNT(variableId) DESC - LIMIT ??`, + SELECT variableId, COUNT(variableId) AS useCount + FROM chart_dimensions cd + JOIN charts c ON cd.chartId = c.id + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.full ->> "$.isPublished" = "true" + GROUP BY variableId + ORDER BY COUNT(variableId) DESC + LIMIT ?? + `, [VARIABLES_TO_PREFETCH] )) as { variableId: number; useCount: number }[] @@ -123,13 +125,17 @@ const obtainAvailableEntitiesForAllGraphers = async ( ) => { const entityNameToIdMap = await mapEntityNamesToEntityIds(trx) - const allPublishedGraphers = (await trx - .select("id", "config") - .from(ChartsTableName) - .whereRaw("config ->> '$.isPublished' = 'true'")) as Pick< - DbRawChart, - "id" | "config" - >[] + const allPublishedGraphers = await db.knexRaw< + Pick & { config: DbRawChartConfig["full"] } + >( + trx, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.full ->> "$.isPublished" = 'true' + ` + ) const availableEntitiesByChartId = new Map() await pMap( diff --git a/db/db.ts b/db/db.ts index f08670e38a1..71726c286c4 100644 --- a/db/db.ts +++ b/db/db.ts @@ -304,10 +304,12 @@ export const getTotalNumberOfCharts = ( ): Promise => { return knexRawFirst<{ count: number }>( knex, + `-- sql + SELECT COUNT(*) AS count + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.full ->> "$.isPublished" = "true" ` - SELECT COUNT(*) AS count - FROM charts - WHERE config->"$.isPublished" = TRUE` ).then((res) => res?.count ?? 0) } diff --git a/db/migration/1719842654592-AddChartConfigsTable.ts b/db/migration/1719842654592-AddChartConfigsTable.ts new file mode 100644 index 00000000000..b2e84bfe643 --- /dev/null +++ b/db/migration/1719842654592-AddChartConfigsTable.ts @@ -0,0 +1,143 @@ +import { MigrationInterface, QueryRunner } from "typeorm" +import { uuidv7 } from "uuidv7" +export class AddChartConfigsTable1719842654592 implements MigrationInterface { + private async createChartConfigsTable( + queryRunner: QueryRunner + ): Promise { + await queryRunner.query(`-- sql + CREATE TABLE chart_configs ( + id char(36) NOT NULL PRIMARY KEY, + patch json NOT NULL, + full json NOT NULL, + slug varchar(255) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(full, '$.slug'))) STORED, + createdAt datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP, + INDEX idx_chart_configs_slug (slug) + ) + `) + } + + private async createConfigIdColumnInChartsTable( + queryRunner: QueryRunner + ): Promise { + // add a new `configId` column to the charts table + // that points to the `chart_configs` table + await queryRunner.query(`-- sql + ALTER TABLE charts + ADD COLUMN configId char(36) UNIQUE AFTER type, + ADD CONSTRAINT charts_configId + FOREIGN KEY (configId) + REFERENCES chart_configs (id) + ON DELETE RESTRICT + ON UPDATE RESTRICT + `) + } + + private async moveConfigsToChartConfigsTable( + queryRunner: QueryRunner + ): Promise { + // make sure that the config's id matches the table's primary key + await queryRunner.query(`-- sql + UPDATE charts + SET config = JSON_REPLACE(config, '$.id', id) + WHERE id != config ->> "$.id"; + `) + + const chartConfigs: { config: string }[] = + await queryRunner.query(`-- sql + select config from charts`) + + // I tried to write this as a chunked builk insert of 500 at a time but + // failed to get it to work without doing strange things. We only run this once + // for ~5000 items so it's not too bad to do it one insert at a time + for (const chartConfig of chartConfigs) { + await queryRunner.query( + `-- sql + INSERT INTO chart_configs (id, patch, full) + VALUES (?, ?, ?)`, + [uuidv7(), chartConfig.config, chartConfig.config] + ) + } + + // update the `configId` column in the `charts` table + await queryRunner.query(`-- sql + UPDATE charts ca + JOIN chart_configs cc + ON ca.id = cc.full ->> '$.id' + SET ca.configId = cc.id + `) + + // now that the `configId` column is filled, make it NOT NULL + await queryRunner.query(`-- sql + ALTER TABLE charts + MODIFY COLUMN configId char(36) NOT NULL; + `) + + // update `createdAt` and `updatedAt` of the chart_configs table + await queryRunner.query(`-- sql + UPDATE chart_configs cc + JOIN charts ca + ON cc.id = ca.configId + SET + cc.createdAt = ca.createdAt, + cc.updatedAt = ca.updatedAt + `) + } + + private async dropConfigColumnFromChartsTable( + queryRunner: QueryRunner + ): Promise { + await queryRunner.query(`-- sql + ALTER TABLE charts + DROP COLUMN slug, + DROP COLUMN type, + DROP COLUMN config + `) + } + + public async up(queryRunner: QueryRunner): Promise { + await this.createChartConfigsTable(queryRunner) + await this.createConfigIdColumnInChartsTable(queryRunner) + await this.moveConfigsToChartConfigsTable(queryRunner) + await this.dropConfigColumnFromChartsTable(queryRunner) + } + + public async down(queryRunner: QueryRunner): Promise { + // add back the config column and its virtual columns + await queryRunner.query(`-- sql + ALTER TABLE charts + ADD COLUMN config JSON AFTER configId, + ADD COLUMN slug VARCHAR(255) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(config, '$.slug'))) VIRTUAL AFTER config, + ADD COLUMN type VARCHAR(255) GENERATED ALWAYS AS (COALESCE(JSON_UNQUOTE(JSON_EXTRACT(config, '$.type')), 'LineChart')) VIRTUAL AFTER slug + `) + + await queryRunner.query(`-- sql + CREATE INDEX idx_charts_slug ON charts (slug) + `) + + // recover configs + await queryRunner.query(`-- sql + UPDATE charts c + JOIN chart_configs cc ON c.configId = cc.id + SET c.config = cc.full + `) + + // make the config column NOT NULL + await queryRunner.query(`-- sql + ALTER TABLE charts + MODIFY COLUMN config JSON NOT NULL; + `) + + // drop the `charts.configId` column + await queryRunner.query(`-- sql + ALTER TABLE charts + DROP FOREIGN KEY charts_configId, + DROP COLUMN configId + `) + + // drop the `chart_configs` table + await queryRunner.query(`-- sql + DROP TABLE chart_configs + `) + } +} diff --git a/db/migration/1720600092980-MakeChartsInheritDefaults.ts b/db/migration/1720600092980-MakeChartsInheritDefaults.ts new file mode 100644 index 00000000000..93edf367602 --- /dev/null +++ b/db/migration/1720600092980-MakeChartsInheritDefaults.ts @@ -0,0 +1,59 @@ +import { MigrationInterface, QueryRunner } from "typeorm" +import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils" +import { defaultGrapherConfig } from "@ourworldindata/grapher" + +export class MakeChartsInheritDefaults1720600092980 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + const charts = (await queryRunner.query( + `-- sql + SELECT id, patch as config FROM chart_configs + ` + )) as { id: string; config: string }[] + + for (const chart of charts) { + const originalConfig = JSON.parse(chart.config) + + // if the schema version is missing, assume it's the latest + if (!originalConfig["$schema"]) { + originalConfig["$schema"] = defaultGrapherConfig["$schema"] + } + + const patchConfig = diffGrapherConfigs( + originalConfig, + defaultGrapherConfig + ) + const fullConfig = mergeGrapherConfigs( + defaultGrapherConfig, + patchConfig + ) + + await queryRunner.query( + `-- sql + UPDATE chart_configs + SET + patch = ?, + full = ? + WHERE id = ? + `, + [ + JSON.stringify(patchConfig), + JSON.stringify(fullConfig), + chart.id, + ] + ) + } + } + + public async down(queryRunner: QueryRunner): Promise { + // we can't recover the original configs, + // but the patched one is the next best thing + await queryRunner.query( + `-- sql + UPDATE chart_configs + SET full = patch + ` + ) + } +} diff --git a/db/migration/1722415645057-AddChartConfigHash.ts b/db/migration/1722415645057-AddChartConfigHash.ts new file mode 100644 index 00000000000..8885900a088 --- /dev/null +++ b/db/migration/1722415645057-AddChartConfigHash.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class AddChartConfigHash1722415645057 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE chart_configs + ADD COLUMN fullMd5 CHAR(24); + `) + + await queryRunner.query(` + UPDATE chart_configs + SET fullMd5 = to_base64(unhex(md5(full))) + `) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE chart_configs + DROP COLUMN fullMd5; + `) + } +} diff --git a/db/model/Chart.ts b/db/model/Chart.ts index 9ed8bd07b8a..86ece2b0648 100644 --- a/db/model/Chart.ts +++ b/db/model/Chart.ts @@ -12,12 +12,12 @@ import { ChartTypeName, RelatedChart, DbPlainPostLink, - DbRawChart, - DbEnrichedChart, - parseChartsRow, + DbPlainChart, parseChartConfig, ChartRedirect, DbPlainTag, + DbRawChartConfig, + DbEnrichedChartConfig, } from "@ourworldindata/types" import { OpenAI } from "openai" import { @@ -42,12 +42,11 @@ export async function mapSlugsToIds( const rows = await db.knexRaw<{ id: number; slug: string }>( knex, `-- sql - SELECT - id, - JSON_UNQUOTE(JSON_EXTRACT(config, "$.slug")) AS slug - FROM charts - WHERE config->>"$.isPublished" = "true" -` + SELECT c.id, cc.slug + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE cc.full ->> "$.isPublished" = "true" + ` ) const slugToId: { [slug: string]: number } = {} @@ -72,16 +71,17 @@ export async function mapSlugsToConfigs( .knexRaw<{ slug: string; config: string; id: number }>( knex, `-- sql -SELECT csr.slug AS slug, c.config AS config, c.id AS id -FROM chart_slug_redirects csr -JOIN charts c -ON csr.chart_id = c.id -WHERE c.config -> "$.isPublished" = true -UNION -SELECT c.slug AS slug, c.config AS config, c.id AS id -FROM charts c -WHERE c.config -> "$.isPublished" = true -` + SELECT csr.slug AS slug, cc.full AS config, c.id AS id + FROM chart_slug_redirects csr + JOIN charts c ON csr.chart_id = c.id + JOIN chart_configs cc ON cc.id = c.configId + WHERE cc.full ->> "$.isPublished" = "true" + UNION + SELECT cc.slug, cc.full AS config, c.id AS id + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE cc.full ->> "$.isPublished" = "true" + ` ) .then((results) => results.map((result) => ({ @@ -94,31 +94,42 @@ WHERE c.config -> "$.isPublished" = true export async function getEnrichedChartBySlug( knex: db.KnexReadonlyTransaction, slug: string -): Promise { - let chart = await db.knexRawFirst( +): Promise<(DbPlainChart & { config: DbEnrichedChartConfig["full"] }) | null> { + let chart = await db.knexRawFirst< + DbPlainChart & { config: DbRawChartConfig["full"] } + >( knex, - `SELECT * FROM charts WHERE config ->> '$.slug' = ?`, + `-- sql + SELECT c.*, cc.full as config + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.slug = ? + `, [slug] ) if (!chart) { - chart = await db.knexRawFirst( + chart = await db.knexRawFirst< + DbPlainChart & { config: DbRawChartConfig["full"] } + >( knex, `-- sql - SELECT - c.* - FROM - chart_slug_redirects csr - JOIN charts c ON csr.chart_id = c.id - WHERE - csr.slug = ?`, + SELECT + c.*, cc.full as config + FROM + chart_slug_redirects csr + JOIN charts c ON csr.chart_id = c.id + JOIN chart_configs cc ON c.configId = cc.id + WHERE + csr.slug = ? + `, [slug] ) } if (!chart) return null - const enrichedChart = parseChartsRow(chart) + const enrichedChart = { ...chart, config: parseChartConfig(chart.config) } return enrichedChart } @@ -126,10 +137,17 @@ export async function getEnrichedChartBySlug( export async function getRawChartById( knex: db.KnexReadonlyTransaction, id: number -): Promise { - const chart = await db.knexRawFirst( +): Promise<(DbPlainChart & { config: DbRawChartConfig["full"] }) | null> { + const chart = await db.knexRawFirst< + DbPlainChart & { config: DbRawChartConfig["full"] } + >( knex, - `SELECT * FROM charts WHERE id = ?`, + `-- sql + SELECT c.*, cc.full AS config + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE id = ? + `, [id] ) if (!chart) return null @@ -139,19 +157,24 @@ export async function getRawChartById( export async function getEnrichedChartById( knex: db.KnexReadonlyTransaction, id: number -): Promise { +): Promise<(DbPlainChart & { config: DbEnrichedChartConfig["full"] }) | null> { const rawChart = await getRawChartById(knex, id) if (!rawChart) return null - return parseChartsRow(rawChart) + return { ...rawChart, config: parseChartConfig(rawChart.config) } } export async function getChartSlugById( knex: db.KnexReadonlyTransaction, id: number ): Promise { - const chart = await db.knexRawFirst>( + const chart = await db.knexRawFirst<{ slug: string }>( knex, - `SELECT config ->> '$.slug' AS slug FROM charts WHERE id = ?`, + `-- sql + SELECT slug + FROM chart_configs cc + JOIN charts c ON c.configId = cc.id + WHERE c.id = ? + `, [id] ) if (!chart) return null @@ -161,10 +184,20 @@ export async function getChartSlugById( export const getChartConfigById = async ( knex: db.KnexReadonlyTransaction, grapherId: number -): Promise | undefined> => { - const grapher = await db.knexRawFirst>( +): Promise< + | (Pick & { config: DbEnrichedChartConfig["full"] }) + | undefined +> => { + const grapher = await db.knexRawFirst< + Pick & { config: DbRawChartConfig["full"] } + >( knex, - `SELECT id, config FROM charts WHERE id=?`, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE c.id=? + `, [grapherId] ) @@ -179,16 +212,24 @@ export const getChartConfigById = async ( export async function getChartConfigBySlug( knex: db.KnexReadonlyTransaction, slug: string -): Promise> { - const row = await db.knexRawFirst>( +): Promise< + Pick & { config: DbEnrichedChartConfig["full"] } +> { + const row = await db.knexRawFirst< + Pick & { config: DbRawChartConfig["full"] } + >( knex, - `SELECT id, config FROM charts WHERE JSON_EXTRACT(config, "$.slug") = ?`, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.slug = ?`, [slug] ) if (!row) throw new JsonError(`No chart found for slug ${slug}`, 404) - return { id: row.id, config: JSON.parse(row.config) } + return { id: row.id, config: parseChartConfig(row.config) } } export async function setChartTags( @@ -287,11 +328,16 @@ export async function getGptTopicSuggestions( ): Promise[]> { if (!OPENAI_API_KEY) throw new JsonError("No OPENAI_API_KEY env found", 500) - const chartConfigOnly: Pick | undefined = await knex - .table("charts") - .select("config") - .where({ id: chartId }) - .first() + const chartConfigOnly = await db.knexRawFirst<{ config: string }>( + knex, + `-- sql + SELECT cc.full as config + FROM chart_configs cc + JOIN charts c ON c.configId = cc.id + WHERE c.id = ? + `, + [chartId] + ) if (!chartConfigOnly) throw new JsonError(`No chart found for id ${chartId}`, 404) const enrichedChartConfig = parseChartConfig(chartConfigOnly.config) @@ -380,15 +426,15 @@ export interface OldChartFieldList { export const oldChartFieldList = ` charts.id, - charts.config->>"$.title" AS title, - charts.config->>"$.slug" AS slug, - charts.config->>"$.type" AS type, - charts.config->>"$.internalNotes" AS internalNotes, - charts.config->>"$.variantName" AS variantName, - charts.config->>"$.isPublished" AS isPublished, - charts.config->>"$.tab" AS tab, - JSON_EXTRACT(charts.config, "$.hasChartTab") = true AS hasChartTab, - JSON_EXTRACT(charts.config, "$.hasMapTab") = true AS hasMapTab, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.slug" AS slug, + chart_configs.full->>"$.type" AS type, + chart_configs.full->>"$.internalNotes" AS internalNotes, + chart_configs.full->>"$.variantName" AS variantName, + chart_configs.full->>"$.isPublished" AS isPublished, + chart_configs.full->>"$.tab" AS tab, + JSON_EXTRACT(chart_configs.full, "$.hasChartTab") = true AS hasChartTab, + JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab, charts.lastEditedAt, charts.lastEditedByUserId, lastEditedByUser.fullName AS lastEditedBy, @@ -417,15 +463,18 @@ export const getMostViewedGrapherIdsByChartType = async ( ): Promise => { const ids = await db.knexRaw<{ id: number }>( knex, - `SELECT c.id - FROM analytics_pageviews a - JOIN charts c ON c.slug = SUBSTRING_INDEX(a.url, '/', -1) - WHERE a.url LIKE "https://ourworldindata.org/grapher/%" - AND c.type = ? - AND c.config ->> "$.isPublished" = "true" - and (c.config ->> "$.hasChartTab" = "true" or c.config ->> "$.hasChartTab" is null) - ORDER BY a.views_365d DESC - LIMIT ?`, + `-- sql + SELECT c.id + FROM analytics_pageviews a + JOIN chart_configs cc ON slug = SUBSTRING_INDEX(a.url, '/', -1) + JOIN charts c ON c.configId = cc.id + WHERE a.url LIKE "https://ourworldindata.org/grapher/%" + AND cc.full ->> "$.type" = ? + AND cc.full ->> "$.isPublished" = "true" + and (cc.full ->> "$.hasChartTab" = "true" or cc.full ->> "$.hasChartTab" is null) + ORDER BY a.views_365d DESC + LIMIT ? + `, [chartType, count] ) return ids.map((row) => row.id) @@ -444,19 +493,20 @@ export const getRelatedChartsForVariable = async ( return db.knexRaw( knex, `-- sql - SELECT - charts.config->>"$.slug" AS slug, - charts.config->>"$.title" AS title, - charts.config->>"$.variantName" AS variantName, - MAX(chart_tags.keyChartLevel) as keyChartLevel - FROM charts - INNER JOIN chart_tags ON charts.id=chart_tags.chartId - WHERE JSON_CONTAINS(config->'$.dimensions', '{"variableId":${variableId}}') - AND charts.config->>"$.isPublished" = "true" - ${excludeChartIds} - GROUP BY charts.id - ORDER BY title ASC - ` + SELECT + chart_configs.slug, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.variantName" AS variantName, + MAX(chart_tags.keyChartLevel) as keyChartLevel + FROM charts + JOIN chart_configs ON charts.configId=chart_configs.id + INNER JOIN chart_tags ON charts.id=chart_tags.chartId + WHERE JSON_CONTAINS(chart_configs.full->'$.dimensions', '{"variableId":${variableId}}') + AND chart_configs.full->>"$.isPublished" = "true" + ${excludeChartIds} + GROUP BY charts.id + ORDER BY title ASC + ` ) } diff --git a/db/model/Gdoc/GdocPost.ts b/db/model/Gdoc/GdocPost.ts index 712625c11d7..838665a34b9 100644 --- a/db/model/Gdoc/GdocPost.ts +++ b/db/model/Gdoc/GdocPost.ts @@ -179,17 +179,18 @@ export class GdocPost extends GdocBase implements OwidGdocPostInterface { }>( knex, `-- sql - SELECT DISTINCT - charts.config->>"$.slug" AS slug, - charts.config->>"$.title" AS title, - charts.config->>"$.variantName" AS variantName, - chart_tags.keyChartLevel - FROM charts - INNER JOIN chart_tags ON charts.id=chart_tags.chartId - WHERE chart_tags.tagId IN (?) - AND charts.config->>"$.isPublished" = "true" - ORDER BY title ASC - `, + SELECT DISTINCT + chart_configs.slug, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.variantName" AS variantName, + chart_tags.keyChartLevel + FROM charts + JOIN chart_configs ON charts.configId=chart_configs.id + INNER JOIN chart_tags ON charts.id=chart_tags.chartId + WHERE chart_tags.tagId IN (?) + AND chart_configs.full->>"$.isPublished" = "true" + ORDER BY title ASC + `, [this.tags.map((tag) => tag.id)] ) diff --git a/db/model/Image.ts b/db/model/Image.ts index 60799fc9782..7042f529eca 100644 --- a/db/model/Image.ts +++ b/db/model/Image.ts @@ -21,10 +21,10 @@ import { } from "@ourworldindata/utils" import { OwidGoogleAuth } from "../OwidGoogleAuth.js" import { - IMAGE_HOSTING_R2_ENDPOINT, - IMAGE_HOSTING_R2_ACCESS_KEY_ID, - IMAGE_HOSTING_R2_SECRET_ACCESS_KEY, - IMAGE_HOSTING_R2_REGION, + R2_ENDPOINT, + R2_ACCESS_KEY_ID, + R2_SECRET_ACCESS_KEY, + R2_REGION, IMAGE_HOSTING_R2_BUCKET_PATH, GDOCS_CLIENT_EMAIL, GDOCS_SHARED_DRIVE_ID, @@ -139,12 +139,12 @@ class ImageStore { export const imageStore = new ImageStore() export const s3Client = new S3Client({ - endpoint: IMAGE_HOSTING_R2_ENDPOINT, + endpoint: R2_ENDPOINT, forcePathStyle: false, - region: IMAGE_HOSTING_R2_REGION, + region: R2_REGION, credentials: { - accessKeyId: IMAGE_HOSTING_R2_ACCESS_KEY_ID, - secretAccessKey: IMAGE_HOSTING_R2_SECRET_ACCESS_KEY, + accessKeyId: R2_ACCESS_KEY_ID, + secretAccessKey: R2_SECRET_ACCESS_KEY, }, }) diff --git a/db/model/Post.ts b/db/model/Post.ts index adf1a3e1f7f..548fe02e87c 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -233,15 +233,16 @@ export const getPostRelatedCharts = async ( knex, `-- sql SELECT DISTINCT - charts.config->>"$.slug" AS slug, - charts.config->>"$.title" AS title, - charts.config->>"$.variantName" AS variantName, + chart_configs.slug, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.variantName" AS variantName, chart_tags.keyChartLevel FROM charts + JOIN chart_configs ON charts.configId=chart_configs.id INNER JOIN chart_tags ON charts.id=chart_tags.chartId INNER JOIN post_tags ON chart_tags.tagId=post_tags.tag_id WHERE post_tags.post_id=${postId} - AND charts.config->>"$.isPublished" = "true" + AND chart_configs.full->>"$.isPublished" = "true" ORDER BY title ASC ` ) @@ -357,7 +358,8 @@ export const getWordpressPostReferencesByChartId = async ( FROM posts p JOIN posts_links pl ON p.id = pl.sourceId - JOIN charts c ON pl.target = c.slug + JOIN chart_configs cc ON pl.target = cc.slug + JOIN charts c ON c.configId = cc.id OR pl.target IN ( SELECT cr.slug @@ -405,7 +407,8 @@ export const getGdocsPostReferencesByChartId = async ( FROM posts_gdocs pg JOIN posts_gdocs_links pgl ON pg.id = pgl.sourceId - JOIN charts c ON pgl.target = c.slug + JOIN chart_configs cc ON pgl.target = cc.slug + JOIN charts c ON c.configId = cc.id OR pgl.target IN ( SELECT cr.slug @@ -490,7 +493,7 @@ export const getRelatedResearchAndWritingForVariable = async ( SELECT DISTINCT pl.target AS linkTargetSlug, pl.componentType AS componentType, - COALESCE(csr.slug, c.slug) AS chartSlug, + COALESCE(csr.slug, cc.slug) AS chartSlug, p.title AS title, p.slug AS postSlug, COALESCE(csr.chart_id, c.id) AS chartId, @@ -510,7 +513,8 @@ export const getRelatedResearchAndWritingForVariable = async ( FROM posts_links pl JOIN posts p ON pl.sourceId = p.id - LEFT JOIN charts c ON pl.target = c.slug + LEFT JOIN chart_configs cc on pl.target = cc.slug + LEFT JOIN charts c ON cc.id = c.configId LEFT JOIN chart_slug_redirects csr ON pl.target = csr.slug LEFT JOIN chart_dimensions cd ON cd.chartId = COALESCE(csr.chart_id, c.id) LEFT JOIN analytics_pageviews pv ON pv.url = CONCAT('https://ourworldindata.org/', p.slug) @@ -545,7 +549,7 @@ export const getRelatedResearchAndWritingForVariable = async ( SELECT DISTINCT pl.target AS linkTargetSlug, pl.componentType AS componentType, - COALESCE(csr.slug, c.slug) AS chartSlug, + COALESCE(csr.slug, cc.slug) AS chartSlug, p.content ->> '$.title' AS title, p.slug AS postSlug, COALESCE(csr.chart_id, c.id) AS chartId, @@ -565,7 +569,8 @@ export const getRelatedResearchAndWritingForVariable = async ( FROM posts_gdocs_links pl JOIN posts_gdocs p ON pl.sourceId = p.id - LEFT JOIN charts c ON pl.target = c.slug + LEFT JOIN chart_configs cc ON pl.target = cc.slug + LEFT JOIN charts c ON c.configId = cc.id LEFT JOIN chart_slug_redirects csr ON pl.target = csr.slug JOIN chart_dimensions cd ON cd.chartId = COALESCE(csr.chart_id, c.id) LEFT JOIN analytics_pageviews pv ON pv.url = CONCAT('https://ourworldindata.org/', p.slug) diff --git a/db/model/Variable.ts b/db/model/Variable.ts index af2588be4b7..8bcd74abe0e 100644 --- a/db/model/Variable.ts +++ b/db/model/Variable.ts @@ -45,6 +45,7 @@ export async function getMergedGrapherConfigForVariable( const grapherConfigETL = row.grapherConfigETL ? JSON.parse(row.grapherConfigETL) : undefined + // TODO(inheritance): use mergeGrapherConfigs instead return _.merge({}, grapherConfigAdmin, grapherConfigETL) } diff --git a/db/tests/basic.test.ts b/db/tests/basic.test.ts index 1527c78cea3..e50e413edd1 100644 --- a/db/tests/basic.test.ts +++ b/db/tests/basic.test.ts @@ -11,12 +11,15 @@ import { TransactionCloseMode, } from "../db.js" import { deleteUser, insertUser, updateUser } from "../model/User.js" +import { uuidv7 } from "uuidv7" import { ChartsTableName, + ChartConfigsTableName, DbInsertChart, DbPlainUser, - DbRawChart, + DbPlainChart, UsersTableName, + DbInsertChartConfig, } from "@ourworldindata/types" import { cleanTestDb, sleep } from "./testHelpers.js" @@ -68,15 +71,22 @@ test("timestamps are automatically created and updated", async () => { .first() expect(user).toBeTruthy() expect(user.email).toBe("admin@example.com") + const configId = uuidv7() + const chartConfig: DbInsertChartConfig = { + id: configId, + patch: "{}", + full: "{}", + } const chart: DbInsertChart = { - config: "{}", + configId, lastEditedAt: new Date(), lastEditedByUserId: user.id, isIndexable: 0, } + await trx.table(ChartConfigsTableName).insert(chartConfig) const res = await trx.table(ChartsTableName).insert(chart) const chartId = res[0] - const created = await knexRawFirst( + const created = await knexRawFirst( trx, "select * from charts where id = ?", [chartId] @@ -90,7 +100,7 @@ test("timestamps are automatically created and updated", async () => { .table(ChartsTableName) .where({ id: chartId }) .update({ isIndexable: 1 }) - const updated = await knexRawFirst( + const updated = await knexRawFirst( trx, "select * from charts where id = ?", [chartId] diff --git a/devTools/schema/generate-default-object-from-schema.ts b/devTools/schema/generate-default-object-from-schema.ts index b831c70b853..3ef1f66ca00 100644 --- a/devTools/schema/generate-default-object-from-schema.ts +++ b/devTools/schema/generate-default-object-from-schema.ts @@ -44,15 +44,31 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { let schema = fs.readJSONSync(schemaFilename) const defs = schema.$defs || {} - const defaultObject = generateDefaultObjectFromSchema(schema, defs) - process.stdout.write(JSON.stringify(defaultObject, undefined, 2)) + const defaultConfig = generateDefaultObjectFromSchema(schema, defs) + const defaultConfigJSON = JSON.stringify(defaultConfig, undefined, 2) + + // save as ts file if requested + if (parsedArgs["save-ts"]) { + const out = parsedArgs["save-ts"] + const content = `// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY + +// GENERATED BY devTools/schema/generate-default-object-from-schema.ts + +import { GrapherInterface } from "@ourworldindata/types" + +export const defaultGrapherConfig = ${defaultConfigJSON} as GrapherInterface` + fs.outputFileSync(out, content) + } + + // write json to stdout + process.stdout.write(defaultConfigJSON) } function help() { console.log(`generate-default-object-from-schema.ts - utility to generate an object with all default values that are given in a JSON schema Usage: - generate-default-object-from-schema.js `) + generate-default-object-from-schema.js --save-ts `) } const parsedArgs = parseArgs(process.argv.slice(2)) diff --git a/devTools/svgTester/dump-data.ts b/devTools/svgTester/dump-data.ts index 0e08ec6983e..f0d17b5c8d6 100644 --- a/devTools/svgTester/dump-data.ts +++ b/devTools/svgTester/dump-data.ts @@ -1,6 +1,8 @@ #! /usr/bin/env node import { getPublishedGraphersBySlug } from "../../baker/GrapherImageBaker.js" +import { defaultGrapherConfig } from "@ourworldindata/grapher" +import { diffGrapherConfigs } from "@ourworldindata/utils" import { TransactionCloseMode, knexReadonlyTransaction } from "../../db/db.js" @@ -23,7 +25,13 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { ) const allGraphers = [...graphersBySlug.values()] const saveJobs: utils.SaveGrapherSchemaAndDataJob[] = allGraphers.map( - (grapher) => ({ config: grapher, outDir }) + (grapher) => { + // since we're not baking defaults, we also exlcude them here + return { + config: diffGrapherConfigs(grapher, defaultGrapherConfig), + outDir, + } + } ) await pMap(saveJobs, utils.saveGrapherSchemaAndData, { diff --git a/devTools/syncGraphersToR2/syncGraphersToR2.ts b/devTools/syncGraphersToR2/syncGraphersToR2.ts new file mode 100644 index 00000000000..44bf798ee54 --- /dev/null +++ b/devTools/syncGraphersToR2/syncGraphersToR2.ts @@ -0,0 +1,285 @@ +import fs from "fs-extra" +import parseArgs from "minimist" +import { + DeleteObjectCommand, + DeleteObjectCommandInput, + ListObjectsCommand, + ListObjectsV2Command, + ListObjectsV2CommandOutput, + PutObjectCommand, + PutObjectCommandInput, + S3Client, +} from "@aws-sdk/client-s3" +import { + GRAPHER_CONFIG_R2_BUCKET, + GRAPHER_CONFIG_R2_BUCKET_PATH, + R2_ACCESS_KEY_ID, + R2_ENDPOINT, + R2_REGION, + R2_SECRET_ACCESS_KEY, +} from "../../settings/serverSettings.js" +import { + knexRaw, + KnexReadonlyTransaction, + knexReadonlyTransaction, +} from "../../db/db.js" +import { + base64ToBytes, + bytesToBase64, + DbRawChartConfig, + differenceOfSets, + excludeUndefined, + HexString, + hexToBytes, + R2GrapherConfigDirectory, +} from "@ourworldindata/utils" +import { string } from "ts-pattern/dist/patterns.js" +import { chunk, take } from "lodash" +import ProgressBar from "progress" + +type HashAndId = Pick + +/** Sync a set of chart configs with R2. Pass in a map of the keys to their md5 hashes and UUIDs + and this function will upsert all missing/outdated ones and delete any that are no longer needed. + + @param s3Client The S3 client to use + @param pathPrefix The path prefix to use for the files (e.g. "config/by-uuid" then everything inside it will be synced) + @param hashesOfFilesToToUpsert A map of the keys to their md5 hashes and UUIDs + @param trx The transaction to use for querying the DB for full configs + @param dryRun Whether to actually make changes to R2 or just log what would + */ +async function syncWithR2( + s3Client: S3Client, + pathPrefix: string, + hashesOfFilesToToUpsert: Map, + trx: KnexReadonlyTransaction, + dryRun: boolean = false +) { + // We'll first get all the files in the R2 bucket under the path prefix + // and check if the hash of each file that exist in R2 matches the hash + // of the file we want to upsert. If it does, we'll remove it from the + // list of files to upsert. If it doesn't, we'll add it to the list of + // files to delete. + + const hashesOfFilesToDelete = new Map() + + // list the files in the R2 bucket. There may be more files in the + // bucket than can be returned in one list operation so loop until + // all files are listed + let continuationToken: string | undefined = undefined + do { + const listObjectsCommandInput = { + Bucket: GRAPHER_CONFIG_R2_BUCKET, + Prefix: pathPrefix, + ContinuationToken: continuationToken, + } + const listObjectsCommandOutput: ListObjectsV2CommandOutput = + await s3Client.send( + new ListObjectsV2Command(listObjectsCommandInput) + ) + if ((listObjectsCommandOutput.Contents?.length ?? 0) > 0) { + listObjectsCommandOutput.Contents!.forEach((object) => { + if (object.Key && object.ETag) { + // For some reason the etag has quotes around it, strip those + const md5 = object.ETag.replace(/"/g, "") as HexString + const md5Base64 = bytesToBase64(hexToBytes(md5)) + + if (hashesOfFilesToToUpsert.has(object.Key)) { + if ( + hashesOfFilesToToUpsert.get(object.Key)?.fullMd5 === + md5Base64 + ) { + hashesOfFilesToToUpsert.delete(object.Key) + } + // If the existing full config in R2 is different then + // we just keep the hashesOfFilesToToUpsert entry around + // which will upsert the new full config later on + } else { + // if the file in R2 is not in the list of files to upsert + // then we should delete it + hashesOfFilesToDelete.set(object.Key, md5Base64) + } + } + }) + } + continuationToken = listObjectsCommandOutput.NextContinuationToken + } while (continuationToken) + + console.log("Number of files to upsert", hashesOfFilesToToUpsert.size) + console.log("Number of files to delete", hashesOfFilesToDelete.size) + + let progressBar = new ProgressBar( + "--- Deleting obsolete configs [:bar] :current/:total :elapseds\n", + { + total: hashesOfFilesToDelete.size, + } + ) + + // We could parallelize the deletes but it's not worth the complexity for most cases IMHO + for (const [key, _] of hashesOfFilesToDelete.entries()) { + const deleteObjectCommandInput: DeleteObjectCommandInput = { + Bucket: GRAPHER_CONFIG_R2_BUCKET, + Key: key, + } + if (!dryRun) + await s3Client.send( + new DeleteObjectCommand(deleteObjectCommandInput) + ) + else console.log("Would have deleted", key) + progressBar.tick() + } + + console.log("Finished deletes") + + progressBar = new ProgressBar( + "--- Storing missing configs [:bar] :current/:total :elapseds\n", + { + total: hashesOfFilesToToUpsert.size, + } + ) + + const errors = [] + + // Chunk the inserts so that we don't need to keep all the full configs in memory + for (const batch of chunk([...hashesOfFilesToToUpsert.entries()], 100)) { + // Get the full configs for the batch + const fullConfigs = await knexRaw< + Pick + >(trx, `select id, full from chart_configs where id in (?)`, [ + batch.map((entry) => entry[1].id), + ]) + const fullConfigMap = new Map( + fullConfigs.map(({ id, full }) => [id, full]) + ) + + // Upload the full configs to R2 in parallel + const uploadPromises = batch.map(async ([key, val]) => { + const id = val.id + const fullMd5 = val.fullMd5 + const full = fullConfigMap.get(id) + if (full === undefined) { + return Promise.reject( + new Error(`Full config not found for id ${id}`) + ) + } + const putObjectCommandInput: PutObjectCommandInput = { + Bucket: GRAPHER_CONFIG_R2_BUCKET, + Key: key, + Body: full, + ContentMD5: fullMd5, + } + if (!dryRun) + await s3Client.send(new PutObjectCommand(putObjectCommandInput)) + else console.log("Would have upserted", key) + progressBar.tick() + return + }) + const promiseResults = await Promise.allSettled(uploadPromises) + const batchErrors = promiseResults + .filter((result) => result.status === "rejected") + .map((result) => result.reason) + errors.push(...batchErrors) + } + + console.log("Finished upserts") + if (errors.length > 0) { + console.error(`${errors.length} Errors during upserts`) + for (const error of errors) { + console.error(error) + } + } +} + +async function main(parsedArgs: parseArgs.ParsedArgs, dryRun: boolean) { + if ( + GRAPHER_CONFIG_R2_BUCKET === undefined || + GRAPHER_CONFIG_R2_BUCKET_PATH === undefined + ) { + console.info("R2 bucket not configured, exiting") + return + } + + const s3Client = new S3Client({ + endpoint: R2_ENDPOINT, + forcePathStyle: false, + region: R2_REGION, + credentials: { + accessKeyId: R2_ACCESS_KEY_ID, + secretAccessKey: R2_SECRET_ACCESS_KEY, + }, + }) + + const hashesOfFilesToToUpsertBySlug = new Map() + const hashesOfFilesToToUpsertByUuid = new Map() + const pathPrefixBySlug = excludeUndefined([ + GRAPHER_CONFIG_R2_BUCKET_PATH, + R2GrapherConfigDirectory.publishedGrapherBySlug, + ]).join("/") + + const pathPrefixByUuid = excludeUndefined([ + GRAPHER_CONFIG_R2_BUCKET_PATH, + R2GrapherConfigDirectory.byUUID, + ]).join("/") + + await knexReadonlyTransaction(async (trx) => { + // Sync charts published by slug + const slugsAndHashesFromDb = await knexRaw< + Pick + >( + trx, + `select slug, fullMd5, id from chart_configs where slug is not null` + ) + + slugsAndHashesFromDb.forEach((row) => { + hashesOfFilesToToUpsertBySlug.set( + `${pathPrefixBySlug}/${row.slug}.json`, + { + fullMd5: row.fullMd5, + id: row.id, + } + ) + }) + + await syncWithR2( + s3Client, + pathPrefixBySlug, + hashesOfFilesToToUpsertBySlug, + trx, + dryRun + ) + + // Sync charts by UUID + const slugsAndHashesFromDbByUuid = await knexRaw< + Pick + >(trx, `select fullMd5, id from chart_configs`) + + slugsAndHashesFromDbByUuid.forEach((row) => { + hashesOfFilesToToUpsertByUuid.set( + `${pathPrefixByUuid}/${row.id}.json`, + { + fullMd5: row.fullMd5, + id: row.id, + } + ) + }) + + await syncWithR2( + s3Client, + pathPrefixByUuid, + hashesOfFilesToToUpsertByUuid, + trx, + dryRun + ) + }) +} + +const parsedArgs = parseArgs(process.argv.slice(2)) +if (parsedArgs["h"]) { + console.log( + `syncGraphersToR2.js - sync grapher configs from the chart_configs table to R2 + +--dry-run: Don't make any actual changes to R2` + ) +} else { + main(parsedArgs, parsedArgs["dry-run"]) +} diff --git a/devTools/syncGraphersToR2/tsconfig.json b/devTools/syncGraphersToR2/tsconfig.json new file mode 100644 index 00000000000..74f2eaadbb6 --- /dev/null +++ b/devTools/syncGraphersToR2/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../tsconfigs/tsconfig.base.json", + "compilerOptions": { + "outDir": "../../itsJustJavascript/devTools/syncGrapherToR2", + "rootDir": "." + }, + "references": [ + { + "path": "../../db" + }, + { + "path": "../../adminSiteServer" + }, + { + "path": "../../settings" + } + ] +} diff --git a/explorer/GrapherGrammar.ts b/explorer/GrapherGrammar.ts index 28bcac3521c..ed67b32c649 100644 --- a/explorer/GrapherGrammar.ts +++ b/explorer/GrapherGrammar.ts @@ -224,8 +224,7 @@ export const GrapherGrammar: Grammar = { hideRelativeToggle: { ...BooleanCellDef, keyword: "hideRelativeToggle", - description: - "Whether to hide the relative mode UI toggle. Default depends on the chart type.", + description: "Whether to hide the relative mode UI toggle", }, timelineMinTime: { ...IntegerCellDef, diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 249488b75bf..fd0aa7a2c20 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -1,5 +1,10 @@ -import { Grapher, GrapherInterface } from "@ourworldindata/grapher" -import { Bounds, deserializeJSONFromHTML } from "@ourworldindata/utils" +import { Grapher } from "@ourworldindata/grapher" +import { + Bounds, + excludeUndefined, + GrapherInterface, + R2GrapherConfigDirectory, +} from "@ourworldindata/utils" import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm" import { TimeLogger } from "./timeLogger" import { png } from "itty-router" @@ -12,6 +17,37 @@ import LatoMedium from "../_common/fonts/LatoLatin-Medium.ttf.bin" import LatoBold from "../_common/fonts/LatoLatin-Bold.ttf.bin" import PlayfairSemiBold from "../_common/fonts/PlayfairDisplayLatin-SemiBold.ttf.bin" import { Env } from "../grapher/thumbnail/[slug].js" +import { R2GrapherConfigDirectory } from "@ourworldindata/types" +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3" + +// async function getFileFromR2(key: string, env: Env): Promise { +// const s3Client = new S3Client({ +// endpoint: env.R2_ENDPOINT, +// forcePathStyle: false, +// region: env.R2_REGION, +// credentials: { +// accessKeyId: env.R2_ACCESS_KEY_ID, +// secretAccessKey: env.R2_SECRET_ACCESS_KEY, +// }, +// }) + +// const params = { +// Bucket: env.GRAPHER_CONFIG_R2_BUCKET, +// Key: key, +// } + +// try { +// console.log("preparing s3 get") +// const response = await s3Client.send(new GetObjectCommand(params)) +// console.log("got s3 response") +// const content = await response.Body.transformToString() +// console.log("got s3 content") +// return content +// } catch (err) { +// if (err.name === "NoSuchKey") return null +// else throw err +// } +// } declare global { // eslint-disable-next-line no-var @@ -143,13 +179,29 @@ async function fetchAndRenderGrapherToSvg({ }) { const grapherLogger = new TimeLogger("grapher") + const url = new URL(`/grapher/${slug}`, env.url) + const slugOnly = url.pathname.split("/").pop() + + // The top level directory is either the bucket path (should be set in dev environments and production) + // or the branch name on preview staging environments + console.log("branch", env.CF_PAGES_BRANCH) + const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_PATH + ? [env.GRAPHER_CONFIG_R2_BUCKET_PATH] + : ["by-branch", env.CF_PAGES_BRANCH] + + const key = excludeUndefined([ + ...topLevelDirectory, + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${slugOnly}.json`, + ]).join("/") + + console.log("fetching grapher config from this key", key) + + console.log("Fetching", key) + // Fetch grapher config and extract it from the HTML - const grapherConfig: GrapherInterface = await env.ASSETS.fetch( - new URL(`/grapher/${slug}`, env.url) - ) - .then((r) => (r.ok ? r : Promise.reject("Failed to load grapher page"))) - .then((r) => r.text()) - .then((html) => deserializeJSONFromHTML(html)) + const grapherConfigText = await getFileFromR2(key, env) + const grapherConfig: GrapherInterface = JSON.parse(grapherConfigText) if (!grapherConfig) { throw new Error("Could not find grapher config") @@ -206,6 +258,10 @@ export const fetchAndRenderGrapher = async ( env, }) + if (!svg) { + return new Response("Not found", { status: 404 }) + } + switch (outType) { case "png": return png(await renderSvgToPng(svg, options)) diff --git a/functions/grapher/thumbnail/[slug].ts b/functions/grapher/thumbnail/[slug].ts index b8bcbadaae0..5ae30672903 100644 --- a/functions/grapher/thumbnail/[slug].ts +++ b/functions/grapher/thumbnail/[slug].ts @@ -6,6 +6,13 @@ export interface Env { fetch: typeof fetch } url: URL + GRAPHER_CONFIG_R2_BUCKET: string + GRAPHER_CONFIG_R2_BUCKET_PATH: string + R2_ENDPOINT: string + R2_REGION: string + R2_ACCESS_KEY_ID: string + R2_SECRET_ACCESS_KEY: string + CF_PAGES_BRANCH: string } const router = Router() diff --git a/package.json b/package.json index 2fa5500a365..7f2fa516a4f 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "revertLastDbMigration": "tsx --tsconfig tsconfig.tsx.json node_modules/typeorm/cli.js migration:revert -d db/dataSource.ts", "startAdminServer": "node --enable-source-maps ./itsJustJavascript/adminSiteServer/app.js", "startAdminDevServer": "tsx watch --ignore '**.mjs' --tsconfig tsconfig.tsx.json adminSiteServer/app.tsx", - "startLocalCloudflareFunctions": "wrangler pages dev localBake --compatibility-date 2023-10-09", + "startLocalCloudflareFunctions": "wrangler pages dev --local --persist-to ./cfstorage", "startDeployQueueServer": "node --enable-source-maps ./itsJustJavascript/baker/startDeployQueueServer.js", "startLernaWatcher": "lerna watch --scope '@ourworldindata/*' -- lerna run build --scope=\\$LERNA_PACKAGE_NAME --include-dependents", "startTmuxServer": "node_modules/tmex/tmex dev \"yarn startLernaWatcher\" \"yarn startAdminDevServer\" \"yarn startViteServer\"", @@ -39,7 +39,8 @@ "testPrettierAll": "yarn prettier --check \"**/*.{tsx,ts,jsx,js,json,md,html,css,scss,yml}\"", "testJest": "lerna run buildTests && jest", "testSiteNavigation": "tsx --tsconfig tsconfig.tsx.json devTools/navigationTest/navigationTest.ts", - "generateDbTypes": "npx @rmp135/sql-ts -c db/sql-ts/sql-ts-config.json" + "generateDbTypes": "npx @rmp135/sql-ts -c db/sql-ts/sql-ts-config.json", + "syncGraphersToR2": "tsx --tsconfig tsconfig.tsx.json devTools/syncGraphersToR2/syncGraphersToR2.ts" }, "dependencies": { "@algolia/autocomplete-js": "^1.17.2", @@ -165,6 +166,7 @@ "url-parse": "^1.5.10", "url-slug": "^3.0.2", "usehooks-ts": "^3.1.0", + "uuidv7": "^1.0.1", "webfontloader": "^1.6.28", "workerpool": "^6.2.0", "yaml": "^2.4.2" @@ -239,7 +241,7 @@ "tsx": "^4.16.2", "vite": "^5.3.4", "vite-plugin-checker": "^0.7.2", - "wrangler": "^3.61.0" + "wrangler": "^3.68.0" }, "prettier": { "trailingComma": "es5", diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.jsdom.test.ts b/packages/@ourworldindata/grapher/src/core/Grapher.jsdom.test.ts index 1b7d4400b71..96ff4d3a4df 100755 --- a/packages/@ourworldindata/grapher/src/core/Grapher.jsdom.test.ts +++ b/packages/@ourworldindata/grapher/src/core/Grapher.jsdom.test.ts @@ -1,6 +1,6 @@ #! /usr/bin/env jest import { Grapher, GrapherProgrammaticInterface } from "../core/Grapher" -import { DEFAULT_GRAPHER_CONFIG_SCHEMA } from "./GrapherConstants" +import { defaultGrapherConfig } from "../schema/defaultGrapherConfig" import { ChartTypeName, EntitySelectionMode, @@ -76,13 +76,13 @@ it("can get dimension slots", () => { it("an empty Grapher serializes to an object that includes only the schema", () => { expect(new Grapher().toObject()).toEqual({ - $schema: DEFAULT_GRAPHER_CONFIG_SCHEMA, + $schema: defaultGrapherConfig.$schema, }) }) it("a bad chart type does not crash grapher", () => { const input = { - $schema: DEFAULT_GRAPHER_CONFIG_SCHEMA, + $schema: defaultGrapherConfig.$schema, type: "fff" as any, } expect(new Grapher(input).toObject()).toEqual(input) @@ -90,7 +90,7 @@ it("a bad chart type does not crash grapher", () => { it("does not preserve defaults in the object (except for the schema)", () => { expect(new Grapher({ tab: GrapherTabOption.chart }).toObject()).toEqual({ - $schema: DEFAULT_GRAPHER_CONFIG_SCHEMA, + $schema: defaultGrapherConfig.$schema, }) }) diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index ed0b5d920ee..8a46e74a5ae 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -114,7 +114,6 @@ import { BASE_FONT_SIZE, CookieKey, ThereWasAProblemLoadingThisChart, - DEFAULT_GRAPHER_CONFIG_SCHEMA, DEFAULT_GRAPHER_WIDTH, DEFAULT_GRAPHER_HEIGHT, DEFAULT_GRAPHER_FRAME_PADDING, @@ -127,6 +126,7 @@ import { isContinentsVariableId, isPopulationVariableETLPath, } from "../core/GrapherConstants" +import { defaultGrapherConfig } from "../schema/defaultGrapherConfig" import { loadVariableDataAndMetadata } from "./loadVariable" import Cookies from "js-cookie" import { @@ -344,7 +344,7 @@ export class Grapher MapChartManager, SlopeChartManager { - @observable.ref $schema = DEFAULT_GRAPHER_CONFIG_SCHEMA + @observable.ref $schema = defaultGrapherConfig.$schema @observable.ref type = ChartTypeName.LineChart @observable.ref id?: number = undefined @observable.ref version = 1 @@ -527,7 +527,7 @@ export class Grapher deleteRuntimeAndUnchangedProps(obj, defaultObject) // always include the schema, even if it's the default - obj.$schema = this.$schema || DEFAULT_GRAPHER_CONFIG_SCHEMA + obj.$schema = this.$schema || defaultGrapherConfig.$schema // todo: nulls got into the DB for this one. we can remove after moving Graphers from DB. if (obj.stackMode === null) delete obj.stackMode @@ -1000,6 +1000,10 @@ export class Grapher this.selection.setSelectedEntities(this.selectedEntityNames) } + @computed get hasData(): boolean { + return this.dimensions.length > 0 || this.newSlugs.length > 0 + } + // Ready to go iff we have retrieved data for every variable associated with the chart @computed get isReady(): boolean { return this.whatAreWeWaitingFor === "" diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index 35f47981580..820811a58b5 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -9,9 +9,6 @@ export const GRAPHER_TIMELINE_CLASS = "timeline-component" export const GRAPHER_SIDE_PANEL_CLASS = "side-panel" export const GRAPHER_SETTINGS_CLASS = "settings-menu-contents" -export const DEFAULT_GRAPHER_CONFIG_SCHEMA = - "https://files.ourworldindata.org/schemas/grapher-schema.004.json" - export const DEFAULT_GRAPHER_ENTITY_TYPE = "country or region" export const DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL = "countries and regions" diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index d28ffab95d9..9e9d08b9902 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -79,3 +79,4 @@ export { type SlideShowManager, SlideShowController, } from "./slideshowController/SlideShowController" +export { defaultGrapherConfig } from "./schema/defaultGrapherConfig" diff --git a/packages/@ourworldindata/grapher/src/schema/.gitignore b/packages/@ourworldindata/grapher/src/schema/.gitignore new file mode 100644 index 00000000000..b1c3b6f48ad --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/.gitignore @@ -0,0 +1 @@ +grapher-schema.*.json diff --git a/packages/@ourworldindata/grapher/src/schema/README.md b/packages/@ourworldindata/grapher/src/schema/README.md index 19a5afc067b..cad2a6eb595 100644 --- a/packages/@ourworldindata/grapher/src/schema/README.md +++ b/packages/@ourworldindata/grapher/src/schema/README.md @@ -6,13 +6,26 @@ The schema is versioned. There is one yaml file with a version number. For nonbr edit the yaml file as is. A github action will then generate a .latest.yaml and two json files (one .latest.json and one with the version number.json) and upload them to S3 so they can be accessed publicly. +If you update the default value of an existing property or you add a new property with a default value, make sure to regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below). You should also write a migration to update the `chart_configs.full` column in the database for all stand-alone charts. + Breaking changes should be done by renaming the schema file to an increased version number. Make sure to also rename the authorative url inside the schema file (the "$id" field at the top level) to point to the new version number json. Then write the migrations from the last to -the current version of the schema, including the migration of pointing to the URL of the new schema version. Also update `DEFAULT_GRAPHER_CONFIG_SCHEMA` in `GrapherConstants.ts` to point to the new schema version number url. +the current version of the schema, including the migration of pointing to the URL of the new schema version.s Checklist for breaking changes: - Rename the schema file to an increased version number - Rename the authorative url inside the schema file to point to the new version number json - Write the migrations from the last to the current version of the schema, including the migration of pointing to the URL of the new schema version -- Update `DEFAULT_GRAPHER_CONFIG_SCHEMA` in `GrapherConstants.ts` to point to the new schema version number url +- Regenerate the default object from the schema and save it to `defaultGrapherConfig.ts` (see below) +- Write a migration to update the `chart_configs.full` column in the database for all stand-alone charts + +To regenerate `defaultGrapherConfig.ts` from the schema, replace `XXX` with the current schema version number and run: + +```bash +# generate json from the yaml schema +nu -c 'open packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.yaml | to json' > packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.json + +# generate the default object from the schema +node itsJustJavascript/devTools/schema/generate-default-object-from-schema.js packages/@ourworldindata/grapher/src/schema/grapher-schema.XXX.json --save-ts packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts +``` diff --git a/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts b/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts new file mode 100644 index 00000000000..dbbb82d9d5c --- /dev/null +++ b/packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts @@ -0,0 +1,81 @@ +// THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY + +// GENERATED BY devTools/schema/generate-default-object-from-schema.ts + +import { GrapherInterface } from "@ourworldindata/types" + +export const defaultGrapherConfig = { + $schema: "https://files.ourworldindata.org/schemas/grapher-schema.004.json", + map: { + projection: "World", + hideTimeline: false, + colorScale: { + equalSizeBins: true, + binningStrategy: "ckmeans", + customNumericColorsActive: false, + colorSchemeInvert: false, + binningStrategyBinCount: 5, + }, + timeTolerance: 0, + toleranceStrategy: "closest", + tooltipUseCustomLabels: false, + time: "latest", + }, + maxTime: "latest", + yAxis: { + removePointsOutsideDomain: false, + scaleType: "linear", + canChangeScaleType: false, + facetDomain: "shared", + }, + tab: "chart", + matchingEntitiesOnly: false, + hasChartTab: true, + hideLegend: false, + hideLogo: false, + hideTimeline: false, + colorScale: { + equalSizeBins: true, + binningStrategy: "ckmeans", + customNumericColorsActive: false, + colorSchemeInvert: false, + binningStrategyBinCount: 5, + }, + scatterPointLabelStrategy: "year", + selectedFacetStrategy: "none", + isPublished: false, + invertColorScheme: false, + hideRelativeToggle: true, + logo: "owid", + entityType: "country or region", + facettingLabelByYVariables: "metric", + addCountryMode: "add-country", + compareEndPointsOnly: false, + type: "LineChart", + hasMapTab: false, + stackMode: "absolute", + minTime: "earliest", + hideAnnotationFieldsInTitle: { + entity: false, + time: false, + changeInPrefix: false, + }, + xAxis: { + removePointsOutsideDomain: false, + scaleType: "linear", + canChangeScaleType: false, + facetDomain: "shared", + }, + hideConnectedScatterLines: false, + showNoDataArea: true, + zoomToSelection: false, + showYearLabels: false, + hideLinesOutsideTolerance: false, + hideTotalValueLabel: false, + hideScatterLabels: false, + sortBy: "total", + sortOrder: "desc", + hideFacetControl: true, + entityTypePlural: "countries", + missingDataStrategy: "auto", +} as GrapherInterface diff --git a/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml b/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml index 18deb0f59b6..055ce130ead 100644 --- a/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml +++ b/packages/@ourworldindata/grapher/src/schema/grapher-schema.004.yaml @@ -57,7 +57,6 @@ $defs: baseColorScheme: type: string description: One of the predefined base color schemes - default: default enum: - YlGn - YlGnBu @@ -258,7 +257,9 @@ properties: - earliest columnSlug: # TODO: remove this once we have a convention of using the first y dimension instead - description: "Column to show in the map tab. Can be a column slug (e.g. in explorers) or a variable ID (as string)." + description: | + Column to show in the map tab. Can be a column slug (e.g. in explorers) or a variable ID (as string). + If not provided, the first y dimension is used. type: string additionalProperties: false maxTime: @@ -281,7 +282,6 @@ properties: - string baseColorScheme: type: string - default: default description: The default color scheme if no color overrides are specified yAxis: $ref: "#/$defs/axis" @@ -356,7 +356,8 @@ properties: description: Reverse the order of colors in the color scheme hideRelativeToggle: type: boolean - description: Whether to hide the relative mode UI toggle. Default depends on the chart type + default: true + description: Whether to hide the relative mode UI toggle comparisonLines: description: List of vertical comparison lines to draw type: array @@ -375,7 +376,6 @@ properties: type: string version: type: integer - default: 1 minimum: 0 logo: type: string @@ -490,13 +490,16 @@ properties: type: integer description: Number of decimal places to show minimum: 0 + default: 2 numSignificantFigures: type: integer description: Number of significant figures to show minimum: 1 + default: 3 zeroDay: type: string description: Iso date day string for the starting date if yearIsDay is used + default: "2020-01-21" additionalProperties: false variableId: type: integer diff --git a/packages/@ourworldindata/types/src/NominalType.ts b/packages/@ourworldindata/types/src/NominalType.ts index f3487f54232..f24497dfb29 100644 --- a/packages/@ourworldindata/types/src/NominalType.ts +++ b/packages/@ourworldindata/types/src/NominalType.ts @@ -20,3 +20,11 @@ declare const __nominal__type: unique symbol export type Nominal = Type & { readonly [__nominal__type]: Identifier } + +export function wrap(obj: T): Nominal { + return obj as Nominal +} + +export function unwrap(obj: Nominal): T { + return obj +} diff --git a/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts b/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts new file mode 100644 index 00000000000..b1db3c1e82b --- /dev/null +++ b/packages/@ourworldindata/types/src/dbTypes/ChartConfigs.ts @@ -0,0 +1,47 @@ +import { JsonString } from "../domainTypes/Various.js" +import { GrapherInterface } from "../grapherTypes/GrapherTypes.js" + +export const ChartConfigsTableName = "chart_configs" +export interface DbInsertChartConfig { + id: string + patch: JsonString + full: JsonString + fullMd5?: string + slug?: string | null + createdAt?: Date + updatedAt?: Date | null +} +export type DbRawChartConfig = Required + +export type DbEnrichedChartConfig = Omit & { + patch: GrapherInterface + full: GrapherInterface +} + +export function parseChartConfig(config: JsonString): GrapherInterface { + return JSON.parse(config) +} + +export function serializeChartConfig(config: GrapherInterface): JsonString { + return JSON.stringify(config) +} + +export function parseChartConfigsRow( + row: DbRawChartConfig +): DbEnrichedChartConfig { + return { + ...row, + patch: parseChartConfig(row.patch), + full: parseChartConfig(row.full), + } +} + +export function serializeChartsRow( + row: DbEnrichedChartConfig +): DbRawChartConfig { + return { + ...row, + patch: serializeChartConfig(row.patch), + full: serializeChartConfig(row.full), + } +} diff --git a/packages/@ourworldindata/types/src/dbTypes/ChartRevisions.ts b/packages/@ourworldindata/types/src/dbTypes/ChartRevisions.ts index 57db30588d3..4d295fb5802 100644 --- a/packages/@ourworldindata/types/src/dbTypes/ChartRevisions.ts +++ b/packages/@ourworldindata/types/src/dbTypes/ChartRevisions.ts @@ -1,6 +1,6 @@ import { JsonString } from "../domainTypes/Various.js" import { GrapherInterface } from "../grapherTypes/GrapherTypes.js" -import { parseChartConfig, serializeChartConfig } from "./Charts.js" +import { parseChartConfig, serializeChartConfig } from "./ChartConfigs.js" export const ChartRevisionsTableName = "chart_revisions" export interface DbInsertChartRevision { diff --git a/packages/@ourworldindata/types/src/dbTypes/Charts.ts b/packages/@ourworldindata/types/src/dbTypes/Charts.ts index f067113c4f4..baa75642201 100644 --- a/packages/@ourworldindata/types/src/dbTypes/Charts.ts +++ b/packages/@ourworldindata/types/src/dbTypes/Charts.ts @@ -1,9 +1,6 @@ -import { JsonString } from "../domainTypes/Various.js" -import { GrapherInterface } from "../grapherTypes/GrapherTypes.js" - export const ChartsTableName = "charts" export interface DbInsertChart { - config: JsonString + configId: string createdAt?: Date id?: number isIndexable?: number @@ -11,31 +8,6 @@ export interface DbInsertChart { lastEditedByUserId: number publishedAt?: Date | null publishedByUserId?: number | null - slug?: string | null - type?: string | null updatedAt?: Date | null } -export type DbRawChart = Required - -export type DbEnrichedChart = Omit & { - config: GrapherInterface -} - -export function parseChartConfig(config: JsonString): GrapherInterface { - return JSON.parse(config) -} - -export function serializeChartConfig(config: GrapherInterface): JsonString { - return JSON.stringify(config) -} - -export function parseChartsRow(row: DbRawChart): DbEnrichedChart { - return { ...row, config: parseChartConfig(row.config) } -} - -export function serializeChartsRow(row: DbEnrichedChart): DbRawChart { - return { - ...row, - config: serializeChartConfig(row.config), - } -} +export type DbPlainChart = Required diff --git a/packages/@ourworldindata/types/src/domainTypes/Various.ts b/packages/@ourworldindata/types/src/domainTypes/Various.ts index 946339baa14..bc23e990f9d 100644 --- a/packages/@ourworldindata/types/src/domainTypes/Various.ts +++ b/packages/@ourworldindata/types/src/domainTypes/Various.ts @@ -65,3 +65,8 @@ export class JsonError extends Error { export interface QueryParams { [key: string]: string | undefined } + +export enum R2GrapherConfigDirectory { + byUUID = "config/by-uuid", + publishedGrapherBySlug = "grapher/by-slug", +} diff --git a/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts b/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts index 0d067710c0d..a24459de6b4 100644 --- a/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts +++ b/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts @@ -106,6 +106,11 @@ export enum TimeBoundValue { positiveInfinity = Infinity, } +export enum TimeBoundValueStr { + unboundedLeft = "earliest", + unboundedRight = "latest", +} + /** * Time tolerance strategy used for maps */ @@ -509,12 +514,12 @@ export enum MapProjectionName { export interface MapConfigInterface { columnSlug?: ColumnSlug - time?: Time + time?: Time | TimeBoundValueStr timeTolerance?: number toleranceStrategy?: ToleranceStrategy hideTimeline?: boolean projection?: MapProjectionName - colorScale?: ColorScaleConfigInterface + colorScale?: Partial tooltipUseCustomLabels?: boolean } @@ -532,8 +537,8 @@ export interface GrapherInterface extends SortConfig { sourceDesc?: string note?: string hideAnnotationFieldsInTitle?: AnnotationFieldsInTitle - minTime?: TimeBound - maxTime?: TimeBound + minTime?: TimeBound | TimeBoundValueStr + maxTime?: TimeBound | TimeBoundValueStr timelineMinTime?: Time timelineMaxTime?: Time dimensions?: OwidChartDimensionInterface[] diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 3832e2b0bd2..928b89712d1 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -18,6 +18,7 @@ export { type RawPageview, type UserCountryInformation, type QueryParams, + R2GrapherConfigDirectory, } from "./domainTypes/Various.js" export { type BreadcrumbItem, type KeyValueProps } from "./domainTypes/Site.js" export { @@ -53,6 +54,7 @@ export { type ValueRange, type Year, TimeBoundValue, + TimeBoundValueStr, type TimeRange, type Color, type ColumnSlug, @@ -414,6 +416,15 @@ export { type DbPlainAnalyticsPageview, AnalyticsPageviewsTableName, } from "./dbTypes/AnalyticsPageviews.js" +export { + type DbInsertChartConfig, + type DbRawChartConfig, + type DbEnrichedChartConfig, + parseChartConfigsRow, + parseChartConfig, + serializeChartConfig, + ChartConfigsTableName, +} from "./dbTypes/ChartConfigs.js" export { type DbPlainChartDimension, type DbInsertChartDimension, @@ -429,13 +440,8 @@ export { } from "./dbTypes/ChartRevisions.js" export { type DbInsertChart, - type DbRawChart, - type DbEnrichedChart, + type DbPlainChart, ChartsTableName, - parseChartConfig, - serializeChartConfig, - parseChartsRow, - serializeChartsRow, } from "./dbTypes/Charts.js" export { type DbPlainChartSlugRedirect, @@ -642,7 +648,7 @@ export { export { RedirectCode, type DbPlainRedirect } from "./dbTypes/Redirects.js" -export type { Nominal } from "./NominalType.js" +export { type Nominal, wrap, unwrap } from "./NominalType.js" export { type DbRawLatestWork, diff --git a/packages/@ourworldindata/utils/src/TimeBounds.ts b/packages/@ourworldindata/utils/src/TimeBounds.ts index 736fc5c4009..08a46390a5b 100644 --- a/packages/@ourworldindata/utils/src/TimeBounds.ts +++ b/packages/@ourworldindata/utils/src/TimeBounds.ts @@ -3,6 +3,7 @@ import { Time, TimeBound, TimeBoundValue, + TimeBoundValueStr, } from "@ourworldindata/types" import { parseIntOrUndefined, @@ -13,11 +14,6 @@ import { isPositiveInfinity, } from "./Util.js" -enum TimeBoundValueStr { - unboundedLeft = "earliest", - unboundedRight = "latest", -} - export const timeFromTimebounds = ( timeBound: TimeBound, minTime: Time, diff --git a/packages/@ourworldindata/utils/src/Util.test.ts b/packages/@ourworldindata/utils/src/Util.test.ts index c1cd463fac5..a10e4c9efd0 100755 --- a/packages/@ourworldindata/utils/src/Util.test.ts +++ b/packages/@ourworldindata/utils/src/Util.test.ts @@ -29,12 +29,17 @@ import { traverseEnrichedBlock, cartesian, formatInlineList, + base64ToBytes, + bytesToBase64, + hexToBytes, + bytesToHex, } from "./Util.js" import { BlockImageSize, OwidEnrichedGdocBlock, SortOrder, } from "@ourworldindata/types" +import { webcrypto as crypto } from "node:crypto" describe(findClosestTime, () => { describe("without tolerance", () => { @@ -795,3 +800,24 @@ describe(formatInlineList, () => { ) }) }) + +function generateRandomBytes(length: number): Uint8Array { + const bytes = new Uint8Array(length) + crypto.getRandomValues(bytes) + return bytes +} + +describe("hex/base64 conversion is reversible", () => { + const originalBytes = generateRandomBytes(33) + const base64String = bytesToBase64(originalBytes) + const roundTrippedBytes = base64ToBytes(base64String) + it("is the same after converting to base64 and back", () => { + expect(originalBytes).toEqual(roundTrippedBytes) + }) + + const hexString = bytesToHex(originalBytes) + const roundTrippedBytesHex = hexToBytes(hexString) + it("is the same after converting to hex and back", () => { + expect(originalBytes).toEqual(roundTrippedBytesHex) + }) +}) diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 567ba543efe..553cb9f028d 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -38,6 +38,7 @@ import { maxBy, memoize, merge, + mergeWith, min, minBy, noop, @@ -102,6 +103,7 @@ export { isNil, isNull, isNumber, + isPlainObject, isString, isUndefined, keyBy, @@ -110,6 +112,7 @@ export { maxBy, memoize, merge, + mergeWith, min, minBy, noop, @@ -179,10 +182,12 @@ import { TagGraphRoot, TagGraphRootName, TagGraphNode, + Nominal, } from "@ourworldindata/types" import { PointVector } from "./PointVector.js" import React from "react" import { match, P } from "ts-pattern" +// import "crypto" export type NoUndefinedValues = { [P in keyof T]: Required> @@ -459,6 +464,42 @@ export const cagr = ( ) } +export type Base64String = Nominal +export type HexString = Nominal + +export function base64ToBytes(base64: Base64String): Uint8Array { + const binString = atob(base64) + return Uint8Array.from(binString, (m) => { + const cp = m.codePointAt(0) + if (cp === undefined) throw new Error("Invalid base64") + return cp + }) +} + +export function bytesToBase64(bytes: Uint8Array): Base64String { + const binString = Array.from(bytes, (byte) => + String.fromCodePoint(byte) + ).join("") + return btoa(binString) as Base64String +} + +export function hexToBytes(hex: string): Uint8Array { + if (hex.length % 2 !== 0) throw new Error("Invalid hex") + const bytes = new Uint8Array(hex.length / 2) + for (let i = 0; i < hex.length; i += 2) { + const parsed = parseInt(hex.slice(i, i + 2), 16) + if (isNaN(parsed)) throw new Error("Invalid hex") + bytes[i / 2] = parsed + } + return bytes +} + +export function bytesToHex(bytes: Uint8Array): HexString { + return Array.from(bytes) + .map((byte) => byte.toString(16).padStart(2, "0")) + .join("") as HexString +} + export const makeAnnotationsSlug = (columnSlug: string): string => `${columnSlug}-annotations` @@ -1121,6 +1162,39 @@ export const omitNullableValues = (object: T): NoUndefinedValues => { return result } +export function omitUndefinedValuesRecursive>( + obj: T +): NoUndefinedValues { + const result: any = {} + for (const key in obj) { + if (isPlainObject(obj[key])) { + // re-apply the function if we encounter a non-empty object + result[key] = omitUndefinedValuesRecursive(obj[key]) + } else if (obj[key] === undefined) { + // omit undefined values + } else { + // otherwise, keep the value + result[key] = obj[key] + } + } + return result +} + +export function omitEmptyObjectsRecursive>( + obj: T +): Partial { + const result: any = {} + for (const key in obj) { + if (isPlainObject(obj[key])) { + const isObjectEmpty = isEmpty(omitEmptyObjectsRecursive(obj[key])) + if (!isObjectEmpty) result[key] = obj[key] + } else { + result[key] = obj[key] + } + } + return result +} + export const isInIFrame = (): boolean => { try { return window.self !== window.top @@ -1747,7 +1821,7 @@ export function filterValidStringValues( return filteredValues } -// TODO: type this correctly once we have moved types into their own top level package +// TODO(inheritance): remove in favour of mergeGrapherConfigs export function mergePartialGrapherConfigs>( ...grapherConfigs: (T | undefined)[] ): T { @@ -1925,3 +1999,19 @@ export function lazy(fn: () => T): () => T { return _value } } + +export function traverseObjects>( + obj: T, + ref: Record, + cb: (objValue: unknown, refValue: unknown, key: string) => unknown +): Partial { + const result: any = {} + for (const key in obj) { + if (isPlainObject(obj[key]) && isPlainObject(ref[key])) { + result[key] = traverseObjects(obj[key], ref[key], cb) + } else { + result[key] = cb(obj[key], ref[key], key) + } + } + return result +} diff --git a/packages/@ourworldindata/utils/src/grapherConfigUtils.test.ts b/packages/@ourworldindata/utils/src/grapherConfigUtils.test.ts new file mode 100644 index 00000000000..882b0d74326 --- /dev/null +++ b/packages/@ourworldindata/utils/src/grapherConfigUtils.test.ts @@ -0,0 +1,396 @@ +#! /usr/bin/env jest + +import { + DimensionProperty, + GrapherInterface, + GrapherTabOption, + MapProjectionName, +} from "@ourworldindata/types" +import { + mergeGrapherConfigs, + diffGrapherConfigs, +} from "./grapherConfigUtils.js" + +describe(mergeGrapherConfigs, () => { + it("merges empty configs", () => { + expect(mergeGrapherConfigs({}, {})).toEqual({}) + expect(mergeGrapherConfigs({ title: "Parent title" }, {})).toEqual({ + title: "Parent title", + }) + expect(mergeGrapherConfigs({}, { title: "Child title" })).toEqual({ + title: "Child title", + }) + }) + + it("doesn't mutate input objects", () => { + const parentConfig = { title: "Title" } + const childConfig = { subtitle: "Subtitle" } + mergeGrapherConfigs(parentConfig, childConfig) + expect(parentConfig).toEqual({ title: "Title" }) + expect(childConfig).toEqual({ subtitle: "Subtitle" }) + }) + + it("merges two objects", () => { + expect( + mergeGrapherConfigs( + { title: "Parent title" }, + { subtitle: "Child subtitle" } + ) + ).toEqual({ + title: "Parent title", + subtitle: "Child subtitle", + }) + expect( + mergeGrapherConfigs( + { title: "Parent title" }, + { title: "Child title" } + ) + ).toEqual({ title: "Child title" }) + expect( + mergeGrapherConfigs( + { title: "Parent title", subtitle: "Parent subtitle" }, + { title: "Child title", hideRelativeToggle: true } + ) + ).toEqual({ + title: "Child title", + subtitle: "Parent subtitle", + hideRelativeToggle: true, + }) + }) + + it("merges three objects", () => { + expect( + mergeGrapherConfigs( + { title: "Parent title" }, + { subtitle: "Child subtitle" }, + { note: "Grandchild note" } + ) + ).toEqual({ + title: "Parent title", + subtitle: "Child subtitle", + note: "Grandchild note", + }) + expect( + mergeGrapherConfigs( + { + title: "Parent title", + subtitle: "Parent subtitle", + sourceDesc: "Parent sources", + }, + { title: "Child title", subtitle: "Child subtitle" }, + { title: "Grandchild title", note: "Grandchild note" } + ) + ).toEqual({ + title: "Grandchild title", + subtitle: "Child subtitle", + note: "Grandchild note", + sourceDesc: "Parent sources", + }) + }) + + it("merges nested objects", () => { + expect( + mergeGrapherConfigs( + { + map: { + projection: MapProjectionName.World, + time: 2000, + }, + }, + { + map: { + projection: MapProjectionName.Africa, + hideTimeline: true, + }, + } + ) + ).toEqual({ + map: { + projection: MapProjectionName.Africa, + time: 2000, + hideTimeline: true, + }, + }) + }) + + it("overwrites arrays", () => { + expect( + mergeGrapherConfigs( + { selectedEntityNames: ["France", "Italy"] }, + { selectedEntityNames: ["Italy", "Spain"] } + ) + ).toEqual({ + selectedEntityNames: ["Italy", "Spain"], + }) + expect( + mergeGrapherConfigs( + { colorScale: { customNumericValues: [1, 2] } }, + { colorScale: { customNumericValues: [3, 4] } } + ) + ).toEqual({ + colorScale: { customNumericValues: [3, 4] }, + }) + }) + + it("doesn't merge configs of different schema versions", () => { + expect(() => + mergeGrapherConfigs( + { $schema: "1", title: "Title A" }, + { $schema: "2", title: "Title B" } + ) + ).toThrowError() + }) + + it("excludes id, slug, version and isPublished from inheritance", () => { + expect( + mergeGrapherConfigs( + { + $schema: "004", + id: 1, + slug: "parent-slug", + version: 1, + title: "Title A", + }, + { title: "Title B" } + ) + ).toEqual({ title: "Title B" }) + expect( + mergeGrapherConfigs( + { + $schema: "004", + id: 1, + slug: "parent-slug", + version: 1, + title: "Title A", + }, + { slug: "child-slug", version: 1, title: "Title B" } + ) + ).toEqual({ + slug: "child-slug", + version: 1, + title: "Title B", + }) + }) + + it("ignores empty objects", () => { + expect( + mergeGrapherConfigs( + { + title: "Parent title", + subtitle: "Parent subtitle", + }, + { + $schema: "004", + id: 1, + slug: "parent-slug", + version: 1, + title: "Title A", + }, + {} + ) + ).toEqual({ + $schema: "004", + id: 1, + slug: "parent-slug", + version: 1, + title: "Title A", + subtitle: "Parent subtitle", + }) + }) + + it("overwrites values with an empty string if requested", () => { + expect( + mergeGrapherConfigs( + { title: "Parent title", subtitle: "Parent subtitle" }, + { subtitle: "" } + ) + ).toEqual({ title: "Parent title", subtitle: "" }) + }) + + it("is associative", () => { + const configA: GrapherInterface = { + title: "Title A", + subtitle: "Subtitle A", + } + const configB: GrapherInterface = { title: "Title B", note: "Note B" } + const configC: GrapherInterface = { + title: "Title C", + subtitle: "Subtitle C", + sourceDesc: "Source C", + } + expect( + mergeGrapherConfigs(configA, mergeGrapherConfigs(configB, configC)) + ).toEqual( + mergeGrapherConfigs(mergeGrapherConfigs(configA, configB), configC) + ) + expect( + mergeGrapherConfigs(mergeGrapherConfigs(configA, configB), configC) + ).toEqual(mergeGrapherConfigs(configA, configB, configC)) + }) +}) + +describe(diffGrapherConfigs, () => { + it("returns the given config if the reference is empty", () => { + expect(diffGrapherConfigs({ title: "Chart" }, {})).toEqual({ + title: "Chart", + }) + }) + + it("returns the given config if it's empty", () => { + expect(diffGrapherConfigs({}, { title: "Reference chart" })).toEqual({}) + }) + + it("drops redundant entries", () => { + expect( + diffGrapherConfigs( + { tab: GrapherTabOption.map }, + { tab: GrapherTabOption.map } + ) + ).toEqual({}) + expect( + diffGrapherConfigs( + { tab: GrapherTabOption.chart, title: "Chart" }, + { tab: GrapherTabOption.chart, title: "Reference chart" } + ) + ).toEqual({ title: "Chart" }) + }) + + it("diffs nested configs correctly", () => { + expect( + diffGrapherConfigs( + { + title: "Chart", + tab: GrapherTabOption.chart, + map: { + projection: MapProjectionName.World, + hideTimeline: true, + }, + }, + { + title: "Reference chart", + tab: GrapherTabOption.chart, + map: { + projection: MapProjectionName.World, + hideTimeline: false, + }, + } + ) + ).toEqual({ title: "Chart", map: { hideTimeline: true } }) + expect( + diffGrapherConfigs( + { + tab: GrapherTabOption.chart, + map: { + projection: MapProjectionName.World, + hideTimeline: true, + }, + }, + { + tab: GrapherTabOption.chart, + map: { + projection: MapProjectionName.World, + hideTimeline: true, + }, + } + ) + ).toEqual({}) + }) + + it("strips undefined values from the config", () => { + expect( + diffGrapherConfigs( + { + tab: GrapherTabOption.chart, + title: "Chart", + subtitle: undefined, + }, + { tab: GrapherTabOption.chart, title: "Reference chart" } + ) + ).toEqual({ title: "Chart" }) + }) + + it("strips empty objects from the config", () => { + expect(diffGrapherConfigs({ map: {} }, {})).toEqual({}) + expect( + diffGrapherConfigs( + { map: { colorScale: { customCategoryColors: {} } } }, + { map: { colorScale: { colorSchemeInvert: false } } } + ) + ).toEqual({}) + }) + + it("doesn't diff $schema, id, version, slug, isPublished or dimensions", () => { + expect( + diffGrapherConfigs( + { + title: "Chart", + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.004.json", + id: 20, + version: 1, + slug: "slug", + isPublished: false, + dimensions: [ + { property: DimensionProperty.y, variableId: 123456 }, + ], + }, + { + title: "Reference chart", + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.004.json", + id: 20, + version: 1, + slug: "slug", + isPublished: false, + dimensions: [ + { property: DimensionProperty.y, variableId: 123456 }, + ], + } + ) + ).toEqual({ + title: "Chart", + $schema: + "https://files.ourworldindata.org/schemas/grapher-schema.004.json", + id: 20, + version: 1, + slug: "slug", + isPublished: false, + dimensions: [{ property: DimensionProperty.y, variableId: 123456 }], + }) + }) + + it("is idempotent", () => { + const config: GrapherInterface = { + tab: GrapherTabOption.chart, + title: "Chart", + subtitle: undefined, + } + const reference: GrapherInterface = { + tab: GrapherTabOption.chart, + title: "Reference chart", + } + const diffedOnce = diffGrapherConfigs(config, reference) + const diffedTwice = diffGrapherConfigs(diffedOnce, reference) + expect(diffedTwice).toEqual(diffedOnce) + }) +}) + +describe("diff+merge", () => { + it("are consistent", () => { + const config: GrapherInterface = { + tab: GrapherTabOption.chart, + title: "Chart", + subtitle: "Chart subtitle", + } + const reference: GrapherInterface = { + tab: GrapherTabOption.chart, + title: "Reference chart", + } + const diffedAndMerged = mergeGrapherConfigs( + reference, + diffGrapherConfigs(config, reference) + ) + const onlyMerged = mergeGrapherConfigs(reference, config) + expect(diffedAndMerged).toEqual(onlyMerged) + }) +}) diff --git a/packages/@ourworldindata/utils/src/grapherConfigUtils.ts b/packages/@ourworldindata/utils/src/grapherConfigUtils.ts new file mode 100644 index 00000000000..2a8db8e0120 --- /dev/null +++ b/packages/@ourworldindata/utils/src/grapherConfigUtils.ts @@ -0,0 +1,93 @@ +import { GrapherInterface } from "@ourworldindata/types" +import { + isEqual, + mergeWith, + uniq, + omit, + pick, + excludeUndefined, + omitUndefinedValuesRecursive, + omitEmptyObjectsRecursive, + traverseObjects, + isEmpty, +} from "./Util" + +const REQUIRED_KEYS = ["$schema", "dimensions"] + +const KEYS_EXCLUDED_FROM_INHERITANCE = [ + "$schema", + "id", + "slug", + "version", + "isPublished", +] + +export function mergeGrapherConfigs( + ...grapherConfigs: GrapherInterface[] +): GrapherInterface { + const configsToMerge = grapherConfigs.filter((c) => !isEmpty(c)) + + // return early if there are no configs to merge + if (configsToMerge.length === 0) return {} + if (configsToMerge.length === 1) return configsToMerge[0] + + // warn if one of the configs is missing a schema version + const configsWithoutSchema = configsToMerge.filter( + (c) => c["$schema"] === undefined + ) + if (configsWithoutSchema.length > 0) { + const configsJson = JSON.stringify(configsWithoutSchema, null, 2) + console.warn( + `About to merge Grapher configs with missing schema information: ${configsJson}` + ) + } + + // abort if the grapher configs have different schema versions + const uniqueSchemas = uniq( + excludeUndefined(configsToMerge.map((c) => c["$schema"])) + ) + if (uniqueSchemas.length > 1) { + throw new Error( + `Can't merge Grapher configs with different schema versions: ${uniqueSchemas.join( + ", " + )}` + ) + } + + // keys that should not be inherited are removed from all but the last config + const cleanedConfigs = configsToMerge.map((config, index) => { + if (index === configsToMerge.length - 1) return config + return omit(config, KEYS_EXCLUDED_FROM_INHERITANCE) + }) + + return mergeWith( + {}, // mergeWith mutates the first argument + ...cleanedConfigs, + (_: unknown, childValue: unknown): any => { + // don't concat arrays, just use the last one + if (Array.isArray(childValue)) { + return childValue + } + } + ) +} + +export function diffGrapherConfigs( + config: GrapherInterface, + reference: GrapherInterface +): GrapherInterface { + const keepKeys = [...REQUIRED_KEYS, ...KEYS_EXCLUDED_FROM_INHERITANCE] + const keep = pick(config, keepKeys) + + const diffed = omitEmptyObjectsRecursive( + omitUndefinedValuesRecursive( + traverseObjects(config, reference, (value, refValue) => { + if (refValue === undefined) return value + if (!isEqual(value, refValue)) return value + return undefined + }) + ) + ) + + return { ...diffed, ...keep } +} diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index 0cd96a487a0..917a33f7b55 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -20,6 +20,12 @@ export { firstOfNonEmptyArray, lastOfNonEmptyArray, mapToObjectLiteral, + type Base64String, + type HexString, + bytesToBase64, + base64ToBytes, + bytesToHex, + hexToBytes, next, previous, domainExtent, @@ -333,3 +339,8 @@ export { } from "./DonateUtils.js" export { isAndroid, isIOS } from "./BrowserUtils.js" + +export { + diffGrapherConfigs, + mergeGrapherConfigs, +} from "./grapherConfigUtils.js" diff --git a/settings/serverSettings.ts b/settings/serverSettings.ts index c6f3c42cbf2..d945b49173c 100644 --- a/settings/serverSettings.ts +++ b/settings/serverSettings.ts @@ -154,22 +154,29 @@ export const IMAGE_HOSTING_R2_BUCKET_SUBFOLDER_PATH: string = IMAGE_HOSTING_R2_BUCKET_PATH.indexOf("/") + 1 ) // extract R2 credentials from rclone config as defaults -export const IMAGE_HOSTING_R2_ENDPOINT: string = - serverSettings.IMAGE_HOSTING_R2_ENDPOINT || +export const R2_ENDPOINT: string = + serverSettings.R2_ENDPOINT || rcloneConfig["owid-r2"]?.endpoint || "https://078fcdfed9955087315dd86792e71a7e.r2.cloudflarestorage.com" -export const IMAGE_HOSTING_R2_ACCESS_KEY_ID: string = - serverSettings.IMAGE_HOSTING_R2_ACCESS_KEY_ID || +export const R2_ACCESS_KEY_ID: string = + serverSettings.R2_ACCESS_KEY_ID || rcloneConfig["owid-r2"]?.access_key_id || "" -export const IMAGE_HOSTING_R2_SECRET_ACCESS_KEY: string = - serverSettings.IMAGE_HOSTING_R2_SECRET_ACCESS_KEY || +export const R2_SECRET_ACCESS_KEY: string = + serverSettings.R2_SECRET_ACCESS_KEY || rcloneConfig["owid-r2"]?.secret_access_key || "" -export const IMAGE_HOSTING_R2_REGION: string = - serverSettings.IMAGE_HOSTING_R2_REGION || - rcloneConfig["owid-r2"]?.region || - "auto" +export const R2_REGION: string = + serverSettings.R2_REGION || rcloneConfig["owid-r2"]?.region || "auto" + +export const GRAPHER_CONFIG_BASE_URL: string = + serverSettings.GRAPHER_CONFIG_BASE_URL || + "https://ourworldindata.org/grapher/" + +export const GRAPHER_CONFIG_R2_BUCKET: string | undefined = + serverSettings.GRAPHER_CONFIG_R2_BUCKET +export const GRAPHER_CONFIG_R2_BUCKET_PATH: string | undefined = + serverSettings.GRAPHER_CONFIG_R2_BUCKET_PATH export const DATA_API_URL: string = clientSettings.DATA_API_URL diff --git a/site/DataPageV2.tsx b/site/DataPageV2.tsx index 83877927fa5..259f2044379 100644 --- a/site/DataPageV2.tsx +++ b/site/DataPageV2.tsx @@ -1,4 +1,5 @@ import { + defaultGrapherConfig, getVariableDataRoute, getVariableMetadataRoute, GrapherProgrammaticInterface, @@ -15,6 +16,7 @@ import { GrapherInterface, ImageMetadata, Url, + diffGrapherConfigs, } from "@ourworldindata/utils" import { MarkdownTextWrap } from "@ourworldindata/components" import React from "react" @@ -85,6 +87,7 @@ export const DataPageV2 = (props: { compact(grapher?.dimensions?.map((d) => d.variableId)) ) + // TODO(inheritance): use mergeGrapherConfigs instead const mergedGrapherConfig = mergePartialGrapherConfigs( datapageData.chartConfig as GrapherInterface, grapher @@ -102,6 +105,12 @@ export const DataPageV2 = (props: { dataApiUrl: DATA_API_URL, } + // We bake the Grapher config without defaults + const grapherConfigToBake = diffGrapherConfigs( + grapherConfig, + defaultGrapherConfig + ) + // Only embed the tags that are actually used by the datapage, instead of the complete JSON object with ~240 properties const minimalTagToSlugMap = pick( tagToSlugMap, @@ -181,7 +190,7 @@ export const DataPageV2 = (props: {