🔨 (db) add chart_configs table

owid · Jul 10, 2024 · f587577 · f587577
1 parent 8c310f3
commit f587577
Show file tree

Hide file tree

Showing 24 changed files with 778 additions and 362 deletions.
diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts
diff --git a/adminSiteServer/testPageRouter.tsx b/adminSiteServer/testPageRouter.tsx
@@ -24,12 +24,12 @@ import {
 import { grapherToSVG } from "../baker/GrapherImageBaker.js"
 import {
     ChartTypeName,
-    ChartsTableName,
-    DbRawChart,
+    DbRawChartConfig,
+    DbPlainChart,
     EntitySelectionMode,
     GrapherTabOption,
     StackMode,
-    parseChartsRow,
+    parseChartConfig,
 } from "@ourworldindata/types"
 import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js"
 import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js"
@@ -134,27 +134,28 @@ async function propsFromQueryParams(
 
     let query = knex
         .table("charts")
-        .whereRaw("publishedAt IS NOT NULL")
-        .orderBy("id", "DESC")
+        .join({ cc: "chart_configs" }, "charts.configId", "cc.id")
+        .whereRaw("charts.publishedAt IS NOT NULL")
+        .orderBy("charts.id", "DESC")
     console.error(query.toSQL())
 
     let tab = params.tab
 
     if (params.type) {
         if (params.type === ChartTypeName.WorldMap) {
-            query = query.andWhereRaw(`config->>"$.hasMapTab" = "true"`)
+            query = query.andWhereRaw(`cc.full->>"$.hasMapTab" = "true"`)
             tab = tab || GrapherTabOption.map
         } else {
             if (params.type === "LineChart") {
                 query = query.andWhereRaw(
                     `(
-                        config->"$.type" = "LineChart"
-                        OR config->"$.type" IS NULL
-                    ) AND COALESCE(config->>"$.hasChartTab", "true") = "true"`
+                        cc.full->"$.type" = "LineChart"
+                        OR cc.full->"$.type" IS NULL
+                    ) AND COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`
                 )
             } else {
                 query = query.andWhereRaw(
-                    `config->"$.type" = :type AND COALESCE(config->>"$.hasChartTab", "true") = "true"`,
+                    `cc.full->"$.type" = :type AND COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`,
                     { type: params.type }
                 )
             }
@@ -164,27 +165,27 @@ async function propsFromQueryParams(
 
     if (params.logLinear) {
         query = query.andWhereRaw(
-            `config->>'$.yAxis.canChangeScaleType' = "true" OR config->>'$.xAxis.canChangeScaleType'  = "true"`
+            `cc.full->>'$.yAxis.canChangeScaleType' = "true" OR cc.full->>'$.xAxis.canChangeScaleType'  = "true"`
         )
         tab = GrapherTabOption.chart
     }
 
     if (params.comparisonLines) {
         query = query.andWhereRaw(
-            `config->'$.comparisonLines[0].yEquals' != ''`
+            `cc.full->'$.comparisonLines[0].yEquals' != ''`
         )
         tab = GrapherTabOption.chart
     }
 
     if (params.stackMode) {
-        query = query.andWhereRaw(`config->'$.stackMode' = :stackMode`, {
+        query = query.andWhereRaw(`cc.full->'$.stackMode' = :stackMode`, {
             stackMode: params.stackMode,
         })
         tab = GrapherTabOption.chart
     }
 
     if (params.relativeToggle) {
-        query = query.andWhereRaw(`config->>'$.hideRelativeToggle' = "false"`)
+        query = query.andWhereRaw(`cc.full->>'$.hideRelativeToggle' = "false"`)
         tab = GrapherTabOption.chart
     }
 
@@ -193,7 +194,7 @@ async function propsFromQueryParams(
         // have a visible categorial legend, and can leave out some that have one.
         // But in practice it seems to work reasonably well.
         query = query.andWhereRaw(
-            `json_length(config->'$.map.colorScale.customCategoryColors') > 1`
+            `json_length(cc.full->'$.map.colorScale.customCategoryColors') > 1`
         )
         tab = GrapherTabOption.map
     }
@@ -219,13 +220,13 @@ async function propsFromQueryParams(
         const mode = params.addCountryMode
         if (mode === EntitySelectionMode.MultipleEntities) {
             query = query.andWhereRaw(
-                `config->'$.addCountryMode' IS NULL OR config->'$.addCountryMode' = :mode`,
+                `cc.full->'$.addCountryMode' IS NULL OR cc.full->'$.addCountryMode' = :mode`,
                 {
                     mode: EntitySelectionMode.MultipleEntities,
                 }
             )
         } else {
-            query = query.andWhereRaw(`config->'$.addCountryMode' = :mode`, {
+            query = query.andWhereRaw(`cc.full->'$.addCountryMode' = :mode`, {
                 mode,
             })
         }
@@ -236,10 +237,10 @@ async function propsFromQueryParams(
     }
 
     if (tab === GrapherTabOption.map) {
-        query = query.andWhereRaw(`config->>"$.hasMapTab" = "true"`)
+        query = query.andWhereRaw(`cc.full->>"$.hasMapTab" = "true"`)
     } else if (tab === GrapherTabOption.chart) {
         query = query.andWhereRaw(
-            `COALESCE(config->>"$.hasChartTab", "true") = "true"`
+            `COALESCE(cc.full->>"$.hasChartTab", "true") = "true"`
         )
     }
 
@@ -277,7 +278,7 @@ async function propsFromQueryParams(
 
     const chartsQuery = query
         .clone()
-        .select("id", "slug")
+        .select(knex.raw('charts.id, cc.full ->> "$.slug" as slug'))
         .limit(perPage)
         .offset(perPage * (page - 1))
 
@@ -467,13 +468,26 @@ getPlainRouteWithROTransaction(
     "/embeds/:id",
     async (req, res, trx) => {
         const id = req.params.id
-        const chartRaw: DbRawChart = await trx
-            .table(ChartsTableName)
-            .where({ id: id })
-            .first()
-        const chartEnriched = parseChartsRow(chartRaw)
-        const viewProps = await getViewPropsFromQueryParams(req.query)
-        if (chartEnriched) {
+        const chartRaw = await db.knexRawFirst<
+            Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] }
+        >(
+            trx,
+            `--sql
+                select ca.id, cc.full as config
+                from charts ca
+                join chart_configs cc
+                on ca.configId = cc.id
+                where ca.id = ?
+            `,
+            [id]
+        )
+
+        if (chartRaw) {
+            const chartEnriched = {
+                ...chartRaw,
+                config: parseChartConfig(chartRaw.config),
+            }
+            const viewProps = await getViewPropsFromQueryParams(req.query)
             const charts = [
                 {
                     id: chartEnriched.id,
@@ -639,9 +653,15 @@ getPlainRouteWithROTransaction(
     testPageRouter,
     "/previews",
     async (req, res, trx) => {
-        const rows = await db.knexRaw(
+        const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
             trx,
-            `SELECT config FROM charts LIMIT 200`
+            `--sql 
+                SELECT cc.full as config
+                FROM charts ca
+                JOIN chart_configs cc
+                ON ca.configId = cc.id
+                LIMIT 200
+            `
         )
         const charts = rows.map((row: any) => JSON.parse(row.config))
 
@@ -653,9 +673,15 @@ getPlainRouteWithROTransaction(
     testPageRouter,
     "/embedVariants",
     async (req, res, trx) => {
-        const rows = await db.knexRaw(
+        const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
             trx,
-            `SELECT config FROM charts WHERE id=64`
+            `--sql
+                SELECT cc.full as config
+                FROM charts ca
+                JOIN chart_configs cc
+                ON ca.configId = cc.id
+                WHERE ca.id=64
+            `
         )
         const charts = rows.map((row: any) => JSON.parse(row.config))
         const viewProps = getViewPropsFromQueryParams(req.query)

diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx
@@ -43,6 +43,8 @@ import {
     FaqDictionary,
     ImageMetadata,
     OwidGdocBaseInterface,
+    DbPlainChart,
+    DbRawChartConfig,
 } from "@ourworldindata/types"
 import ProgressBar from "progress"
 import {
@@ -482,16 +484,24 @@ export const bakeSingleGrapherChart = async (
 export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
     // TODO: this transaction is only RW because somewhere inside it we fetch images
     async (bakedSiteDir: string, knex: db.KnexReadWriteTransaction) => {
-        const chartsToBake: { id: number; config: string; slug: string }[] =
-            await knexRaw(
-                knex,
-                `-- sql
-                SELECT
-                    id, config, config->>'$.slug' as slug
-                FROM charts WHERE JSON_EXTRACT(config, "$.isPublished")=true
-                ORDER BY JSON_EXTRACT(config, "$.slug") ASC
+        const chartsToBake = await knexRaw<
+            Pick<DbPlainChart, "id"> & {
+                config: DbRawChartConfig["full"]
+                slug: string
+            }
+        >(
+            knex,
+            `-- sql
+                    SELECT
+                        c.id,
+                        cc.full as config,
+                        cc.full->>'$.slug' as slug
+                    FROM charts c
+                    JOIN chart_configs cc ON c.configId = cc.id
+                    WHERE JSON_EXTRACT(cc.full, "$.isPublished")=true
+                    ORDER BY JSON_EXTRACT(cc.full, "$.slug") ASC
                 `
-            )
+        )
 
         const newSlugs = chartsToBake.map((row) => row.slug)
         await fs.mkdirp(bakedSiteDir + "/grapher")

diff --git a/baker/GrapherBakingUtils.ts b/baker/GrapherBakingUtils.ts
@@ -82,7 +82,12 @@ export const bakeGrapherUrls = async (
 
         const rows = await db.knexRaw<{ version: number }>(
             knex,
-            `SELECT charts.config->>"$.version" AS version FROM charts WHERE charts.id=?`,
+            `-- sql
+                SELECT cc.full->>"$.version" AS version
+                FROM charts c
+                JOIN chart_configs cc ON c.configId = cc.id
+                WHERE c.id=?
+            `,
             [chartId]
         )
         if (!rows.length) {

diff --git a/baker/GrapherImageBaker.tsx b/baker/GrapherImageBaker.tsx
@@ -1,7 +1,8 @@
 import {
     DbPlainChartSlugRedirect,
-    DbRawChart,
+    DbPlainChart,
     GrapherInterface,
+    DbRawChartConfig,
 } from "@ourworldindata/types"
 import { Grapher, GrapherProgrammaticInterface } from "@ourworldindata/grapher"
 import { MultipleOwidVariableDataDimensionsMap } from "@ourworldindata/utils"
@@ -79,9 +80,16 @@ export async function getPublishedGraphersBySlug(
     const graphersById: Map<number, GrapherInterface> = new Map()
 
     // Select all graphers that are published
-    const sql = `SELECT id, config FROM charts WHERE config->>"$.isPublished" = "true"`
-
-    const query = db.knexRaw<Pick<DbRawChart, "id" | "config">>(knex, sql)
+    const sql = `-- sql
+        SELECT c.id, cc.full as config
+        FROM charts c
+        JOIN chart_configs cc ON c.configId = cc.id
+        WHERE cc.full ->> "$.isPublished" = 'true'
+    `
+
+    const query = db.knexRaw<
+        Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] }
+    >(knex, sql)
     for (const row of await query) {
         const grapher = JSON.parse(row.config)
 

diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx
@@ -700,19 +700,21 @@ export class SiteBaker {
                 knex,
                 `-- sql
                 SELECT
-                    config ->> '$.slug' as slug,
-                    config ->> '$.subtitle' as subtitle,
-                    config ->> '$.note' as note
+                    cc.full ->> '$.slug' as slug,
+                    cc.full ->> '$.subtitle' as subtitle,
+                    cc.full ->> '$.note' as note
                 FROM
-                    charts
+                    charts c
+                JOIN
+                    chart_configs cc ON c.configId = cc.id
                 WHERE
-                    JSON_EXTRACT(config, "$.isPublished") = true
+                    JSON_EXTRACT(cc.full, "$.isPublished") = true
                 AND (
-                    JSON_EXTRACT(config, "$.subtitle") LIKE "%#dod:%"
-                    OR JSON_EXTRACT(config, "$.note") LIKE "%#dod:%"
+                    JSON_EXTRACT(cc.full, "$.subtitle") LIKE "%#dod:%"
+                    OR JSON_EXTRACT(cc.full, "$.note") LIKE "%#dod:%"
                 )
                 ORDER BY
-                    JSON_EXTRACT(config, "$.slug") ASC
+                    JSON_EXTRACT(cc.full, "$.slug") ASC
             `
             )
 

diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts
@@ -123,19 +123,20 @@ const getChartsRecords = async (
         `-- sql
         WITH indexable_charts_with_entity_names AS (
             SELECT c.id,
-                   config ->> "$.slug"                    AS slug,
-                   config ->> "$.title"                   AS title,
-                   config ->> "$.variantName"             AS variantName,
-                   config ->> "$.subtitle"                AS subtitle,
-                   JSON_LENGTH(config ->> "$.dimensions") AS numDimensions,
+                   cc.full ->> "$.slug"                    AS slug,
+                   cc.full ->> "$.title"                   AS title,
+                   cc.full ->> "$.variantName"             AS variantName,
+                   cc.full ->> "$.subtitle"                AS subtitle,
+                   JSON_LENGTH(cc.full ->> "$.dimensions") AS numDimensions,
                    c.publishedAt,
                    c.updatedAt,
                    JSON_ARRAYAGG(e.name)                  AS entityNames
             FROM charts c
+                     LEFT JOIN chart_configs cc ON c.configId = cc.id
                      LEFT JOIN charts_x_entities ce ON c.id = ce.chartId
                      LEFT JOIN entities e ON ce.entityId = e.id
-            WHERE config ->> "$.isPublished" = 'true'
-              AND is_indexable IS TRUE
+            WHERE cc.full ->> "$.isPublished" = 'true'
+                AND c.is_indexable IS TRUE
             GROUP BY c.id
         )
         SELECT c.id,

diff --git a/baker/countryProfiles.tsx b/baker/countryProfiles.tsx
@@ -7,6 +7,8 @@ import {
     DbEnrichedVariable,
     VariablesTableName,
     parseVariablesRow,
+    DbRawChartConfig,
+    parseChartConfig,
 } from "@ourworldindata/types"
 import * as lodash from "lodash"
 import {
@@ -45,13 +47,20 @@ const countryIndicatorGraphers = async (
     trx: db.KnexReadonlyTransaction
 ): Promise<GrapherInterface[]> =>
     bakeCache(countryIndicatorGraphers, async () => {
-        const graphers = (
-            await trx
-                .table("charts")
-                .whereRaw(
-                    "publishedAt is not null and config->>'$.isPublished' = 'true' and is_indexable is true"
-                )
-        ).map((c: any) => JSON.parse(c.config)) as GrapherInterface[]
+        const configs = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
+            trx,
+            `-- sql
+                SELECT cc.full as config
+                FROM charts c
+                JOIN chart_configs cc ON cc.id = c.configId
+                WHERE
+                    c.publishedAt is not null
+                    AND cc.full->>'$.isPublished' = 'true'
+                    AND c.is_indexable is true
+            `
+        )
+
+        const graphers = configs.map((c: any) => parseChartConfig(c.config))
 
         return graphers.filter(checkShouldShowIndicator)
     })

diff --git a/baker/redirects.ts b/baker/redirects.ts
@@ -88,9 +88,11 @@ export const getGrapherRedirectsMap = async (
     }>(
         knex,
         `-- sql
-        SELECT chart_slug_redirects.slug as oldSlug, charts.config ->> "$.slug" as newSlug
-        FROM chart_slug_redirects INNER JOIN charts ON charts.id=chart_id
-    `
+            SELECT chart_slug_redirects.slug as oldSlug, chart_configs.full ->> "$.slug" as newSlug
+            FROM chart_slug_redirects
+            INNER JOIN charts ON charts.id=chart_id
+            INNER JOIN chart_configs ON chart_configs.id=charts.configId
+        `
     )) as Array<{ oldSlug: string; newSlug: string }>
 
     return new Map(