diff --git a/.vscode/launch.json b/.vscode/launch.json index 25ccd948758..0d98f6f5573 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -129,4 +129,4 @@ "port": 9000 } ] -} \ No newline at end of file +} diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index a9efc963bbc..dbe31c4b697 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -11,6 +11,7 @@ import { type RawPageview, Topic, PostReference, + ChartRedirect, DimensionProperty, } from "@ourworldindata/utils" import { computed, observable, runInAction, when } from "mobx" @@ -56,12 +57,6 @@ export const getFullReferencesCount = (references: References): number => { ) } -export interface ChartRedirect { - id: number - slug: string - chartId: number -} - export interface Namespace { name: string description?: string diff --git a/adminSiteClient/ChartEditorPage.tsx b/adminSiteClient/ChartEditorPage.tsx index 242f86fc70a..7d1d351ef86 100644 --- a/adminSiteClient/ChartEditorPage.tsx +++ b/adminSiteClient/ChartEditorPage.tsx @@ -25,6 +25,7 @@ import { Topic, GrapherInterface, GrapherStaticFormat, + ChartRedirect, DimensionProperty, } from "@ourworldindata/types" import { Grapher } from "@ourworldindata/grapher" @@ -34,7 +35,6 @@ import { EditorDatabase, Log, References, - ChartRedirect, ChartEditorManager, Dataset, getFullReferencesCount, diff --git a/adminSiteClient/EditorReferencesTab.tsx b/adminSiteClient/EditorReferencesTab.tsx index c515e71fe68..38a0448e142 100644 --- a/adminSiteClient/EditorReferencesTab.tsx +++ b/adminSiteClient/EditorReferencesTab.tsx @@ -2,14 +2,17 @@ import React from "react" import { observer } from "mobx-react" import { ChartEditor, - ChartRedirect, References, getFullReferencesCount, } from "./ChartEditor.js" import { computed, action, observable, runInAction } from "mobx" import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js" import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" -import { stringifyUnknownError, formatValue } from "@ourworldindata/utils" +import { + stringifyUnknownError, + formatValue, + ChartRedirect, +} from "@ourworldindata/utils" const BASE_URL = BAKED_GRAPHER_URL.replace(/^https?:\/\//, "") diff --git a/adminSiteServer/adminRouter.tsx b/adminSiteServer/adminRouter.tsx index 5fa2164acfb..3589ef947ee 100644 --- a/adminSiteServer/adminRouter.tsx +++ b/adminSiteServer/adminRouter.tsx @@ -42,7 +42,7 @@ import { renderDataPageV2, renderPreviewDataPageOrGrapherPage, } from "../baker/GrapherBaker.js" -import { Chart } from "../db/model/Chart.js" +import { getChartConfigBySlug } from "../db/model/Chart.js" import { getVariableMetadata } from "../db/model/Variable.js" import { DbPlainDatasetFile, DbPlainDataset } from "@ourworldindata/types" @@ -324,11 +324,12 @@ adminRouter.get("/datapage-preview/:id", async (req, res) => { }) adminRouter.get("/grapher/:slug", async (req, res) => { - const entity = await Chart.getBySlug(req.params.slug) - if (!entity) throw new JsonError("No such chart", 404) - const previewDataPageOrGrapherPage = db.knexReadonlyTransaction( - async (knex) => renderPreviewDataPageOrGrapherPage(entity.config, knex) + async (knex) => { + const entity = await getChartConfigBySlug(knex, req.params.slug) + if (!entity) throw new JsonError("No such chart", 404) + return renderPreviewDataPageOrGrapherPage(entity.config, knex) + } ) res.send(previewDataPageOrGrapherPage) }) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 50f2e8692af..6aa278b48bc 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -13,7 +13,16 @@ import { DATA_API_URL, } from "../settings/serverSettings.js" import { expectInt, isValidSlug } from "../serverUtils/serverUtil.js" -import { OldChart, Chart, getGrapherById } from "../db/model/Chart.js" +import { + OldChartFieldList, + assignTagsForCharts, + getChartConfigById, + getChartSlugById, + getGptTopicSuggestions, + getRedirectsByChartId, + oldChartFieldList, + setChartTags, +} from "../db/model/Chart.js" import { Request, CurrentUser } from "./authentication.js" import { getMergedGrapherConfigForVariable, @@ -57,10 +66,16 @@ import { DbPlainTag, grapherKeysToSerialize, DbRawVariable, - DbRawOrigin, parseOriginsRow, PostsTableName, DbRawPost, + DbRawSuggestedChartRevision, + DbPlainChartSlugRedirect, + DbRawChart, + DbInsertChartRevision, + serializeChartConfig, + DbRawOrigin, + DbRawPostGdoc, DbPlainDataset, } from "@ourworldindata/types" import { @@ -75,10 +90,9 @@ import { syncDatasetToGitRepo, removeDatasetFromGitRepo, } from "./gitDataExport.js" -import { ChartRevision } from "../db/model/ChartRevision.js" import { SuggestedChartRevision } from "../db/model/SuggestedChartRevision.js" import { denormalizeLatestCountryData } from "../baker/countryProfiles.js" -import { ChartRedirect, References } from "../adminSiteClient/ChartEditor.js" +import { References } from "../adminSiteClient/ChartEditor.js" import { DeployQueueServer } from "../baker/DeployQueueServer.js" import { FunctionalRouter } from "./FunctionalRouter.js" import { escape } from "mysql" @@ -106,6 +120,7 @@ import { deleteRouteWithRWTransaction, putRouteWithRWTransaction, postRouteWithRWTransaction, + patchRouteWithRWTransaction, } from "./routerHelpers.js" const apiRouter = new FunctionalRouter() @@ -148,8 +163,24 @@ const enqueueLightningChange = async ( }) } -async function getLogsByChartId(chartId: number): Promise { - const logs = await db.queryMysql( +async function getLogsByChartId( + knex: db.KnexReadonlyTransaction, + chartId: number +): Promise< + { + userId: number + config: string + userName: string + createdAt: Date + }[] +> { + const logs = await db.knexRaw<{ + userId: number + config: string + userName: string + createdAt: Date + }>( + knex, `SELECT userId, config, fullName as userName, l.createdAt FROM chart_revisions l LEFT JOIN users u on u.id = userId @@ -189,27 +220,18 @@ const getReferencesByChartId = async ( } } -const getRedirectsByChartId = async ( - chartId: number -): Promise => - await db.queryMysql( - ` - SELECT id, slug, chart_id as chartId - FROM chart_slug_redirects - WHERE chart_id = ? - ORDER BY id ASC`, - [chartId] - ) - -const expectChartById = async (chartId: any): Promise => { - const chart = await getGrapherById(expectInt(chartId)) - if (chart) return chart +const expectChartById = async ( + knex: db.KnexReadonlyTransaction, + chartId: any +): Promise => { + const chart = await getChartConfigById(knex, expectInt(chartId)) + if (chart) return chart.config throw new JsonError(`No chart found for id ${chartId}`, 404) } const saveGrapher = async ( - transactionContext: db.TransactionContext, + knex: db.KnexReadWriteTransaction, user: CurrentUser, newConfig: GrapherInterface, existingConfig?: GrapherInterface, @@ -218,7 +240,8 @@ const saveGrapher = async ( ) => { // Slugs need some special logic to ensure public urls remain consistent whenever possible async function isSlugUsedInRedirect() { - const rows = await transactionContext.query( + const rows = await db.knexRaw( + knex, `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`, // -1 is a placeholder ID that will never exist; but we cannot use NULL because // in that case we would always get back an empty resultset @@ -228,7 +251,8 @@ const saveGrapher = async ( } async function isSlugUsedInOtherGrapher() { - const rows = await transactionContext.query( + const rows = await db.knexRaw>( + knex, `SELECT id FROM charts WHERE id != ? AND config->>"$.isPublished" = "true" AND JSON_EXTRACT(config, "$.slug") = ?`, // -1 is a placeholder ID that will never exist; but we cannot use NULL because // in that case we would always get back an empty resultset @@ -255,11 +279,13 @@ const saveGrapher = async ( existingConfig.slug !== newConfig.slug ) { // Changing slug of an existing chart, delete any old redirect and create new one - await transactionContext.execute( + await db.knexRaw( + knex, `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`, [existingConfig.id, existingConfig.slug] ) - await transactionContext.execute( + await db.knexRaw( + knex, `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, [existingConfig.id, existingConfig.slug] ) @@ -280,12 +306,14 @@ const saveGrapher = async ( let chartId = existingConfig && existingConfig.id const newJsonConfig = JSON.stringify(newConfig) if (existingConfig) - await transactionContext.query( + await db.knexRaw( + knex, `UPDATE charts SET config=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=? WHERE id = ?`, [newJsonConfig, now, now, user.id, chartId] ) else { - const result = await transactionContext.execute( + const result = await db.knexRawInsert( + knex, `INSERT INTO charts (config, createdAt, updatedAt, lastEditedAt, lastEditedByUserId) VALUES (?)`, [[newJsonConfig, now, now, now, user.id]] ) @@ -293,25 +321,38 @@ const saveGrapher = async ( } // Record this change in version history - const log = new ChartRevision() - log.chartId = chartId as number - log.userId = user.id - log.config = newConfig - // TODO: the orm needs to support this but it does not :( - log.createdAt = new Date() - log.updatedAt = new Date() - await transactionContext.manager.save(log) + + const chartRevisionLog = { + chartId: chartId as number, + userId: user.id, + config: serializeChartConfig(newConfig), + createdAt: new Date(), + updatedAt: new Date(), + } satisfies DbInsertChartRevision + await db.knexRaw( + knex, + `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?)`, + [ + [ + chartRevisionLog.chartId, + chartRevisionLog.userId, + chartRevisionLog.config, + chartRevisionLog.createdAt, + chartRevisionLog.updatedAt, + ], + ] + ) // Remove any old dimensions and store the new ones // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json - await transactionContext.execute( - `DELETE FROM chart_dimensions WHERE chartId=?`, - [chartId] - ) + await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chartId, + ]) const newDimensions = newConfig.dimensions ?? [] for (const [i, dim] of newDimensions.entries()) { - await transactionContext.execute( + await db.knexRaw( + knex, `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?)`, [[chartId, dim.variableId, dim.property, i]] ) @@ -320,11 +361,9 @@ const saveGrapher = async ( // So we can generate country profiles including this chart data if (newConfig.isPublished && referencedVariablesMightChange) // TODO: remove this ad hoc knex transaction context when we switch the function to knex - await db.knexReadWriteTransaction((trx) => - denormalizeLatestCountryData( - trx, - newDimensions.map((d) => d.variableId) - ) + await denormalizeLatestCountryData( + knex, + newDimensions.map((d) => d.variableId) ) if ( @@ -332,7 +371,8 @@ const saveGrapher = async ( (!existingConfig || !existingConfig.isPublished) ) { // Newly published, set publication info - await transactionContext.execute( + await db.knexRaw( + knex, `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `, [now, user.id, chartId] ) @@ -343,7 +383,8 @@ const saveGrapher = async ( existingConfig.isPublished ) { // Unpublishing chart, delete any existing redirects to it - await transactionContext.execute( + await db.knexRaw( + knex, `DELETE FROM chart_slug_redirects WHERE chart_id = ?`, [existingConfig.id] ) @@ -354,11 +395,12 @@ const saveGrapher = async ( return chartId } -apiRouter.get("/charts.json", async (req, res) => { +getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 - const charts = await db.queryMysql( - ` - SELECT ${OldChart.listFields} FROM charts + const charts = await db.knexRaw( + trx, + `-- sql + SELECT ${oldChartFieldList} FROM charts JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId ORDER BY charts.lastEditedAt DESC LIMIT ? @@ -366,7 +408,7 @@ apiRouter.get("/charts.json", async (req, res) => { [limit] ) - await Chart.assignTagsForCharts(charts) + await assignTagsForCharts(trx, charts) return { charts } }) @@ -425,8 +467,10 @@ apiRouter.get("/charts.csv", async (req, res) => { return csv }) -apiRouter.get("/charts/:chartId.config.json", async (req, res) => - expectChartById(req.params.chartId) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.config.json", + async (req, res, trx) => expectChartById(trx, req.params.chartId) ) apiRouter.get("/editorData/namespaces.json", async (req, res) => { @@ -449,9 +493,16 @@ apiRouter.get("/editorData/namespaces.json", async (req, res) => { } }) -apiRouter.get("/charts/:chartId.logs.json", async (req, res) => ({ - logs: await getLogsByChartId(parseInt(req.params.chartId as string)), -})) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.logs.json", + async (req, res, trx) => ({ + logs: await getLogsByChartId( + trx, + parseInt(req.params.chartId as string) + ), + }) +) getRouteWithROTransaction( apiRouter, @@ -467,19 +518,25 @@ getRouteWithROTransaction( } ) -apiRouter.get("/charts/:chartId.redirects.json", async (req, res) => ({ - redirects: await getRedirectsByChartId( - parseInt(req.params.chartId as string) - ), -})) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.redirects.json", + async (req, res, trx) => ({ + redirects: await getRedirectsByChartId( + trx, + parseInt(req.params.chartId as string) + ), + }) +) getRouteWithROTransaction( apiRouter, "/charts/:chartId.pageviews.json", async (req, res, trx) => { - const slug = await Chart.getById( + const slug = await getChartSlugById( + trx, parseInt(req.params.chartId as string) - ).then((chart) => chart?.config?.slug) + ) if (!slug) return {} const pageviewsByUrl = await db.knexRawFirst( @@ -597,64 +654,74 @@ apiRouter.get( } ) -apiRouter.post("/charts", async (req, res) => { - const chartId = await db.transaction(async (t) => { - return saveGrapher(t, res.locals.user, req.body) - }) +postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { + const chartId = await saveGrapher(trx, res.locals.user, req.body) + return { success: true, chartId: chartId } }) -apiRouter.post("/charts/:chartId/setTags", async (req, res) => { - const chartId = expectInt(req.params.chartId) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/setTags", + async (req, res, trx) => { + const chartId = expectInt(req.params.chartId) - await Chart.setTags(chartId, req.body.tags) + await setChartTags(trx, chartId, req.body.tags) - return { success: true } -}) + return { success: true } + } +) -apiRouter.put("/charts/:chartId", async (req, res) => { - const existingConfig = await expectChartById(req.params.chartId) +putRouteWithRWTransaction( + apiRouter, + "/charts/:chartId", + async (req, res, trx) => { + const existingConfig = await expectChartById(trx, req.params.chartId) - await db.transaction(async (t) => { - await saveGrapher(t, res.locals.user, req.body, existingConfig) - }) + await saveGrapher(trx, res.locals.user, req.body, existingConfig) - const logs = await getLogsByChartId(existingConfig.id as number) - return { success: true, chartId: existingConfig.id, newLog: logs[0] } -}) - -apiRouter.delete("/charts/:chartId", async (req, res) => { - const chart = await expectChartById(req.params.chartId) - const links = await Link.getPublishedLinksTo([chart.slug!]) - if (links.length) { - const sources = links.map((link) => link.source.slug).join(", ") - throw new Error( - `Cannot delete chart in-use in the following published documents: ${sources}` - ) + const logs = await getLogsByChartId(trx, existingConfig.id as number) + return { success: true, chartId: existingConfig.id, newLog: logs[0] } } +) - await db.transaction(async (t) => { - await t.execute(`DELETE FROM chart_dimensions WHERE chartId=?`, [ - chart.id, - ]) - await t.execute(`DELETE FROM chart_slug_redirects WHERE chart_id=?`, [ +deleteRouteWithRWTransaction( + apiRouter, + "/charts/:chartId", + async (req, res, trx) => { + const chart = await expectChartById(trx, req.params.chartId) + const links = await Link.getPublishedLinksTo([chart.slug!]) + if (links.length) { + const sources = links.map((link) => link.source.slug).join(", ") + throw new Error( + `Cannot delete chart in-use in the following published documents: ${sources}` + ) + } + + await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ chart.id, ]) - await t.execute( + await db.knexRaw( + trx, + `DELETE FROM chart_slug_redirects WHERE chart_id=?`, + [chart.id] + ) + await db.knexRaw( + trx, `DELETE FROM suggested_chart_revisions WHERE chartId=?`, [chart.id] ) - await t.execute(`DELETE FROM charts WHERE id=?`, [chart.id]) - }) + await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) - if (chart.isPublished) - await triggerStaticBuild( - res.locals.user, - `Deleting chart ${chart.slug}` - ) + if (chart.isPublished) + await triggerStaticBuild( + res.locals.user, + `Deleting chart ${chart.slug}` + ) - return { success: true } -}) + return { success: true } + } +) apiRouter.get("/suggested-chart-revisions", async (req, res) => { const isValidSortBy = (sortBy: string) => { @@ -1133,9 +1200,10 @@ apiRouter.get( } ) -apiRouter.post( +postRouteWithRWTransaction( + apiRouter, "/suggested-chart-revisions/:suggestedChartRevisionId/update", - async (req, res) => { + async (req, res, trx) => { const suggestedChartRevisionId = expectInt( req.params.suggestedChartRevisionId ) @@ -1146,112 +1214,121 @@ apiRouter.post( decisionReason: string } - await db.transaction(async (t) => { - const suggestedChartRevision = await db.mysqlFirst( - `SELECT id, chartId, suggestedConfig, originalConfig, status FROM suggested_chart_revisions WHERE id=?`, - [suggestedChartRevisionId] - ) - if (!suggestedChartRevision) { - throw new JsonError( - `No suggested chart revision found for id '${suggestedChartRevisionId}'`, - 404 - ) - } - if (suggestedConfig !== undefined && suggestedConfig !== null) { - suggestedChartRevision.suggestedConfig = suggestedConfig - } else { - suggestedChartRevision.suggestedConfig = JSON.parse( - suggestedChartRevision.suggestedConfig - ) - } - suggestedChartRevision.originalConfig = JSON.parse( - suggestedChartRevision.originalConfig + // TODO: remove the :any type when the code below is refactored + const suggestedChartRevision: any = await db.knexRawFirst< + Pick< + DbRawSuggestedChartRevision, + "id" | "chartId" | "suggestedConfig" | "originalConfig" + > + >( + trx, + `SELECT id, chartId, suggestedConfig, originalConfig, status FROM suggested_chart_revisions WHERE id=?`, + [suggestedChartRevisionId] + ) + if (!suggestedChartRevision) { + throw new JsonError( + `No suggested chart revision found for id '${suggestedChartRevisionId}'`, + 404 ) - suggestedChartRevision.existingConfig = await expectChartById( - suggestedChartRevision.chartId + } + if (suggestedConfig !== undefined && suggestedConfig !== null) { + suggestedChartRevision.suggestedConfig = suggestedConfig + } else { + suggestedChartRevision.suggestedConfig = JSON.parse( + suggestedChartRevision.suggestedConfig ) + } + suggestedChartRevision.originalConfig = JSON.parse( + suggestedChartRevision.originalConfig + ) + suggestedChartRevision.existingConfig = await expectChartById( + trx, + suggestedChartRevision.chartId + ) - const canApprove = SuggestedChartRevision.checkCanApprove( - suggestedChartRevision - ) - const canReject = SuggestedChartRevision.checkCanReject( - suggestedChartRevision - ) - const canFlag = SuggestedChartRevision.checkCanFlag( - suggestedChartRevision - ) - const canPending = SuggestedChartRevision.checkCanPending( - suggestedChartRevision - ) + const canApprove = SuggestedChartRevision.checkCanApprove( + suggestedChartRevision + ) + const canReject = SuggestedChartRevision.checkCanReject( + suggestedChartRevision + ) + const canFlag = SuggestedChartRevision.checkCanFlag( + suggestedChartRevision + ) + const canPending = SuggestedChartRevision.checkCanPending( + suggestedChartRevision + ) - const canUpdate = - (status === "approved" && canApprove) || - (status === "rejected" && canReject) || - (status === "pending" && canPending) || - (status === "flagged" && canFlag) - if (!canUpdate) { - throw new JsonError( - `Suggest chart revision ${suggestedChartRevisionId} cannot be ` + - `updated with status="${status}".`, - 404 - ) - } + const canUpdate = + (status === "approved" && canApprove) || + (status === "rejected" && canReject) || + (status === "pending" && canPending) || + (status === "flagged" && canFlag) + if (!canUpdate) { + throw new JsonError( + `Suggest chart revision ${suggestedChartRevisionId} cannot be ` + + `updated with status="${status}".`, + 404 + ) + } - await t.execute( - ` + await db.knexRaw( + trx, + ` UPDATE suggested_chart_revisions SET status=?, decisionReason=?, updatedAt=?, updatedBy=? WHERE id = ? `, - [ - status, - decisionReason, - new Date(), - res.locals.user.id, - suggestedChartRevisionId, - ] - ) + [ + status, + decisionReason, + new Date(), + res.locals.user.id, + suggestedChartRevisionId, + ] + ) - // Update config ONLY when APPROVE button is clicked - // Makes sense when the suggested config is a sugegstion by GPT, otherwise is redundant but we are cool with it - if (status === SuggestedChartRevisionStatus.approved) { - await t.execute( - ` + // Update config ONLY when APPROVE button is clicked + // Makes sense when the suggested config is a sugegstion by GPT, otherwise is redundant but we are cool with it + if (status === SuggestedChartRevisionStatus.approved) { + await db.knexRaw( + trx, + ` UPDATE suggested_chart_revisions SET suggestedConfig=? WHERE id = ? `, - [ - JSON.stringify(suggestedChartRevision.suggestedConfig), - suggestedChartRevisionId, - ] - ) - } - // note: the calls to saveGrapher() below will never overwrite a config - // that has been changed since the suggestedConfig was created, because - // if the config has been changed since the suggestedConfig was created - // then canUpdate will be false (so an error would have been raised - // above). - if (status === "approved" && canApprove) { - await saveGrapher( - t, - res.locals.user, - suggestedChartRevision.suggestedConfig, - suggestedChartRevision.existingConfig - ) - } else if ( - status === "rejected" && - canReject && - suggestedChartRevision.status === "approved" - ) { - await saveGrapher( - t, - res.locals.user, - suggestedChartRevision.originalConfig, - suggestedChartRevision.existingConfig - ) - } - }) + [ + JSON.stringify(suggestedChartRevision.suggestedConfig), + suggestedChartRevisionId, + ] + ) + } + // note: the calls to saveGrapher() below will never overwrite a config + // that has been changed since the suggestedConfig was created, because + // if the config has been changed since the suggestedConfig was created + // then canUpdate will be false (so an error would have been raised + // above). + + if (status === "approved" && canApprove) { + await saveGrapher( + trx, + res.locals.user, + suggestedChartRevision.suggestedConfig, + suggestedChartRevision.existingConfig + ) + } else if ( + status === "rejected" && + canReject && + suggestedChartRevision.status === "approved" + ) { + await saveGrapher( + trx, + res.locals.user, + suggestedChartRevision.originalConfig, + suggestedChartRevision.existingConfig + ) + } return { success: true } } @@ -1396,15 +1473,18 @@ WHERE ${whereClause}`) } ) -apiRouter.patch("/chart-bulk-update", async (req, res) => { - const patchesList = req.body as GrapherConfigPatch[] - const chartIds = new Set(patchesList.map((patch) => patch.id)) +patchRouteWithRWTransaction( + apiRouter, + "/chart-bulk-update", + async (req, res, trx) => { + const patchesList = req.body as GrapherConfigPatch[] + const chartIds = new Set(patchesList.map((patch) => patch.id)) - await db.transaction(async (manager) => { - const configsAndIds = await manager.query( - `SELECT id, config FROM charts where id IN (?)`, - [[...chartIds.values()]] - ) + const configsAndIds = await db.knexRaw< + Pick + >(trx, `SELECT id, config FROM charts where id IN (?)`, [ + [...chartIds.values()], + ]) const configMap = new Map( configsAndIds.map((item: any) => [ item.id, @@ -1422,17 +1502,17 @@ apiRouter.patch("/chart-bulk-update", async (req, res) => { for (const [id, newConfig] of configMap.entries()) { await saveGrapher( - manager, + trx, res.locals.user, newConfig, oldValuesConfigMap.get(id), false ) } - }) - return { success: true } -}) + return { success: true } + } +) apiRouter.get( "/variable-annotations", @@ -1547,9 +1627,10 @@ getRouteWithROTransaction( variable.catalogPath += `#${variable.shortName}` } - const charts = await db.queryMysql( + const charts = await db.knexRaw( + trx, ` - SELECT ${OldChart.listFields} + SELECT ${oldChartFieldList} FROM charts JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId @@ -1560,7 +1641,7 @@ getRouteWithROTransaction( [variableId] ) - await Chart.assignTagsForCharts(charts) + await assignTagsForCharts(trx, charts) const grapherConfig = await getMergedGrapherConfigForVariable( variableId, @@ -1693,17 +1774,19 @@ getRouteWithROTransaction( if (!dataset) throw new JsonError(`No dataset by id '${datasetId}'`, 404) - const zipFile = await db.knexRawFirst( + const zipFile = await db.knexRawFirst<{ filename: string }>( trx, `SELECT filename FROM dataset_files WHERE datasetId=?`, [datasetId] ) if (zipFile) dataset.zipFile = zipFile - const variables: Pick< - DbRawVariable, - "id" | "name" | "description" | "display" | "catalogPath" - >[] = await db.knexRaw( + const variables = await db.knexRaw< + Pick< + DbRawVariable, + "id" | "name" | "description" | "display" | "catalogPath" + > + >( trx, ` SELECT v.id, v.name, v.description, v.display, v.catalogPath @@ -1720,7 +1803,7 @@ getRouteWithROTransaction( dataset.variables = variables // add all origins - const origins: DbRawOrigin[] = await db.knexRaw( + const origins: DbRawOrigin[] = await db.knexRaw( trx, ` select distinct @@ -1737,7 +1820,11 @@ getRouteWithROTransaction( dataset.origins = parsedOrigins - const sources = await db.knexRaw( + const sources = await db.knexRaw<{ + id: number + name: string + description: string + }>( trx, ` SELECT s.id, s.name, s.description @@ -1757,10 +1844,10 @@ getRouteWithROTransaction( } }) - const charts = await db.knexRaw( + const charts = await db.knexRaw( trx, ` - SELECT ${OldChart.listFields} + SELECT ${oldChartFieldList} FROM charts JOIN chart_dimensions AS cd ON cd.chartId = charts.id JOIN variables AS v ON cd.variableId = v.id @@ -1774,9 +1861,9 @@ getRouteWithROTransaction( dataset.charts = charts - await Chart.assignTagsForCharts(charts as any) + await assignTagsForCharts(trx, charts) - const tags = await db.knexRaw( + const tags = await db.knexRaw<{ id: number; name: string }>( trx, ` SELECT t.id, t.name @@ -1788,7 +1875,11 @@ getRouteWithROTransaction( ) dataset.tags = tags - const availableTags = await db.knexRaw( + const availableTags = await db.knexRaw<{ + id: number + name: string + parentName: string + }>( trx, ` SELECT t.id, t.name, p.name AS parentName @@ -1967,26 +2058,56 @@ apiRouter.get("/redirects.json", async (req, res) => ({ ORDER BY r.id DESC`), })) -apiRouter.get("/tags/:tagId.json", async (req, res) => { - const tagId = expectInt(req.params.tagId) as number | null - - // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff - // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag - // every time we create a new chart etcs - const uncategorized = tagId === UNCATEGORIZED_TAG_ID - - const tag = await db.mysqlFirst( - ` +getRouteWithROTransaction( + apiRouter, + "/tags/:tagId.json", + async (req, res, trx) => { + const tagId = expectInt(req.params.tagId) as number | null + + // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff + // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag + // every time we create a new chart etcs + const uncategorized = tagId === UNCATEGORIZED_TAG_ID + + // TODO: when we have types for our endpoints, make tag of that type instead of any + const tag: any = await db.knexRawFirst< + Pick< + DbPlainTag, + | "id" + | "name" + | "specialType" + | "updatedAt" + | "parentId" + | "slug" + | "isBulkImport" + > + >( + trx, + `-- sql SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug, p.isBulkImport FROM tags t LEFT JOIN tags p ON t.parentId=p.id WHERE t.id = ? `, - [tagId] - ) + [tagId] + ) - // Datasets tagged with this tag - const datasets = await db.queryMysql( - ` + // Datasets tagged with this tag + const datasets = await db.knexRaw< + Pick< + DbPlainDataset, + | "id" + | "namespace" + | "name" + | "description" + | "createdAt" + | "updatedAt" + | "dataEditedAt" + | "isPrivate" + | "nonRedistributable" + > & { dataEditedByUserName: string } + >( + trx, + `-- sql SELECT d.id, d.namespace, @@ -2004,39 +2125,45 @@ apiRouter.get("/tags/:tagId.json", async (req, res) => { WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} ORDER BY d.dataEditedAt DESC `, - uncategorized ? [] : [tagId] - ) - tag.datasets = datasets + uncategorized ? [] : [tagId] + ) + tag.datasets = datasets - // The other tags for those datasets - if (tag.datasets.length) { - if (uncategorized) { - for (const dataset of tag.datasets) dataset.tags = [] - } else { - const datasetTags = await db.queryMysql( - ` + // The other tags for those datasets + if (tag.datasets.length) { + if (uncategorized) { + for (const dataset of tag.datasets) dataset.tags = [] + } else { + const datasetTags = await db.knexRaw<{ + datasetId: number + id: number + name: string + }>( + trx, + `-- sql SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt JOIN tags t ON dt.tagId = t.id WHERE dt.datasetId IN (?) `, - [tag.datasets.map((d: any) => d.id)] - ) - const tagsByDatasetId = lodash.groupBy( - datasetTags, - (t) => t.datasetId - ) - for (const dataset of tag.datasets) { - dataset.tags = tagsByDatasetId[dataset.id].map((t) => - lodash.omit(t, "datasetId") + [tag.datasets.map((d: any) => d.id)] ) + const tagsByDatasetId = lodash.groupBy( + datasetTags, + (t) => t.datasetId + ) + for (const dataset of tag.datasets) { + dataset.tags = tagsByDatasetId[dataset.id].map((t) => + lodash.omit(t, "datasetId") + ) + } } } - } - // Charts using datasets under this tag - const charts = await db.queryMysql( - ` - SELECT ${OldChart.listFields} FROM charts + // Charts using datasets under this tag + const charts = await db.knexRaw( + trx, + `-- sql + SELECT ${oldChartFieldList} FROM charts LEFT JOIN chart_tags ct ON ct.chartId=charts.id JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId @@ -2044,132 +2171,167 @@ apiRouter.get("/tags/:tagId.json", async (req, res) => { GROUP BY charts.id ORDER BY charts.updatedAt DESC `, - uncategorized ? [] : [tagId] - ) - tag.charts = charts + uncategorized ? [] : [tagId] + ) + tag.charts = charts - await Chart.assignTagsForCharts(charts) + await assignTagsForCharts(trx, charts) - // Subcategories - const children = await db.queryMysql( - ` + // Subcategories + const children = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.parentId = ? `, - [tag.id] - ) - tag.children = children + [tag.id] + ) + tag.children = children - // Possible parents to choose from - const possibleParents = await db.queryMysql(` + // Possible parents to choose from + const possibleParents = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.parentId IS NULL AND t.isBulkImport IS FALSE - `) - tag.possibleParents = possibleParents + ` + ) + tag.possibleParents = possibleParents - return { - tag, + return { + tag, + } } -}) +) -apiRouter.put("/tags/:tagId", async (req: Request) => { - const tagId = expectInt(req.params.tagId) - const tag = (req.body as { tag: any }).tag - await db.execute( - `UPDATE tags SET name=?, updatedAt=?, parentId=?, slug=? WHERE id=?`, - [tag.name, new Date(), tag.parentId, tag.slug, tagId] - ) - if (tag.slug) { - // See if there's a published gdoc with a matching slug. - // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, - // where the page for the topic is just an article. - const gdoc: OwidGdocPostInterface[] = await db.execute( - `SELECT slug FROM posts_gdocs pg +putRouteWithRWTransaction( + apiRouter, + "/tags/:tagId", + async (req: Request, res, trx) => { + const tagId = expectInt(req.params.tagId) + const tag = (req.body as { tag: any }).tag + await db.knexRaw( + trx, + `UPDATE tags SET name=?, updatedAt=?, parentId=?, slug=? WHERE id=?`, + [tag.name, new Date(), tag.parentId, tag.slug, tagId] + ) + if (tag.slug) { + // See if there's a published gdoc with a matching slug. + // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, + // where the page for the topic is just an article. + const gdoc = await db.knexRaw>( + trx, + `SELECT slug FROM posts_gdocs pg WHERE EXISTS ( SELECT 1 FROM posts_gdocs_x_tags gt WHERE pg.id = gt.gdocId AND gt.tagId = ? ) AND pg.published = TRUE`, - [tag.id] - ) - if (!gdoc.length) { - return { - success: true, - tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. + [tag.id] + ) + if (!gdoc.length) { + return { + success: true, + tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. Are you sure you haven't made a typo?`, + } } } + return { success: true } } - return { success: true } -}) +) -apiRouter.post("/tags/new", async (req: Request) => { - const tag = (req.body as { tag: any }).tag - const now = new Date() - const result = await db.execute( - `INSERT INTO tags (parentId, name, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, - [tag.parentId, tag.name, now, now] - ) - return { success: true, tagId: result.insertId } -}) +postRouteWithRWTransaction( + apiRouter, + "/tags/new", + async (req: Request, res, trx) => { + const tag = (req.body as { tag: any }).tag + const now = new Date() + const result = await db.knexRawInsert( + trx, + `INSERT INTO tags (parentId, name, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, + [tag.parentId, tag.name, now, now] + ) + return { success: true, tagId: result.insertId } + } +) -apiRouter.get("/tags.json", async (req, res) => { - const tags = await db.queryMysql(` +getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { + const tags = await db.knexRaw( + trx, + `-- sql SELECT t.id, t.name, t.parentId, t.specialType FROM tags t LEFT JOIN tags p ON t.parentId=p.id WHERE t.isBulkImport IS FALSE AND (t.parentId IS NULL OR p.isBulkImport IS FALSE) ORDER BY t.name ASC - `) + ` + ) return { tags, } }) -apiRouter.delete("/tags/:tagId/delete", async (req, res) => { - const tagId = expectInt(req.params.tagId) +deleteRouteWithRWTransaction( + apiRouter, + "/tags/:tagId/delete", + async (req, res, trx) => { + const tagId = expectInt(req.params.tagId) - await db.transaction(async (t) => { - await t.execute(`DELETE FROM tags WHERE id=?`, [tagId]) - }) + await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) - return { success: true } -}) + return { success: true } + } +) -apiRouter.post("/charts/:chartId/redirects/new", async (req: Request) => { - const chartId = expectInt(req.params.chartId) - const fields = req.body as { slug: string } - const result = await db.execute( - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [chartId, fields.slug] - ) - const redirectId = result.insertId - const redirect = await db.mysqlFirst( - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [redirectId] - ) - return { success: true, redirect: redirect } -}) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/redirects/new", + async (req: Request, res, trx) => { + const chartId = expectInt(req.params.chartId) + const fields = req.body as { slug: string } + const result = await db.knexRawInsert( + trx, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [chartId, fields.slug] + ) + const redirectId = result.insertId + const redirect = await db.knexRaw( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [redirectId] + ) + return { success: true, redirect: redirect } + } +) -apiRouter.delete("/redirects/:id", async (req, res) => { - const id = expectInt(req.params.id) +deleteRouteWithRWTransaction( + apiRouter, + "/redirects/:id", + async (req, res, trx) => { + const id = expectInt(req.params.id) - const redirect = await db.mysqlFirst( - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [id] - ) + const redirect = await db.knexRawFirst( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [id] + ) - if (!redirect) throw new JsonError(`No redirect found for id ${id}`, 404) + if (!redirect) + throw new JsonError(`No redirect found for id ${id}`, 404) - await db.execute(`DELETE FROM chart_slug_redirects WHERE id=?`, [id]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect from ${redirect.slug}` - ) + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [ + id, + ]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect from ${redirect.slug}` + ) - return { success: true } -}) + return { success: true } + } +) apiRouter.get("/posts.json", async (req) => { const raw_rows = await db.queryMysql( @@ -2218,13 +2380,17 @@ apiRouter.get("/posts.json", async (req) => { return { posts: rows } }) -postRouteWithRWTransaction(apiRouter, "/posts/:postId/setTags", async (req, res, trx) => { - const postId = expectInt(req.params.postId) +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/setTags", + async (req, res, trx) => { + const postId = expectInt(req.params.postId) - await setTagsForPost(trx, postId, req.body.tagIds) + await setTagsForPost(trx, postId, req.body.tagIds) - return { success: true } -}) + return { success: true } + } +) getRouteWithROTransaction( apiRouter, @@ -2550,13 +2716,18 @@ apiRouter.post("/gdocs/:gdocId/setTags", async (req, res) => { return { success: true } }) -apiRouter.get( +getRouteWithROTransaction( + apiRouter, `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, - async (req: Request): Promise> => { + async ( + req: Request, + res, + trx + ): Promise> => { const chartId = parseIntOrUndefined(req.params.chartId) if (!chartId) throw new JsonError(`Invalid chart ID`, 400) - const topics = await Chart.getGptTopicSuggestions(chartId) + const topics = await getGptTopicSuggestions(trx, chartId) if (!topics.length) throw new JsonError( diff --git a/adminSiteServer/mockSiteRouter.tsx b/adminSiteServer/mockSiteRouter.tsx index ed17b63f9a2..afb75c55067 100644 --- a/adminSiteServer/mockSiteRouter.tsx +++ b/adminSiteServer/mockSiteRouter.tsx @@ -34,7 +34,10 @@ import { countriesIndexPage, } from "../baker/countryProfiles.js" import { makeSitemap } from "../baker/sitemap.js" -import { Chart, OldChart } from "../db/model/Chart.js" +import { + getChartConfigBySlug, + getChartVariableData, +} from "../db/model/Chart.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" import { grapherToSVG } from "../baker/GrapherImageBaker.js" @@ -168,13 +171,16 @@ mockSiteRouter.get("/collection/custom", async (_, res) => { }) mockSiteRouter.get("/grapher/:slug", async (req, res) => { - const entity = await Chart.getBySlug(req.params.slug) - if (!entity) throw new JsonError("No such chart", 404) - - // XXX add dev-prod parity for this - res.set("Access-Control-Allow-Origin", "*") const previewDataPageOrGrapherPage = await db.knexReadonlyTransaction( - async (knex) => renderPreviewDataPageOrGrapherPage(entity.config, knex) + async (knex) => { + const entity = await getChartConfigBySlug(knex, req.params.slug) + if (!entity) throw new JsonError("No such chart", 404) + + // XXX add dev-prod parity for this + res.set("Access-Control-Allow-Origin", "*") + + return renderPreviewDataPageOrGrapherPage(entity.config, knex) + } ) res.send(previewDataPageOrGrapherPage) }) @@ -197,7 +203,9 @@ mockSiteRouter.get("/thank-you", async (req, res) => mockSiteRouter.get("/data-insights/:pageNumberOrSlug?", async (req, res) => { const totalPageCount = calculateDataInsightIndexPageCount( await db - .getPublishedDataInsights(db.knexInstance()) + .getPublishedDataInsights( + db.knexInstance() as db.KnexReadonlyTransaction + ) .then((insights) => insights.length) ) async function renderIndexPage(pageNumber: number) { @@ -313,10 +321,12 @@ mockSiteRouter.use( mockSiteRouter.use("/assets", express.static("dist/assets")) mockSiteRouter.use("/grapher/exports/:slug.svg", async (req, res) => { - const grapher = await OldChart.getBySlug(req.params.slug) - const vardata = await grapher.getVariableData() - res.setHeader("Content-Type", "image/svg+xml") - res.send(await grapherToSVG(grapher.config, vardata)) + await db.knexReadonlyTransaction(async (knex) => { + const grapher = await getChartConfigBySlug(knex, req.params.slug) + const vardata = await getChartVariableData(grapher.config) + res.setHeader("Content-Type", "image/svg+xml") + res.send(await grapherToSVG(grapher.config, vardata)) + }) }) mockSiteRouter.use( diff --git a/adminSiteServer/testPageRouter.tsx b/adminSiteServer/testPageRouter.tsx index 21101eba4b7..4f9626c2c75 100644 --- a/adminSiteServer/testPageRouter.tsx +++ b/adminSiteServer/testPageRouter.tsx @@ -4,7 +4,10 @@ import { Router } from "express" import React from "react" import { renderToHtmlPage, expectInt } from "../serverUtils/serverUtil.js" -import { OldChart, Chart } from "../db/model/Chart.js" +import { + getChartConfigBySlug, + getChartVariableData, +} from "../db/model/Chart.js" import { Head } from "../site/Head.js" import * as db from "../db/db.js" import { @@ -21,9 +24,12 @@ import { import { grapherToSVG } from "../baker/GrapherImageBaker.js" import { ChartTypeName, + ChartsTableName, + DbRawChart, EntitySelectionMode, GrapherTabOption, StackMode, + parseChartsRow, } from "@ourworldindata/types" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" @@ -110,6 +116,7 @@ interface ExplorerTestPageQueryParams { } async function propsFromQueryParams( + knex: db.KnexReadonlyTransaction, params: EmbedTestPageQueryParams ): Promise { const page = params.page @@ -124,7 +131,8 @@ async function propsFromQueryParams( parseStringArrayOrUndefined(params.namespaces) ?? (params.namespace ? [params.namespace] : []) - let query = Chart.createQueryBuilder("charts") + let query = knex + .table("charts") .where("publishedAt IS NOT NULL") .limit(perPage) .offset(perPage * (page - 1)) @@ -265,7 +273,7 @@ async function propsFromQueryParams( ) } - const charts: ChartItem[] = (await query.getMany()).map((c) => ({ + const charts: ChartItem[] = (await query).map((c) => ({ id: c.id, slug: c.config.slug ?? "", })) @@ -274,7 +282,7 @@ async function propsFromQueryParams( charts.forEach((c) => (c.slug += `?tab=${tab}`)) } - const count = await query.getCount() + const count = await charts.length const numPages = Math.ceil(count / perPage) const originalUrl = Url.fromURL(params.originalUrl) @@ -430,40 +438,46 @@ function EmbedTestPage(props: EmbedTestPageProps) { } testPageRouter.get("/embeds", async (req, res) => { - const props = await propsFromQueryParams({ - ...req.query, - originalUrl: req.originalUrl, - }) + const props = await db.knexReadonlyTransaction((trx) => + propsFromQueryParams(trx, { + ...req.query, + originalUrl: req.originalUrl, + }) + ) res.send(renderToHtmlPage()) }) testPageRouter.get("/embeds/:id", async (req, res) => { const id = req.params.id - const chart = await Chart.createQueryBuilder() - .where("id = :id", { id: id }) - .getOne() - const viewProps = getViewPropsFromQueryParams(req.query) - if (chart) { - const charts = [ - { - id: chart.id, - slug: `${chart.config.slug}${ - req.query.tab ? `?tab=${req.query.tab}` : "" - }`, - }, - ] - res.send( - renderToHtmlPage( - + await db.knexReadonlyTransaction(async (trx) => { + const chartRaw: DbRawChart = await trx + .table(ChartsTableName) + .where({ id: id }) + .first() + const chartEnriched = parseChartsRow(chartRaw) + const viewProps = await getViewPropsFromQueryParams(req.query) + if (chartEnriched) { + const charts = [ + { + id: chartEnriched.id, + slug: `${chartEnriched.config.slug}${ + req.query.tab ? `?tab=${req.query.tab}` : "" + }`, + }, + ] + res.send( + renderToHtmlPage( + + ) ) - ) - } else { - res.send("Could not find chart ID") - } + } else { + res.send("Could not find chart ID") + } + }) }) function PreviewTestPage(props: { charts: any[] }) { @@ -627,9 +641,12 @@ testPageRouter.get("/embedVariants", async (req, res) => { }) testPageRouter.get("/:slug.svg", async (req, res) => { - const grapher = await OldChart.getBySlug(req.params.slug) - const vardata = await grapher.getVariableData() - res.send(await grapherToSVG(grapher.config, vardata)) + await db.knexReadonlyTransaction(async (trx) => { + const grapher = await getChartConfigBySlug(trx, req.params.slug) + const vardata = await getChartVariableData(grapher.config) + const svg = await grapherToSVG(grapher.config, vardata) + res.send(svg) + }) }) testPageRouter.get("/explorers", async (req, res) => { diff --git a/baker/DeployUtils.ts b/baker/DeployUtils.ts index b7e2df21bb7..179c1a2c172 100644 --- a/baker/DeployUtils.ts +++ b/baker/DeployUtils.ts @@ -60,7 +60,10 @@ const triggerBakeAndDeploy = async ( if (!lightningQueue.every((change) => change.slug)) throw new Error("Lightning deploy is missing a slug") - await baker.bakeGDocPosts(lightningQueue.map((c) => c.slug!)) + await baker.bakeGDocPosts( + knex, + lightningQueue.map((c) => c.slug!) + ) } else { await baker.bakeAll(knex) } diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index ba3b80c54c3..0f1400a0ed7 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -289,6 +289,7 @@ export async function renderDataPageV2( // Get the charts this variable is being used in (aka "related charts") // and exclude the current chart to avoid duplicates datapageData.allCharts = await getRelatedChartsForVariable( + knex, variableId, grapher && "id" in grapher ? [grapher.id as number] : [] ) diff --git a/baker/GrapherBakingUtils.ts b/baker/GrapherBakingUtils.ts index f502e7557d7..bd1079eef4b 100644 --- a/baker/GrapherBakingUtils.ts +++ b/baker/GrapherBakingUtils.ts @@ -10,7 +10,7 @@ import { BAKED_SITE_EXPORTS_BASE_URL } from "../settings/clientSettings.js" import * as db from "../db/db.js" import { bakeGraphersToSvgs } from "../baker/GrapherImageBaker.js" import { warn } from "../serverUtils/errorLog.js" -import { Chart } from "../db/model/Chart.js" +import { mapSlugsToIds } from "../db/model/Chart.js" import md5 from "md5" import { Url, Tag } from "@ourworldindata/utils" @@ -52,9 +52,12 @@ export interface GrapherExports { get: (grapherUrl: string) => ChartExportMeta | undefined } -export const bakeGrapherUrls = async (urls: string[]) => { +export const bakeGrapherUrls = async ( + knex: db.KnexReadonlyTransaction, + urls: string[] +) => { const currentExports = await getGrapherExportsByUrl() - const slugToId = await Chart.mapSlugsToIds() + const slugToId = await mapSlugsToIds(knex) const toBake = [] // Check that we need to bake this url, and don't already have an export @@ -77,7 +80,8 @@ export const bakeGrapherUrls = async (urls: string[]) => { continue } - const rows = await db.queryMysql( + const rows = await db.knexRaw<{ version: number }>( + knex, `SELECT charts.config->>"$.version" AS version FROM charts WHERE charts.id=?`, [chartId] ) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 0d4577117ab..203495c14e8 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -85,8 +85,8 @@ import { Image } from "../db/model/Image.js" import { generateEmbedSnippet } from "../site/viteUtils.js" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" import { - Chart, getChartEmbedUrlsInPublishedWordpressPosts, + mapSlugsToConfigs, } from "../db/model/Chart.js" import { BAKED_BASE_URL, @@ -198,7 +198,7 @@ export class SiteBaker { await getChartEmbedUrlsInPublishedWordpressPosts(knex) ) - await bakeGrapherUrls(grapherUrls) + await bakeGrapherUrls(knex, grapherUrls) this.grapherExports = await getGrapherExportsByUrl() this.progressBar.tick({ name: "✅ baked embeds" }) @@ -300,6 +300,7 @@ export class SiteBaker { // dictionaries. _prefetchedAttachmentsCache: PrefetchedAttachments | undefined = undefined private async getPrefetchedGdocAttachments( + knex: db.KnexReadonlyTransaction, picks?: [string[], string[], string[], string[]] ): Promise { if (!this._prefetchedAttachmentsCache) { @@ -326,7 +327,7 @@ export class SiteBaker { ) // Includes redirects - const publishedChartsRaw = await Chart.mapSlugsToConfigs() + const publishedChartsRaw = await mapSlugsToConfigs(knex) const publishedCharts: LinkedChart[] = await Promise.all( publishedChartsRaw.map(async (chart) => { const tab = chart.config.tab ?? GrapherTabOption.chart @@ -483,7 +484,7 @@ export class SiteBaker { } // Bake all GDoc posts, or a subset of them if slugs are provided - async bakeGDocPosts(slugs?: string[]) { + async bakeGDocPosts(knex: db.KnexReadonlyTransaction, slugs?: string[]) { await db.getConnection() if (!this.bakeSteps.has("gdocPosts")) return const publishedGdocs = await GdocPost.getPublishedGdocPosts() @@ -504,7 +505,7 @@ export class SiteBaker { } for (const publishedGdoc of gdocsToBake) { - const attachments = await this.getPrefetchedGdocAttachments([ + const attachments = await this.getPrefetchedGdocAttachments(knex, [ publishedGdoc.linkedDocumentIds, publishedGdoc.linkedImageFilenames, publishedGdoc.linkedChartSlugs.grapher, @@ -698,17 +699,14 @@ export class SiteBaker { } } - private async bakeDataInsights() { + private async bakeDataInsights(knex: db.KnexReadonlyTransaction) { if (!this.bakeSteps.has("dataInsights")) return - const latestDataInsights = await db.getPublishedDataInsights( - db.knexInstance(), - 5 - ) + const latestDataInsights = await db.getPublishedDataInsights(knex, 5) const publishedDataInsights = await GdocDataInsight.getPublishedDataInsights() for (const dataInsight of publishedDataInsights) { - const attachments = await this.getPrefetchedGdocAttachments([ + const attachments = await this.getPrefetchedGdocAttachments(knex, [ dataInsight.linkedDocumentIds, dataInsight.linkedImageFilenames, dataInsight.linkedChartSlugs.grapher, @@ -769,13 +767,13 @@ export class SiteBaker { } } - private async bakeAuthors() { + private async bakeAuthors(knex: db.KnexReadonlyTransaction) { if (!this.bakeSteps.has("authors")) return const publishedAuthors = await GdocAuthor.getPublishedAuthors() for (const publishedAuthor of publishedAuthors) { - const attachments = await this.getPrefetchedGdocAttachments([ + const attachments = await this.getPrefetchedGdocAttachments(knex, [ publishedAuthor.linkedDocumentIds, publishedAuthor.linkedImageFilenames, publishedAuthor.linkedChartSlugs.grapher, @@ -983,9 +981,9 @@ export class SiteBaker { } await this.bakeDetailsOnDemand() await this.validateGrapherDodReferences() - await this.bakeGDocPosts() - await this.bakeDataInsights() - await this.bakeAuthors() + await this.bakeGDocPosts(knex) + await this.bakeDataInsights(knex) + await this.bakeAuthors(knex) await this.bakeDriveImages() } diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts index 37c5225b946..a9e8582d36b 100644 --- a/baker/algolia/indexChartsToAlgolia.ts +++ b/baker/algolia/indexChartsToAlgolia.ts @@ -17,7 +17,21 @@ const computeScore = (record: Omit): number => { const getChartsRecords = async ( knex: db.KnexReadonlyTransaction ): Promise => { - const chartsToIndex = await db.queryMysql(` + const chartsToIndex = await db.knexRaw<{ + id: number + slug: string + title: string + variantName: string + subtitle: string + availableEntities: string | string[] // initially this is a string but after parsing it is an array and the code below uses mutability + numDimensions: string + publishedAt: string + updatedAt: string + tags: string + keyChartForTags: string | string[] + }>( + knex, + `-- sql SELECT c.id, config ->> "$.slug" AS slug, config ->> "$.title" AS title, @@ -36,14 +50,15 @@ const getChartsRecords = async ( AND is_indexable IS TRUE GROUP BY c.id HAVING COUNT(t.id) >= 1 - `) + ` + ) for (const c of chartsToIndex) { if (c.availableEntities !== null) { // This is a very rough way to check for the Algolia record size limit, but it's better than the update failing // because we exceed the 20KB record size limit if (c.availableEntities.length < 12000) - c.availableEntities = JSON.parse(c.availableEntities) + c.availableEntities = JSON.parse(c.availableEntities as string) else { console.info( `Chart ${c.id} has too many entities, skipping its entities` @@ -53,7 +68,7 @@ const getChartsRecords = async ( } c.tags = JSON.parse(c.tags) - c.keyChartForTags = JSON.parse(c.keyChartForTags).filter( + c.keyChartForTags = JSON.parse(c.keyChartForTags as string).filter( (t: string | null) => t ) } @@ -68,7 +83,7 @@ const getChartsRecords = async ( const relatedArticles = (await getRelatedArticles(knex, c.id)) ?? [] const linksFromGdocs = await Link.getPublishedLinksTo( - c.slug, + [c.slug], OwidGdocLinkType.Grapher ) @@ -80,18 +95,18 @@ const getChartsRecords = async ( }).plaintext const record = { - objectID: c.id, + objectID: c.id.toString(), chartId: c.id, slug: c.slug, title: c.title, variantName: c.variantName, subtitle: plaintextSubtitle, - availableEntities: c.availableEntities, + availableEntities: c.availableEntities as string[], numDimensions: parseInt(c.numDimensions), publishedAt: c.publishedAt, updatedAt: c.updatedAt, - tags: c.tags, - keyChartForTags: c.keyChartForTags, + tags: c.tags as any as string[], + keyChartForTags: c.keyChartForTags as string[], titleLength: c.title.length, // Number of references to this chart in all our posts and pages numRelatedArticles: relatedArticles.length + linksFromGdocs.length, diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts index 3dde2b93ab9..14f6770c0ec 100644 --- a/baker/algolia/indexExplorersToAlgolia.ts +++ b/baker/algolia/indexExplorersToAlgolia.ts @@ -2,9 +2,11 @@ import cheerio from "cheerio" import { isArray } from "lodash" import { match } from "ts-pattern" import { + GrapherInterface, checkIsPlainObjectWithGuard, identity, keyBy, + parseChartConfig, } from "@ourworldindata/utils" import { getAlgoliaClient } from "./configureAlgolia.js" import * as db from "../../db/db.js" @@ -12,7 +14,6 @@ import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" import { chunkParagraphs } from "../chunk.js" import { SearchIndexName } from "../../site/search/searchTypes.js" -import { Chart } from "../../db/model/Chart.js" type ExplorerBlockColumns = { type: "columns" @@ -46,7 +47,7 @@ type ExplorerRecord = { function extractTextFromExplorer( blocksString: string, - graphersUsedInExplorers: Record + graphersUsedInExplorers: Record ): string { const blockText = new Set() const blocks = JSON.parse(blocksString) @@ -81,7 +82,6 @@ function extractTextFromExplorer( if (grapherId !== undefined) { const chartConfig = graphersUsedInExplorers[grapherId] - ?.config if (chartConfig) { blockText.add( @@ -117,24 +117,29 @@ const getExplorerRecords = async ( const pageviews = await getAnalyticsPageviewsByUrlObj(knex) // Fetch info about all charts used in explorers, as linked by the explorer_charts table - const graphersUsedInExplorers = await db - .knexRaw<{ chartId: number }>( - knex, - ` - SELECT DISTINCT chartId - FROM explorer_charts - ` + const graphersUsedInExplorersRaw = await db.knexRaw<{ + id: string + config: string + }>( + knex, + `-- sql + select id, config + from charts + where id in ( + SELECT DISTINCT chartId + FROM explorer_charts ) - .then((results: { chartId: number }[]) => - results.map(({ chartId }) => chartId) - ) - .then((ids) => Promise.all(ids.map((id) => Chart.findOneBy({ id })))) - .then((charts) => keyBy(charts, "id")) + ` + ) + const graphersUsedInExplorersEnriched = graphersUsedInExplorersRaw.map( + (row) => parseChartConfig(row.config) + ) + const graphersUsedInExplorers = keyBy(graphersUsedInExplorersEnriched, "id") const explorerRecords = await db .knexRaw>( knex, - ` + `-- sql SELECT slug, COALESCE(config->>"$.explorerSubtitle", "null") AS subtitle, COALESCE(config->>"$.explorerTitle", "null") AS title, diff --git a/baker/bakeGdocPost.ts b/baker/bakeGdocPost.ts index 7b5a7d9adb6..f0365ab246c 100644 --- a/baker/bakeGdocPost.ts +++ b/baker/bakeGdocPost.ts @@ -20,7 +20,9 @@ yargs(hideBin(process.argv)) const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) await db.getConnection() - await baker.bakeGDocPosts([slug]) + await db.knexReadonlyTransaction((trx) => + baker.bakeGDocPosts(trx, [slug]) + ) process.exit(0) } ) diff --git a/baker/bakeGdocPosts.ts b/baker/bakeGdocPosts.ts index 1e9fe3dd1e8..597e399a8b5 100644 --- a/baker/bakeGdocPosts.ts +++ b/baker/bakeGdocPosts.ts @@ -25,7 +25,9 @@ yargs(hideBin(process.argv)) const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) await db.getConnection() - await baker.bakeGDocPosts(slugs) + await db.knexReadonlyTransaction((trx) => + baker.bakeGDocPosts(trx, slugs) + ) process.exit(0) } ) diff --git a/baker/batchTagWithGpt.ts b/baker/batchTagWithGpt.ts index 9a290a416e1..1c7591e1dcd 100644 --- a/baker/batchTagWithGpt.ts +++ b/baker/batchTagWithGpt.ts @@ -1,5 +1,5 @@ import * as db from "../db/db.js" -import { Chart } from "../db/model/Chart.js" +import { getGptTopicSuggestions } from "../db/model/Chart.js" import yargs from "yargs" import { hideBin } from "yargs/helpers" @@ -22,17 +22,21 @@ export const batchTagWithGpt = async ({ debug, limit, }: BatchTagWithGptArgs = {}) => { - await batchTagChartsWithGpt({ debug, limit }) + db.knexReadonlyTransaction((trx) => + batchTagChartsWithGpt(trx, { debug, limit }) + ) } -const batchTagChartsWithGpt = async ({ - debug, - limit, -}: BatchTagWithGptArgs = {}) => { +const batchTagChartsWithGpt = async ( + knex: db.KnexReadonlyTransaction, + { debug, limit }: BatchTagWithGptArgs = {} +) => { // Identify all charts that need tagging. Get all charts that aren't tagged // with a topic tag or the "Unlisted" tag. This includes charts that have no // tags at all) - const chartsToTag = await db.queryMysql(` + const chartsToTag = await db.knexRaw<{ id: number }>( + knex, + `-- sql SELECT id FROM charts WHERE id @@ -44,11 +48,12 @@ const batchTagChartsWithGpt = async ({ ) GROUP BY id ${limit ? `LIMIT ${limit}` : ""} - `) + ` + ) // Iterate through the charts and tag them with GPT-suggested topics for (const chart of chartsToTag) { - const gptTopicSuggestions = await Chart.getGptTopicSuggestions(chart.id) + const gptTopicSuggestions = await getGptTopicSuggestions(knex, chart.id) for (const tag of gptTopicSuggestions) { if (debug) console.log("Tagging chart", chart.id, "with", tag.id) @@ -56,9 +61,12 @@ const batchTagChartsWithGpt = async ({ // exist, giving priority to the existing tags. This is to make sure // already curated tags and their associated key chart levels and // validation statuses are preserved. - await db.queryMysql(` + await db.knexRaw( + knex, + `-- sql INSERT IGNORE into chart_tags (chartId, tagId) VALUES (${chart.id},${tag.id}) - `) + ` + ) } } } @@ -84,7 +92,9 @@ if (require.main === module) { }, async (argv) => { try { - await batchTagChartsWithGpt(argv) + await db.knexReadonlyTransaction((trx) => + batchTagChartsWithGpt(trx, argv) + ) } finally { await db.closeTypeOrmAndKnexConnections() } diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 012fbf6551d..a930500aff9 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -73,7 +73,10 @@ import { ExplorerProgram } from "../explorer/ExplorerProgram.js" import { ExplorerPageUrlMigrationSpec } from "../explorer/urlMigrations/ExplorerPageUrlMigrationSpec.js" import { ExplorerPage } from "../site/ExplorerPage.js" import { DataInsightsIndexPage } from "../site/DataInsightsIndexPage.js" -import { Chart } from "../db/model/Chart.js" +import { + getChartConfigBySlug, + getEnrichedChartById, +} from "../db/model/Chart.js" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" import { ExplorerFullQueryParams } from "../explorer/ExplorerConstants.js" @@ -222,7 +225,7 @@ export const renderPost = async ( .map((el) => el.attribs["src"].trim()) // This can be slow if uncached! - await bakeGrapherUrls(grapherUrls) + await bakeGrapherUrls(knex, grapherUrls) grapherExports = await getGrapherExportsByUrl() } @@ -551,15 +554,16 @@ export const renderProminentLinks = async ( let title try { + // TODO: consider prefetching the related information instead of inline with 3 awaits here title = $block.find("title").text() || (!isCanonicalInternalUrl(resolvedUrl) ? null // attempt fallback for internal urls only : resolvedUrl.isExplorer - ? await getExplorerTitleByUrl(resolvedUrl) + ? await getExplorerTitleByUrl(knex, resolvedUrl) : resolvedUrl.isGrapher && resolvedUrl.slug - ? (await Chart.getBySlug(resolvedUrl.slug))?.config - ?.title // optim? + ? (await getChartConfigBySlug(knex, resolvedUrl.slug)) + ?.config?.title // optim? : resolvedUrl.slug && ( await getFullPostBySlugFromSnapshot( @@ -699,7 +703,10 @@ export const renderExplorerPage = async ( ) } -const getExplorerTitleByUrl = async (url: Url): Promise => { +const getExplorerTitleByUrl = async ( + knex: KnexReadonlyTransaction, + url: Url +): Promise => { if (!url.isExplorer || !url.slug) return // todo / optim: ok to instanciate multiple simple-git? const explorerAdminServer = new ExplorerAdminServer(GIT_CMS_DIR) @@ -711,8 +718,12 @@ const getExplorerTitleByUrl = async (url: Url): Promise => { return ( explorer.grapherConfig.title ?? (explorer.grapherConfig.grapherId - ? (await Chart.getById(explorer.grapherConfig.grapherId)) - ?.config?.title + ? ( + await getEnrichedChartById( + knex, + explorer.grapherConfig.grapherId + ) + )?.config?.title : undefined) ) } diff --git a/baker/sitemap.ts b/baker/sitemap.ts index d1f8adafe51..3d8f4c120a5 100644 --- a/baker/sitemap.ts +++ b/baker/sitemap.ts @@ -1,10 +1,14 @@ import { encodeXML } from "entities" -import { Chart } from "../db/model/Chart.js" import { BAKED_BASE_URL, BAKED_GRAPHER_URL, } from "../settings/serverSettings.js" -import { dayjs, countries, queryParamsToStr } from "@ourworldindata/utils" +import { + dayjs, + countries, + queryParamsToStr, + ChartsTableName, +} from "@ourworldindata/utils" import * as db from "../db/db.js" import urljoin from "url-join" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -77,7 +81,7 @@ export const makeSitemap = async ( ) const charts = (await knex - .table(Chart.table) + .table(ChartsTableName) .select(knex.raw(`updatedAt, config->>"$.slug" AS slug`)) .whereRaw('config->"$.isPublished" = true')) as { updatedAt: Date diff --git a/db/db.ts b/db/db.ts index f43e2f6ed82..a08afaa7ed4 100644 --- a/db/db.ts +++ b/db/db.ts @@ -147,7 +147,7 @@ export const knexRaw = async ( ): Promise => (await knex.raw(str, params ?? []))[0] export const knexRawFirst = async ( - knex: Knex, + knex: KnexReadonlyTransaction, str: string, params?: any[] ): Promise => { @@ -156,6 +156,12 @@ export const knexRawFirst = async ( return results[0] } +export const knexRawInsert = async ( + knex: KnexReadWriteTransaction, + str: string, + params?: any[] +): Promise<{ insertId: number }> => await knex.raw(str, params ?? []) + /** * In the backporting workflow, the users create gdoc posts for posts. As long as these are not yet published, * we still want to bake them from the WP posts. Once the users presses publish there though, we want to stop @@ -236,12 +242,12 @@ export const getPublishedExplorersBySlug = async ( } export const getPublishedDataInsights = ( - knex: Knex, + knex: KnexReadonlyTransaction, limit = Number.MAX_SAFE_INTEGER // default to no limit ): Promise => { return knexRaw( knex, - ` + `-- sql SELECT content->>'$.title' AS title, publishedAt, @@ -263,10 +269,12 @@ export const getPublishedDataInsights = ( ) as Promise } -export const getPublishedDataInsightCount = (): Promise => { +export const getPublishedDataInsightCount = ( + knex: KnexReadonlyTransaction +): Promise => { return knexRawFirst<{ count: number }>( - knexInstance(), - ` + knex, + `-- sql SELECT COUNT(*) AS count FROM posts_gdocs WHERE content->>'$.type' = '${OwidGdocType.DataInsight}' @@ -275,19 +283,23 @@ export const getPublishedDataInsightCount = (): Promise => { ).then((res) => res?.count ?? 0) } -export const getTotalNumberOfCharts = (): Promise => { +export const getTotalNumberOfCharts = ( + knex: KnexReadonlyTransaction +): Promise => { return knexRawFirst<{ count: number }>( - knexInstance(), - ` + knex, + `-- sql SELECT COUNT(*) AS count FROM charts WHERE config->"$.isPublished" = TRUE` ).then((res) => res?.count ?? 0) } -export const getTotalNumberOfInUseGrapherTags = (): Promise => { +export const getTotalNumberOfInUseGrapherTags = ( + knex: KnexReadonlyTransaction +): Promise => { return knexRawFirst<{ count: number }>( - knexInstance(), + knex, ` SELECT COUNT(DISTINCT(tagId)) AS count FROM chart_tags @@ -302,7 +314,7 @@ export const getTotalNumberOfInUseGrapherTags = (): Promise => { * For usage with GdocFactory.load, until we refactor Gdocs to be entirely Knex-based. */ export const getHomepageId = ( - knex: Knex + knex: KnexReadonlyTransaction ): Promise => { return knexRawFirst<{ id: string }>( knex, diff --git a/db/model/Chart.ts b/db/model/Chart.ts index a988b4ab41e..985d6b6ab7e 100644 --- a/db/model/Chart.ts +++ b/db/model/Chart.ts @@ -24,6 +24,11 @@ import { ChartTypeName, RelatedChart, DbPlainPostLink, + DbRawChart, + DbEnrichedChart, + parseChartsRow, + parseChartConfig, + ChartRedirect, } from "@ourworldindata/types" import { OpenAI } from "openai" import { @@ -54,43 +59,48 @@ export class Chart extends BaseEntity { publishedByUser!: Relation @OneToMany(() => ChartRevision, (rev) => rev.chart) logs!: Relation +} +// Only considers published charts, because only in that case the mapping slug -> id is unique +export async function mapSlugsToIds( + knex: db.KnexReadonlyTransaction +): Promise<{ [slug: string]: number }> { + const redirects = await db.knexRaw<{ chart_id: number; slug: string }>( + knex, + `SELECT chart_id, slug FROM chart_slug_redirects` + ) + const rows = await db.knexRaw<{ id: number; slug: string }>( + knex, + `-- sql + SELECT + id, + JSON_UNQUOTE(JSON_EXTRACT(config, "$.slug")) AS slug + FROM charts + WHERE config->>"$.isPublished" = "true" +` + ) - static table: string = "charts" - - // Only considers published charts, because only in that case the mapping slug -> id is unique - static async mapSlugsToIds(): Promise<{ [slug: string]: number }> { - const redirects = await db.queryMysql( - `SELECT chart_id, slug FROM chart_slug_redirects` - ) - const rows = await db.queryMysql(` - SELECT - id, - JSON_UNQUOTE(JSON_EXTRACT(config, "$.slug")) AS slug - FROM charts - WHERE config->>"$.isPublished" = "true" - `) - - const slugToId: { [slug: string]: number } = {} - for (const row of redirects) { - slugToId[row.slug] = row.chart_id - } - for (const row of rows) { - slugToId[row.slug] = row.id - } - return slugToId + const slugToId: { [slug: string]: number } = {} + for (const row of redirects) { + slugToId[row.slug] = row.chart_id } + for (const row of rows) { + slugToId[row.slug] = row.id + } + return slugToId +} - // Same as mapSlugsToIds but gets the configs also - // e.g. [ - // { slug: 'old-slug', id: 101, config: { isPublished: true, ...} }, - // { slug: 'new-slug', id: 101, config: { isPublished: true, ...} }, - // ] - static async mapSlugsToConfigs(): Promise< - { slug: string; id: number; config: GrapherInterface }[] - > { - return db - .queryMysql( - ` +// Same as mapSlugsToIds but gets the configs also +// e.g. [ +// { slug: 'old-slug', id: 101, config: { isPublished: true, ...} }, +// { slug: 'new-slug', id: 101, config: { isPublished: true, ...} }, +// ] +export async function mapSlugsToConfigs( + knex: db.KnexReadonlyTransaction +): Promise<{ slug: string; id: number; config: GrapherInterface }[]> { + return db + .knexRaw<{ slug: string; config: string; id: number }>( + knex, + ` SELECT csr.slug AS slug, c.config AS config, c.id AS id FROM chart_slug_redirects csr JOIN charts c @@ -101,121 +111,230 @@ SELECT c.slug AS slug, c.config AS config, c.id AS id FROM charts c WHERE c.config -> "$.isPublished" = true ` - ) - .then((results) => - results.map( - (result: { slug: string; id: number; config: string }) => ({ - ...result, - config: JSON.parse(result.config), - }) - ) - ) - } + ) + .then((results) => + results.map((result) => ({ + ...result, + config: JSON.parse(result.config), + })) + ) +} - static async getBySlug(slug: string): Promise { - const slugToIdMap = await this.mapSlugsToIds() - const chartId = slugToIdMap[slug] - if (chartId === undefined) return null - return await Chart.findOneBy({ id: chartId }) - } +export async function getEnrichedChartBySlug( + knex: db.KnexReadonlyTransaction, + slug: string +): Promise { + let chart = await db.knexRawFirst( + knex, + `SELECT * FROM charts WHERE config ->> '$.slug' = ?`, + [slug] + ) - static async getById(id: number): Promise { - return await Chart.findOneBy({ id }) + if (!chart) { + chart = await db.knexRawFirst( + knex, + `select c.* + from chart_slug_redirects csr + join charts c on csr.chart_id = c.id + where csr.slug = ?`, + [slug] + ) } - static async setTags( - chartId: number, - tags: DbChartTagJoin[] - ): Promise { - await db.transaction(async (t) => { - const tagRows = tags.map((tag) => [ - tag.id, - chartId, - tag.keyChartLevel ?? KeyChartLevel.None, - tag.isApproved ? 1 : 0, - ]) - await t.execute(`DELETE FROM chart_tags WHERE chartId=?`, [chartId]) - if (tagRows.length) - await t.execute( - `INSERT INTO chart_tags (tagId, chartId, keyChartLevel, isApproved) VALUES ?`, - [tagRows] - ) + if (!chart) return null + + const enrichedChart = parseChartsRow(chart) + + return enrichedChart +} + +export async function getRawChartById( + knex: db.KnexReadonlyTransaction, + id: number +): Promise { + const chart = await db.knexRawFirst( + knex, + `SELECT * FROM charts WHERE id = ?`, + [id] + ) + if (!chart) return null + return chart +} + +export async function getEnrichedChartById( + knex: db.KnexReadonlyTransaction, + id: number +): Promise { + const rawChart = await getRawChartById(knex, id) + if (!rawChart) return null + return parseChartsRow(rawChart) +} + +export async function getChartSlugById( + knex: db.KnexReadonlyTransaction, + id: number +): Promise { + const chart = await db.knexRawFirst>( + knex, + `SELECT config ->> '$.slug' FROM charts WHERE id = ?`, + [id] + ) + if (!chart) return null + return chart.slug +} + +export const getChartConfigById = async ( + knex: db.KnexReadonlyTransaction, + grapherId: number +): Promise | undefined> => { + const grapher = await db.knexRawFirst>( + knex, + `SELECT id, config FROM charts WHERE id=?`, + [grapherId] + ) - const parentIds = tags.length - ? ((await t.query("select parentId from tags where id in (?)", [ - tags.map((t) => t.id), - ])) as { parentId: number }[]) - : [] - - // A chart is indexable if it is not tagged "Unlisted" and has at - // least one public parent tag - const isIndexable = tags.some((t) => t.name === "Unlisted") - ? false - : parentIds.some((t) => - PUBLIC_TAG_PARENT_IDS.includes(t.parentId) - ) - await t.execute("update charts set is_indexable = ? where id = ?", [ - isIndexable, - chartId, - ]) - }) + if (!grapher) return undefined + + return { + id: grapher.id, + config: parseChartConfig(grapher.config), } +} + +export async function getChartConfigBySlug( + knex: db.KnexReadonlyTransaction, + slug: string +): Promise> { + const row = await db.knexRawFirst>( + knex, + `SELECT id, config FROM charts WHERE JSON_EXTRACT(config, "$.slug") = ?`, + [slug] + ) + + if (!row) throw new JsonError(`No chart found for slug ${slug}`, 404) + + return { id: row.id, config: JSON.parse(row.config) } +} - static async assignTagsForCharts( - charts: { id: number; tags: any[] }[] - ): Promise { - const chartTags = await db.queryMysql(` - SELECT ct.chartId, ct.tagId, ct.keyChartLevel, ct.isApproved, t.name as tagName FROM chart_tags ct +export async function setChartTags( + knex: db.KnexReadWriteTransaction, + chartId: number, + tags: DbChartTagJoin[] +): Promise { + const tagRows = tags.map((tag) => [ + tag.id, + chartId, + tag.keyChartLevel ?? KeyChartLevel.None, + tag.isApproved ? 1 : 0, + ]) + await db.knexRaw(knex, `DELETE FROM chart_tags WHERE chartId=?`, [chartId]) + if (tagRows.length) + await db.knexRaw( + knex, + `INSERT INTO chart_tags (tagId, chartId, keyChartLevel, isApproved) VALUES ?`, + [tagRows] + ) + + const parentIds = tags.length + ? await db.knexRaw<{ parentId: number }>( + knex, + "select parentId from tags where id in (?)", + [tags.map((t) => t.id)] + ) + : [] + + // A chart is indexable if it is not tagged "Unlisted" and has at + // least one public parent tag + const isIndexable = tags.some((t) => t.name === "Unlisted") + ? false + : parentIds.some((t) => PUBLIC_TAG_PARENT_IDS.includes(t.parentId)) + await db.knexRaw(knex, "update charts set is_indexable = ? where id = ?", [ + isIndexable, + chartId, + ]) +} + +export async function assignTagsForCharts( + knex: db.KnexReadonlyTransaction, + charts: { + id: number + tags?: { + id: number + name: string + keyChartLevel: number + isApproved: boolean + }[] + }[] +): Promise { + const chartTags = await db.knexRaw<{ + chartId: number + tagId: number + keyChartLevel: number + isApproved: number | undefined + tagName: string + }>( + knex, + `-- sql + SELECT ct.chartId, ct.tagId, ct.keyChartLevel, ct.isApproved, t.name as tagName + FROM chart_tags ct JOIN charts c ON c.id=ct.chartId JOIN tags t ON t.id=ct.tagId - `) - - for (const chart of charts) { - chart.tags = [] - } - - const chartsById = lodash.keyBy(charts, (c) => c.id) - - for (const ct of chartTags) { - const chart = chartsById[ct.chartId] - if (chart) - chart.tags.push({ - id: ct.tagId, - name: ct.tagName, - keyChartLevel: ct.keyChartLevel, - isApproved: !!ct.isApproved, - }) - } - } + ` + ) - static async getGptTopicSuggestions( - chartId: number - ): Promise[]> { - if (!OPENAI_API_KEY) - throw new JsonError("No OPENAI_API_KEY env found", 500) + for (const chart of charts) { + chart.tags = [] + } - const chart = await Chart.findOneBy({ - id: chartId, - }) - if (!chart) throw new JsonError(`No chart found for id ${chartId}`, 404) + const chartsById = lodash.keyBy(charts, (c) => c.id) + + for (const ct of chartTags) { + const chart = chartsById[ct.chartId] + if (chart) + chart.tags!.push({ + id: ct.tagId, + name: ct.tagName, + keyChartLevel: ct.keyChartLevel, + isApproved: !!ct.isApproved, + }) + } +} - const topics: Pick[] = await db.queryMysql(` +export async function getGptTopicSuggestions( + knex: db.KnexReadonlyTransaction, + chartId: number +): Promise[]> { + if (!OPENAI_API_KEY) throw new JsonError("No OPENAI_API_KEY env found", 500) + + const chartConfigOnly: Pick | undefined = await knex + .table("charts") + .select("config") + .where({ id: chartId }) + .first() + if (!chartConfigOnly) + throw new JsonError(`No chart found for id ${chartId}`, 404) + const enrichedChartConfig = parseChartConfig(chartConfigOnly.config) + + const topics: Pick[] = await db.knexRaw( + knex, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.slug IS NOT NULL AND t.parentId IN (${PUBLIC_TAG_PARENT_IDS.join(",")}) - `) + ` + ) - if (!topics.length) throw new JsonError("No topics found", 404) + if (!topics.length) throw new JsonError("No topics found", 404) - const prompt = ` + const prompt = ` You will be provided with the chart metadata (delimited with XML tags), as well as a list of possible topics (delimited with XML tags). Classify the chart into two of the provided topics. - ${chart.config.title} - ${chart.config.subtitle} - ${chart.config.originUrl} + ${enrichedChartConfig.title} + ${enrichedChartConfig.subtitle} + ${enrichedChartConfig.originUrl} ${topics.map( @@ -231,37 +350,54 @@ WHERE c.config -> "$.isPublished" = true { "id": 2, "name": "Topic 2" } ]` - const openai = new OpenAI({ - apiKey: OPENAI_API_KEY, - }) - const completion = await openai.chat.completions.create({ - messages: [{ role: "user", content: prompt }], - model: "gpt-4-1106-preview", - }) - - const json = completion.choices[0]?.message?.content - if (!json) throw new JsonError("No response from GPT", 500) - - const selectedTopics: unknown = JSON.parse(json) - - if (lodash.isArray(selectedTopics)) { - // We only want to return topics that are in the list of possible - // topics, in case of hallucinations - const confirmedTopics = selectedTopics.filter((topic) => - topics.map((t) => t.id).includes(topic.id) - ) - - return confirmedTopics - } else { - console.error("GPT returned invalid response", json) - return [] - } + const openai = new OpenAI({ + apiKey: OPENAI_API_KEY, + }) + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: prompt }], + model: "gpt-4-1106-preview", + }) + + const json = completion.choices[0]?.message?.content + if (!json) throw new JsonError("No response from GPT", 500) + + const selectedTopics: unknown = JSON.parse(json) + + if (lodash.isArray(selectedTopics)) { + // We only want to return topics that are in the list of possible + // topics, in case of hallucinations + const confirmedTopics = selectedTopics.filter((topic) => + topics.map((t) => t.id).includes(topic.id) + ) + + return confirmedTopics + } else { + console.error("GPT returned invalid response", json) + return [] } } -// TODO integrate this old logic with typeorm -export class OldChart { - static listFields = ` +export interface OldChartFieldList { + id: number + title: string + slug: string + type: string + internalNotes: string + variantName: string + isPublished: boolean + tab: string + hasChartTab: boolean + hasMapTab: boolean + lastEditedAt: Date + lastEditedByUserId: number + lastEditedBy: string + publishedAt: Date + publishedByUserId: number + publishedBy: string + isExplorable: boolean +} + +export const oldChartFieldList = ` charts.id, charts.config->>"$.title" AS title, charts.config->>"$.slug" AS slug, @@ -279,55 +415,27 @@ export class OldChart { charts.publishedByUserId, publishedByUser.fullName AS publishedBy ` +// TODO: replace this with getBySlug and pick - static async getBySlug(slug: string): Promise { - const row = await db.mysqlFirst( - `SELECT id, config FROM charts WHERE JSON_EXTRACT(config, "$.slug") = ?`, - [slug] - ) - - return new OldChart(row.id, JSON.parse(row.config)) - } - - id: number - config: any - constructor(id: number, config: Record) { - this.id = id - this.config = config - - // XXX todo make the relationship between chart models and chart configuration more defined - this.config.id = id - } - - async getVariableData(): Promise { - const variableIds = lodash.uniq( - this.config.dimensions!.map((d: any) => d.variableId) - ) - const allVariablesDataAndMetadataMap = - await getDataForMultipleVariables(variableIds as number[]) - return allVariablesDataAndMetadataMap - } -} - -export const getGrapherById = async (grapherId: number): Promise => { - const grapher = ( - await db.queryMysql(`SELECT id, config FROM charts WHERE id=?`, [ - grapherId, - ]) - )[0] - - if (!grapher) return undefined - - const config = JSON.parse(grapher.config) - config.id = grapher.id - return config +export async function getChartVariableData( + config: GrapherInterface +): Promise { + const variableIds = lodash.uniq( + config.dimensions!.map((d: any) => d.variableId) + ) + const allVariablesDataAndMetadataMap = await getDataForMultipleVariables( + variableIds as number[] + ) + return allVariablesDataAndMetadataMap } export const getMostViewedGrapherIdsByChartType = async ( + knex: db.KnexReadonlyTransaction, chartType: ChartTypeName, count = 10 ): Promise => { - const ids = await db.queryMysql( + const ids = await db.knexRaw<{ id: number }>( + knex, `SELECT c.id FROM analytics_pageviews a JOIN charts c ON c.slug = SUBSTRING_INDEX(a.url, '/', -1) @@ -339,10 +447,11 @@ export const getMostViewedGrapherIdsByChartType = async ( LIMIT ?`, [chartType, count] ) - return ids.map((row: any) => row.id) + return ids.map((row) => row.id) } export const getRelatedChartsForVariable = async ( + knex: db.KnexReadonlyTransaction, variableId: number, chartIdsToExclude: number[] = [] ): Promise => { @@ -351,7 +460,9 @@ export const getRelatedChartsForVariable = async ( ? `AND charts.id NOT IN (${chartIdsToExclude.join(", ")})` : "" - return db.queryMysql(`-- sql + return db.knexRaw( + knex, + `-- sql SELECT charts.config->>"$.slug" AS slug, charts.config->>"$.title" AS title, @@ -364,7 +475,8 @@ export const getRelatedChartsForVariable = async ( ${excludeChartIds} GROUP BY charts.id ORDER BY title ASC - `) + ` + ) } export const getChartEmbedUrlsInPublishedWordpressPosts = async ( @@ -375,7 +487,7 @@ export const getChartEmbedUrlsInPublishedWordpressPosts = async ( "target" | "queryString" >[] = await db.knexRaw( knex, - ` + `-- sql SELECT pl.target, pl.queryString @@ -419,3 +531,17 @@ export const getChartEmbedUrlsInPublishedWordpressPosts = async ( return `${BAKED_BASE_URL}/${row.target}${row.queryString}` }) } + +export const getRedirectsByChartId = async ( + knex: db.KnexReadonlyTransaction, + chartId: number +): Promise => + await db.knexRaw( + knex, + `-- sql + SELECT id, slug, chart_id as chartId + FROM chart_slug_redirects + WHERE chart_id = ? + ORDER BY id ASC`, + [chartId] + ) diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 0ac51610f54..8b4c0797302 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -45,7 +45,7 @@ import { gdocToArchie } from "./gdocToArchie.js" import { archieToEnriched } from "./archieToEnriched.js" import { Link } from "../Link.js" import { imageStore } from "../Image.js" -import { Chart } from "../Chart.js" +import { getChartConfigById, mapSlugsToIds } from "../Chart.js" import { BAKED_BASE_URL, BAKED_GRAPHER_EXPORTS_BASE_URL, @@ -619,36 +619,42 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { } async loadLinkedCharts(): Promise { - const slugToIdMap = await Chart.mapSlugsToIds() - const linkedGrapherCharts = await Promise.all( - [...this.linkedChartSlugs.grapher.values()].map( - async (originalSlug) => { - const chartId = slugToIdMap[originalSlug] - if (!chartId) return - const chart = await Chart.findOneBy({ id: chartId }) - if (!chart) return - const resolvedSlug = chart.config.slug ?? "" - const resolvedTitle = chart.config.title ?? "" - const tab = chart.config.tab ?? GrapherTabOption.chart - const datapageIndicator = - await getVariableOfDatapageIfApplicable(chart.config) - const linkedChart: LinkedChart = { - originalSlug, - title: resolvedTitle, - tab, - resolvedUrl: `${BAKED_GRAPHER_URL}/${resolvedSlug}`, - thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${resolvedSlug}.svg`, - tags: [], - indicatorId: datapageIndicator?.id, - } - return linkedChart - } - ) - ).then(excludeNullish) + const { linkedGrapherCharts, publishedExplorersBySlug } = + await db.knexReadonlyTransaction(async (trx) => { + const slugToIdMap = await mapSlugsToIds(trx) + const linkedGrapherCharts = await Promise.all( + [...this.linkedChartSlugs.grapher.values()].map( + async (originalSlug) => { + const chartId = slugToIdMap[originalSlug] + if (!chartId) return + const chart = await getChartConfigById(trx, chartId) + if (!chart) return + const resolvedSlug = chart.config.slug ?? "" + const resolvedTitle = chart.config.title ?? "" + const tab = + chart.config.tab ?? GrapherTabOption.chart + const datapageIndicator = + await getVariableOfDatapageIfApplicable( + chart.config + ) + const linkedChart: LinkedChart = { + originalSlug, + title: resolvedTitle, + tab, + resolvedUrl: `${BAKED_GRAPHER_URL}/${resolvedSlug}`, + thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${resolvedSlug}.svg`, + tags: [], + indicatorId: datapageIndicator?.id, + } + return linkedChart + } + ) + ).then(excludeNullish) - const publishedExplorersBySlug = await db.knexReadonlyTransaction( - (trx) => db.getPublishedExplorersBySlug(trx) - ) + const publishedExplorersBySlug = + await db.getPublishedExplorersBySlug(trx) + return { linkedGrapherCharts, publishedExplorersBySlug } + }) const linkedExplorerCharts = await Promise.all( this.linkedChartSlugs.explorer.map((originalSlug) => { @@ -772,10 +778,13 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { [] ) - const chartIdsBySlug = await Chart.mapSlugsToIds() - const publishedExplorersBySlug = await db.knexReadonlyTransaction( - (trx) => db.getPublishedExplorersBySlug(trx) - ) + const { chartIdsBySlug, publishedExplorersBySlug } = + await db.knexReadonlyTransaction(async (trx) => { + const chartIdsBySlug = await mapSlugsToIds(trx) + const publishedExplorersBySlug = + await db.getPublishedExplorersBySlug(trx) + return { chartIdsBySlug, publishedExplorersBySlug } + }) const linkErrors: OwidGdocErrorMessage[] = this.links.reduce( (errors: OwidGdocErrorMessage[], link): OwidGdocErrorMessage[] => { diff --git a/db/model/Gdoc/GdocDataInsight.ts b/db/model/Gdoc/GdocDataInsight.ts index 9017bea5e78..c57049e2996 100644 --- a/db/model/Gdoc/GdocDataInsight.ts +++ b/db/model/Gdoc/GdocDataInsight.ts @@ -47,7 +47,7 @@ export class GdocDataInsight _loadSubclassAttachments = async (): Promise => { // TODO: refactor these classes to properly use knex - not going to start it now this.latestDataInsights = await db.getPublishedDataInsights( - db.knexInstance(), + db.knexInstance() as db.KnexReadonlyTransaction, // TODO: replace this with a transaction that is passed in 5 ) } diff --git a/db/model/Gdoc/GdocHomepage.ts b/db/model/Gdoc/GdocHomepage.ts index 198f8dca111..7295ce72df6 100644 --- a/db/model/Gdoc/GdocHomepage.ts +++ b/db/model/Gdoc/GdocHomepage.ts @@ -55,15 +55,13 @@ export class GdocHomepage } _loadSubclassAttachments = async (): Promise => { + const knex = db.knexInstance() as db.KnexReadonlyTransaction this.homepageMetadata = { - chartCount: await db.getTotalNumberOfCharts(), + chartCount: await db.getTotalNumberOfCharts(knex), // TODO: replace this with a transaction that is passed in topicCount: UNIQUE_TOPIC_COUNT, } // TODO: refactor these classes to properly use knex - not going to start it now - this.latestDataInsights = await db.getPublishedDataInsights( - db.knexInstance(), - 4 - ) + this.latestDataInsights = await db.getPublishedDataInsights(knex, 4) } } diff --git a/db/tests/basic.test.ts b/db/tests/basic.test.ts index 784686d470c..29b208569df 100644 --- a/db/tests/basic.test.ts +++ b/db/tests/basic.test.ts @@ -9,12 +9,18 @@ import { knexReadWriteTransaction, KnexReadonlyTransaction, KnexReadWriteTransaction, + knexRawFirst, knexReadonlyTransaction, } from "../db.js" import { DataSource } from "typeorm" import { deleteUser, insertUser, updateUser, User } from "../model/User.js" -import { Chart } from "../model/Chart.js" -import { DbPlainUser, UsersTableName } from "@ourworldindata/types" +import { + ChartsTableName, + DbInsertChart, + DbPlainUser, + DbRawChart, + UsersTableName, +} from "@ourworldindata/types" let knexInstance: Knex | undefined = undefined let typeOrmConnection: DataSource | undefined = undefined @@ -68,32 +74,46 @@ function sleep(time: number, value: any): Promise { } test("timestamps are automatically created and updated", async () => { - const chart = new Chart() - chart.config = {} - chart.lastEditedAt = new Date() - chart.lastEditedByUserId = 1 - await chart.save() - const created: Chart | null = await Chart.findOne({ where: { id: 1 } }) - expect(created).not.toBeNull() - if (created) { - expect(created.createdAt).not.toBeNull() - expect(created.updatedAt).toBeNull() - await sleep(1000, undefined) - created.lastEditedAt = new Date() - await created.save() - const updated: Chart | null = await Chart.findOne({ where: { id: 1 } }) - expect(updated).not.toBeNull() - if (updated) { - expect(updated.createdAt).not.toBeNull() - expect(updated.updatedAt).not.toBeNull() - expect( - updated.updatedAt.getTime() - updated.createdAt.getTime() - ).toBeGreaterThan(800) - expect( - updated.updatedAt.getTime() - updated.createdAt.getTime() - ).toBeLessThanOrEqual(2000) + knexReadWriteTransaction(async (trx) => { + const chart: DbInsertChart = { + config: "{}", + lastEditedAt: new Date(), + lastEditedByUserId: 1, + is_indexable: 0, } - } + await trx.table(ChartsTableName).insert(chart) + const created = await knexRawFirst( + trx, + "select * from charts where id = 1", + [] + ) + expect(created).not.toBeNull() + if (created) { + expect(created.createdAt).not.toBeNull() + expect(created.updatedAt).toBeNull() + await sleep(1000, undefined) + await trx + .table(ChartsTableName) + .where({ id: 1 }) + .update({ is_indexable: 1 }) + const updated = await knexRawFirst( + trx, + "select * from charts where id = 1", + [] + ) + expect(updated).not.toBeNull() + if (updated) { + expect(updated.createdAt).not.toBeNull() + expect(updated.updatedAt).not.toBeNull() + expect( + updated.updatedAt!.getTime() - updated.createdAt.getTime() + ).toBeGreaterThan(800) + expect( + updated.updatedAt!.getTime() - updated.createdAt.getTime() + ).toBeLessThanOrEqual(2000) + } + } + }, knexInstance) }) test("knex interface", async () => { diff --git a/devTools/svgTester/dump-chart-ids.ts b/devTools/svgTester/dump-chart-ids.ts index 6dac33bcc8c..f66ba1db423 100644 --- a/devTools/svgTester/dump-chart-ids.ts +++ b/devTools/svgTester/dump-chart-ids.ts @@ -3,7 +3,10 @@ import fs from "fs-extra" import parseArgs from "minimist" -import { closeTypeOrmAndKnexConnections } from "../../db/db.js" +import { + closeTypeOrmAndKnexConnections, + knexReadonlyTransaction, +} from "../../db/db.js" import { getMostViewedGrapherIdsByChartType } from "../../db/model/Chart.js" import { CHART_TYPES } from "./utils.js" @@ -14,10 +17,17 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { try { const outFile = parsedArgs["o"] ?? DEFAULT_OUT_FILE - const promises = CHART_TYPES.flatMap((chartType) => - getMostViewedGrapherIdsByChartType(chartType, CHART_COUNT_PER_TYPE) - ) - const chartIds = (await Promise.all(promises)).flatMap((ids) => ids) + const chartIds = await knexReadonlyTransaction(async (trx) => { + const promises = CHART_TYPES.flatMap((chartType) => + getMostViewedGrapherIdsByChartType( + trx, + chartType, + CHART_COUNT_PER_TYPE + ) + ) + const chartIds = (await Promise.all(promises)).flatMap((ids) => ids) + return chartIds + }) console.log(`Writing ${chartIds.length} chart ids to ${outFile}`) diff --git a/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts b/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts index 3218337b802..4f6d399064e 100644 --- a/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts +++ b/packages/@ourworldindata/types/src/grapherTypes/GrapherTypes.ts @@ -680,3 +680,9 @@ export enum GrapherStaticFormat { landscape = "landscape", square = "square", } + +export interface ChartRedirect { + id: number + slug: string + chartId: number +} diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 3759e4d2e3a..f4800c3255a 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -109,6 +109,7 @@ export { type SeriesName, type LegacyGrapherQueryParams, GrapherStaticFormat, + type ChartRedirect, type DetailsMarker, } from "./grapherTypes/GrapherTypes.js"