🔨 (types) migrate Dataset and Source to knex

adminSiteServer/adminRouter.tsx

@@ -8,7 +8,7 @@ import { expectInt, renderToHtmlPage } from "../serverUtils/serverUtil.js"
 import { logInWithCredentials, logOut } from "./authentication.js"
 import { LoginPage } from "./LoginPage.js"
 import * as db from "../db/db.js"
-import { Dataset } from "../db/model/Dataset.js"
+import { writeDatasetCSV } from "../db/model/Dataset.js"
 import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js"
 import {
     renderExplorerPage,
@@ -43,6 +43,7 @@ import {
 } from "../baker/GrapherBaker.js"
 import { Chart } from "../db/model/Chart.js"
 import { getVariableMetadata } from "../db/model/Variable.js"
+import { DatasetFilesRow, DatasetsRow } from "@ourworldindata/types"
 
 // Used for rate-limiting important endpoints (login, register) to prevent brute force attacks
 const limiterMiddleware = (
@@ -138,17 +139,21 @@ adminRouter.get("/datasets/:datasetId.csv", async (req, res) => {
     const datasetId = expectInt(req.params.datasetId)
 
     const datasetName = (
-        await db.mysqlFirst(`SELECT name FROM datasets WHERE id=?`, [datasetId])
-    ).name
-    res.attachment(filenamify(datasetName) + ".csv")
+        await db.knexRawFirst<Pick<DatasetsRow, "name">>(
+            `SELECT name FROM datasets WHERE id=?`,
+            [datasetId]
+        )
+    )?.name
+
+    res.attachment(filenamify(datasetName!) + ".csv")
 
     const writeStream = new Writable({
         write(chunk, encoding, callback) {
             res.write(chunk.toString())
             callback(null)
         },
     })
-    await Dataset.writeCSV(datasetId, writeStream)
+    await writeDatasetCSV(db.knexInstance(), datasetId, writeStream)
     res.end()
 })
 
@@ -157,11 +162,10 @@ adminRouter.get("/datasets/:datasetId/downloadZip", async (req, res) => {
 
     res.attachment("additional-material.zip")
 
-    const file = await db.mysqlFirst(
-        `SELECT filename, file FROM dataset_files WHERE datasetId=?`,
-        [datasetId]
-    )
-    res.send(file.file)
+    const file = await db.knexRawFirst<
+        Pick<DatasetFilesRow, "filename" | "file">
+    >(`SELECT filename, file FROM dataset_files WHERE datasetId=?`, [datasetId])
+    res.send(file?.file)
 })
 
 adminRouter.get("/posts/preview/:postId", async (req, res) => {

adminSiteServer/apiRouter.ts

@@ -51,8 +51,10 @@ import {
     OwidGdoc,
 } from "@ourworldindata/utils"
 import {
+    DatasetTagsRow,
     GrapherInterface,
     OwidGdocLinkType,
+    TagsRow,
     grapherKeysToSerialize,
 } from "@ourworldindata/types"
 import {
@@ -63,7 +65,7 @@ import {
     CountryNameFormat,
     CountryDefByKey,
 } from "../adminSiteClient/CountryNameFormat.js"
-import { Dataset } from "../db/model/Dataset.js"
+import { getDatasetById, setTagsForDataset } from "../db/model/Dataset.js"
 import { User } from "../db/model/User.js"
 import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
 import { GdocBase, Tag as TagEntity } from "../db/model/Gdoc/GdocBase.js"
@@ -1744,7 +1746,7 @@ apiRouter.get(
 )
 
 apiRouter.get("/datasets.json", async (req) => {
-    const datasets = await db.queryMysql(`
+    const datasets = await db.knexRaw<Record<string, any>>(`
         WITH variable_counts AS (
             SELECT
                 v.datasetId,
@@ -1775,7 +1777,9 @@ apiRouter.get("/datasets.json", async (req) => {
         ORDER BY ad.dataEditedAt DESC
     `)
 
-    const tags = await db.queryMysql(`
+    const tags = await db.knexRaw<
+        Pick<TagsRow, "id" | "name"> & Pick<DatasetTagsRow, "datasetId">
+    >(`
         SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt
         JOIN tags t ON dt.tagId = t.id
     `)
@@ -1794,7 +1798,7 @@ apiRouter.get("/datasets.json", async (req) => {
 apiRouter.get("/datasets/:datasetId.json", async (req: Request) => {
     const datasetId = expectInt(req.params.datasetId)
 
-    const dataset = await db.mysqlFirst(
+    const dataset = await db.knexRawFirst<Record<string, any>>(
         `
         SELECT d.id,
             d.namespace,
@@ -1863,7 +1867,7 @@ apiRouter.get("/datasets/:datasetId.json", async (req: Request) => {
 
     dataset.origins = origins
 
-    const sources = await db.queryMysql(
+    const sources = await db.knexRaw(
         `
         SELECT s.id, s.name, s.description
         FROM sources AS s
@@ -1926,12 +1930,13 @@ apiRouter.put("/datasets/:datasetId", async (req: Request, res: Response) => {
     // Only updates `nonRedistributable` and `tags`, other fields come from ETL
     // and are not editable
     const datasetId = expectInt(req.params.datasetId)
-    const dataset = await Dataset.findOneBy({ id: datasetId })
+    const knex = db.knexInstance()
+    const dataset = await getDatasetById(knex, datasetId)
     if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
 
-    await db.transaction(async (t) => {
+    await knex.transaction(async (t) => {
         const newDataset = (req.body as { dataset: any }).dataset
-        await t.execute(
+        await t.raw(
             `
             UPDATE datasets
             SET
@@ -1949,27 +1954,24 @@ apiRouter.put("/datasets/:datasetId", async (req: Request, res: Response) => {
         )
 
         const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId])
-        await t.execute(`DELETE FROM dataset_tags WHERE datasetId=?`, [
-            datasetId,
-        ])
+        await t.raw(`DELETE FROM dataset_tags WHERE datasetId=?`, [datasetId])
         if (tagRows.length)
-            await t.execute(
+            await t.raw(
                 `INSERT INTO dataset_tags (tagId, datasetId) VALUES ?`,
                 [tagRows]
             )
-    })
 
-    // Note: not currently in transaction
-    try {
-        await syncDatasetToGitRepo(datasetId, {
-            oldDatasetName: dataset.name,
-            commitName: res.locals.user.fullName,
-            commitEmail: res.locals.user.email,
-        })
-    } catch (err) {
-        logErrorAndMaybeSendToBugsnag(err, req)
-        // Continue
-    }
+        try {
+            await syncDatasetToGitRepo(t, datasetId, {
+                oldDatasetName: dataset.name!,
+                commitName: res.locals.user.fullName,
+                commitEmail: res.locals.user.email,
+            })
+        } catch (err) {
+            logErrorAndMaybeSendToBugsnag(err, req)
+            // Continue
+        }
+    })
 
     return { success: true }
 })
@@ -1978,10 +1980,11 @@ apiRouter.post(
     "/datasets/:datasetId/setArchived",
     async (req: Request, res: Response) => {
         const datasetId = expectInt(req.params.datasetId)
-        const dataset = await Dataset.findOneBy({ id: datasetId })
+        const knex = db.knexInstance()
+        const dataset = await getDatasetById(knex, datasetId)
         if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
-        await db.transaction(async (t) => {
-            await t.execute(`UPDATE datasets SET isArchived = 1 WHERE id=?`, [
+        await knex.transaction(async (t) => {
+            await t.raw(`UPDATE datasets SET isArchived = 1 WHERE id=?`, [
                 datasetId,
             ])
         })
@@ -1994,7 +1997,7 @@ apiRouter.post(
     async (req: Request, res: Response) => {
         const datasetId = expectInt(req.params.datasetId)
 
-        await Dataset.setTags(datasetId, req.body.tagIds)
+        await setTagsForDataset(db.knexInstance(), datasetId, req.body.tagIds)
 
         return { success: true }
     }
@@ -2005,28 +2008,25 @@ apiRouter.delete(
     async (req: Request, res: Response) => {
         const datasetId = expectInt(req.params.datasetId)
 
-        const dataset = await Dataset.findOneBy({ id: datasetId })
+        const knex = db.knexInstance()
+        const dataset = await getDatasetById(knex, datasetId)
         if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
 
-        await db.transaction(async (t) => {
-            await t.execute(
+        await knex.transaction(async (t) => {
+            await t.raw(
                 `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`,
                 [datasetId]
             )
-            await t.execute(`DELETE FROM dataset_files WHERE datasetId=?`, [
-                datasetId,
-            ])
-            await t.execute(`DELETE FROM variables WHERE datasetId=?`, [
+            await t.raw(`DELETE FROM dataset_files WHERE datasetId=?`, [
                 datasetId,
             ])
-            await t.execute(`DELETE FROM sources WHERE datasetId=?`, [
-                datasetId,
-            ])
-            await t.execute(`DELETE FROM datasets WHERE id=?`, [datasetId])
+            await t.raw(`DELETE FROM variables WHERE datasetId=?`, [datasetId])
+            await t.raw(`DELETE FROM sources WHERE datasetId=?`, [datasetId])
+            await t.raw(`DELETE FROM datasets WHERE id=?`, [datasetId])
         })
 
         try {
-            await removeDatasetFromGitRepo(dataset.name, dataset.namespace, {
+            await removeDatasetFromGitRepo(dataset.name!, dataset.namespace, {
                 commitName: res.locals.user.fullName,
                 commitEmail: res.locals.user.email,
             })
@@ -2044,12 +2044,13 @@ apiRouter.post(
     async (req: Request, res: Response) => {
         const datasetId = expectInt(req.params.datasetId)
 
-        const dataset = await Dataset.findOneBy({ id: datasetId })
+        const knex = db.knexInstance()
+        const dataset = await getDatasetById(knex, datasetId)
         if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404)
 
         if (req.body.republish) {
-            await db.transaction(async (t) => {
-                await t.execute(
+            await knex.transaction(async (t) => {
+                await t.raw(
                     `
             UPDATE charts
             SET config = JSON_SET(config, "$.version", config->"$.version" + 1)
@@ -2450,16 +2451,16 @@ apiRouter.post("/posts/:postId/unlinkGdoc", async (req: Request) => {
 
 apiRouter.get("/sources/:sourceId.json", async (req: Request) => {
     const sourceId = expectInt(req.params.sourceId)
-    const source = await db.mysqlFirst(
+    const source = await db.knexRawFirst<Record<string, any>>(
         `
         SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace
         FROM sources AS s
         JOIN active_datasets AS d ON d.id=s.datasetId
         WHERE s.id=?`,
         [sourceId]
     )
-    source.description = JSON.parse(source.description)
-    source.variables = await db.queryMysql(
+    if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404)
+    source.variables = await db.knexRaw(
         `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`,
         [sourceId]
     )

adminSiteServer/exportGitData.ts

@@ -1,14 +1,16 @@
 import * as db from "../db/db.js"
 import { syncDatasetToGitRepo } from "./gitDataExport.js"
-import { Dataset } from "../db/model/Dataset.js"
+import { DatasetsRow, DatasetsRowTableName } from "@ourworldindata/types"
 
 const main = async () => {
-    await db.getConnection()
-    for (const dataset of await Dataset.findBy({ namespace: "owid" })) {
+    const knex = db.knexInstance()
+    const datasets = await knex<DatasetsRow>(DatasetsRowTableName).where({
+        namespace: "owid",
+    })
+    for (const dataset of datasets) {
         if (!dataset.isPrivate && !dataset.nonRedistributable)
-            await syncDatasetToGitRepo(dataset.id, { commitOnly: true })
+            await syncDatasetToGitRepo(knex, dataset.id, { commitOnly: true })
     }
-    await db.closeTypeOrmAndKnexConnections()
 }
 
 main()