Simplify authors fields in posts table, introduce better DB interacti…

…on for Posts (#3027)

This PR does two things:
- it changes the `authors` field in the `posts` table to contain simply an ordered list of authors (instead of an unordered list of json objects with `{author: string, order: number}` type)
- it splits the `PostRow` type that represents a result for of the posts table into two, one `PostRowRaw` where authors and formattingOptions are still string fields and one `PostRowEnriched` where the json fields are parsed and the typed accordingly.

adminSiteClient/PostsIndexPage.tsx

@@ -34,9 +34,9 @@ interface PostIndexMeta {
     title: string
     type: string
     status: string
-    authors: string[]
+    authors: string[] | null
     slug: string
-    updatedAtInWordpress: string
+    updatedAtInWordpress: string | null
     tags: ChartTagJoin[] | null
     gdocSuccessorId: string | undefined
     gdocSuccessorPublished: boolean
@@ -209,7 +209,7 @@ class PostRow extends React.Component<PostRowProps> {
         return (
             <tr>
                 <td>{post.title || "(no title)"}</td>
-                <td>{post.authors.join(", ")}</td>
+                <td>{post.authors?.join(", ")}</td>
                 <td>{post.type}</td>
                 <td>{post.status}</td>
                 <td>{post.slug}</td>
@@ -273,7 +273,7 @@ export class PostsIndexPage extends React.Component {
                     post.title,
                     post.slug,
                     `${post.id}`,
-                    post.authors.join(" "),
+                    post.authors?.join(" "),
                 ]
             )
             return posts.filter(filterFn)

adminSiteServer/adminRouter.tsx

@@ -178,6 +178,14 @@ adminRouter.get("/posts/compare/:postId", async (req, res) => {
         "archieml",
         "archieml_update_statistics"
     ).from(db.knexTable(Post.postsTable).where({ id: postId }))
+    if (
+        archieMlText.length === 0 ||
+        archieMlText[0].archieml === null ||
+        archieMlText[0].archieml_update_statistics === null
+    )
+        throw new Error(
+            `Could not compare posts because archieml was not present in the database for ${postId}`
+        )
     const archieMlJson = JSON.parse(archieMlText[0].archieml) as OwidGdocJSON
     const updateStatsJson = JSON.parse(
         archieMlText[0].archieml_update_statistics

adminSiteServer/apiRouter.ts

@@ -37,7 +37,7 @@ import {
     OwidGdocInterface,
     parseIntOrUndefined,
     parseToOperation,
-    PostRow,
+    PostRowEnriched,
     PostRowWithGdocPublishStatus,
     SuggestedChartRevisionStatus,
     variableAnnotationAllowedColumnNamesAndTypes,
@@ -48,7 +48,6 @@ import {
     DimensionProperty,
     TaggableType,
     ChartTagJoin,
-    sortBy,
 } from "@ourworldindata/utils"
 import {
     GrapherInterface,
@@ -2323,7 +2322,7 @@ apiRouter.get("/posts.json", async (req) => {
              posts.slug as slug,
              status,
              updated_at_in_wordpress,
-             posts.authors, -- authors is a json array of objects with name and order
+             posts.authors,
              posts_tags_aggregated.tags as tags,
              gdocSuccessorId,
              gdocSuccessor.published as isGdocSuccessorPublished,
@@ -2343,11 +2342,7 @@ apiRouter.get("/posts.json", async (req) => {
         tags: JSON.parse(row.tags),
         isGdocSuccessorPublished: !!row.isGdocSuccessorPublished,
         gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors),
-        authors: row.authors
-            ? sortBy(JSON.parse(row.authors), "order").map(
-                  (author) => author.author
-              )
-            : [],
+        authors: JSON.parse(row.authors),
     }))
 
     return { posts: rows }
@@ -2370,7 +2365,7 @@ apiRouter.get("/posts/:postId.json", async (req: Request, res: Response) => {
         .knexTable(postsTable)
         .where({ id: postId })
         .select("*")
-        .first()) as PostRow | undefined
+        .first()) as PostRowEnriched | undefined
     return camelCaseProperties({ ...post })
 })
 
@@ -2393,6 +2388,11 @@ apiRouter.post("/posts/:postId/createGdoc", async (req: Request) => {
             400
         )
     }
+    if (post.archieml === null)
+        throw new JsonError(
+            `ArchieML was not present for post with id ${postId}`,
+            500
+        )
     const tagsByPostId = await getTagsByPostId()
     const tags =
         tagsByPostId.get(postId)?.map(({ id }) => TagEntity.create({ id })) ||

baker/GrapherBaker.tsx

@@ -44,7 +44,7 @@ import {
 import * as db from "../db/db.js"
 import { glob } from "glob"
 import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js"
-import { bySlug, getPostBySlug, parsePostAuthors } from "../db/model/Post.js"
+import { getPostEnrichedBySlug } from "../db/model/Post.js"
 import { ChartTypeName, GrapherInterface } from "@ourworldindata/grapher"
 import workerpool from "workerpool"
 import ProgressBar from "progress"
@@ -298,11 +298,11 @@ export async function renderDataPageV2({
                 citation,
             }
         } else {
-            const post = await bySlug(slug)
+            const post = await getPostEnrichedBySlug(slug)
             if (post) {
-                const authors = parsePostAuthors(post.authors)
+                const authors = post.authors
                 const citation = getShortPageCitation(
-                    authors,
+                    authors ?? [],
                     post.title,
                     post.published_at
                 )
@@ -359,7 +359,7 @@ export const renderPreviewDataPageOrGrapherPage = async (
 
 const renderGrapherPage = async (grapher: GrapherInterface) => {
     const postSlug = urlToSlug(grapher.originUrl || "")
-    const post = postSlug ? await getPostBySlug(postSlug) : undefined
+    const post = postSlug ? await getPostEnrichedBySlug(postSlug) : undefined
     const relatedCharts =
         post && isWordpressDBEnabled
             ? await getRelatedCharts(post.id)

baker/postUpdatedHook.ts

@@ -4,7 +4,12 @@ import parseArgs from "minimist"
 import { BAKE_ON_CHANGE } from "../settings/serverSettings.js"
 import { DeployQueueServer } from "./DeployQueueServer.js"
 import { exit } from "../db/cleanup.js"
-import { PostRow, extractFormattingOptions } from "@ourworldindata/utils"
+import {
+    PostRowEnriched,
+    extractFormattingOptions,
+    sortBy,
+    serializePostRow,
+} from "@ourworldindata/utils"
 import * as wpdb from "../db/wpdb.js"
 import * as db from "../db/db.js"
 import {
@@ -96,7 +101,10 @@ const syncPostToGrapher = async (
     const wpPost = rows[0]
 
     const formattingOptions = extractFormattingOptions(wpPost.post_content)
-
+    const authors: string[] = sortBy(
+        JSON.parse(wpPost.authors),
+        (item: { author: string; order: number }) => item.order
+    ).map((author: { author: string; order: number }) => author.author)
     const postRow = wpPost
         ? ({
               id: wpPost.ID,
@@ -114,14 +122,14 @@ const syncPostToGrapher = async (
                   wpPost.post_modified_gmt === zeroDateString
                       ? "1970-01-01 00:00:00"
                       : wpPost.post_modified_gmt,
-              authors: wpPost.authors,
+              authors: authors,
               excerpt: wpPost.post_excerpt,
               created_at_in_wordpress:
                   wpPost.created_at === zeroDateString
                       ? "1970-01-01 00:00:00"
                       : wpPost.created_at,
               formattingOptions: formattingOptions,
-          } as PostRow)
+          } as PostRowEnriched)
         : undefined
 
     await db.knexInstance().transaction(async (transaction) => {
@@ -134,11 +142,7 @@ const syncPostToGrapher = async (
             )
             postRow.content = contentWithBlocksInlined
 
-            const rowForDb = {
-                ...postRow,
-                // TODO: it's not nice that we have to stringify this here
-                formattingOptions: JSON.stringify(postRow.formattingOptions),
-            }
+            const rowForDb = serializePostRow(postRow)
 
             if (!existsInGrapher)
                 await transaction.table(postsTable).insert(rowForDb)

baker/siteRenderers.tsx

@@ -50,12 +50,12 @@ import {
     JsonError,
     KeyInsight,
     OwidGdocInterface,
-    PostRow,
     Url,
     IndexPost,
     mergePartialGrapherConfigs,
     OwidGdocType,
     extractFormattingOptions,
+    PostRowRaw,
 } from "@ourworldindata/utils"
 import { CountryProfileSpec } from "../site/countryProfileProjects.js"
 import { formatPost } from "./formatWordpressPost.js"
@@ -435,7 +435,7 @@ export const entriesByYearPage = async (year?: number) => {
         .join("tags", { "tags.id": "post_tags.tag_id" })
         .where({ "tags.name": "Entries" })
         .select("title", "posts.slug", "published_at")) as Pick<
-        PostRow,
+        PostRowRaw,
         "title" | "slug" | "published_at"
     >[]
 

db/migrateWpPostsToArchieMl.ts

@@ -8,6 +8,7 @@ import {
     OwidGdocType,
     RelatedChart,
     EnrichedBlockAllCharts,
+    parsePostAuthors,
 } from "@ourworldindata/utils"
 import * as Post from "./model/Post.js"
 import fs from "fs"
@@ -18,7 +19,6 @@ import {
     adjustHeadingLevels,
 } from "./model/Gdoc/htmlToEnriched.js"
 import { getRelatedCharts, isPostCitable } from "./wpdb.js"
-import { parsePostAuthors } from "./model/Post.js"
 
 // slugs from all the linear entries we want to migrate from @edomt
 const entries = new Set([
@@ -80,7 +80,7 @@ const migrate = async (): Promise<void> => {
     const errors = []
     await db.getConnection()
 
-    const posts = await Post.select(
+    const rawPosts = await Post.select(
         "id",
         "slug",
         "title",
@@ -94,8 +94,14 @@ const migrate = async (): Promise<void> => {
         "featured_image"
     ).from(db.knexTable(Post.postsTable)) //.where("id", "=", "24808"))
 
-    for (const post of posts) {
+    for (const postRaw of rawPosts) {
         try {
+            const post = {
+                ...postRaw,
+                authors: postRaw.authors
+                    ? parsePostAuthors(postRaw.authors)
+                    : null,
+            }
             const isEntry = entries.has(post.slug)
             const text = post.content
             let relatedCharts: RelatedChart[] = []
@@ -109,11 +115,10 @@ const migrate = async (): Promise<void> => {
             )
             if (
                 shouldIncludeMaxAsAuthor &&
+                post.authors &&
                 !post.authors.includes("Max Roser")
             ) {
-                const authorsJson = JSON.parse(post.authors)
-                authorsJson.push({ author: "Max Roser", order: Infinity })
-                post.authors = JSON.stringify(authorsJson)
+                post.authors.push("Max Roser")
             }
 
             // We don't get the first and last nodes if they are comments.
@@ -196,9 +201,9 @@ const migrate = async (): Promise<void> => {
                     body: archieMlBodyElements,
                     toc: [],
                     title: post.title,
-                    subtitle: post.excerpt,
-                    excerpt: post.excerpt,
-                    authors: parsePostAuthors(post.authors),
+                    subtitle: post.excerpt ?? "",
+                    excerpt: post.excerpt ?? "",
+                    authors: post.authors ?? [],
                     "featured-image": post.featured_image.split("/").at(-1),
                     dateline: dateline,
                     // TODO: this discards block level elements - those might be needed?
@@ -263,7 +268,7 @@ const migrate = async (): Promise<void> => {
                 }
             }
         } catch (e) {
-            console.error("Caught an exception", post.id)
+            console.error("Caught an exception", postRaw.id)
             errors.push(e)
         }
     }

db/model/Post.ts

@@ -1,14 +1,18 @@
 import * as db from "../db.js"
 import { Knex } from "knex"
-import { PostRow, sortBy } from "@ourworldindata/utils"
+import {
+    PostRowEnriched,
+    PostRowRaw,
+    parsePostRow,
+} from "@ourworldindata/utils"
 
 export const postsTable = "posts"
 
-export const table = "posts"
-
-export const select = <K extends keyof PostRow>(
+export const select = <K extends keyof PostRowRaw>(
     ...args: K[]
-): { from: (query: Knex.QueryBuilder) => Promise<Pick<PostRow, K>[]> } => ({
+): {
+    from: (query: Knex.QueryBuilder) => Promise<Pick<PostRowRaw, K>[]>
+} => ({
     from: (query) => query.select(...args) as any,
 })
 
@@ -46,17 +50,6 @@ export const setTags = async (
             )
     })
 
-export const bySlug = async (slug: string): Promise<PostRow | undefined> =>
-    (await db.knexTable("posts").where({ slug: slug }))[0]
-
-/** The authors field in the posts table is a json column that contains an array of
-    { order: 1, authors: "Max Mustermann" } like records. This function parses the
-    string and returns a simple string array of author names in the correct order  */
-export const parsePostAuthors = (authorsJson: string): string[] => {
-    const authors = JSON.parse(authorsJson)
-    return sortBy(authors, ["order"]).map((author) => author.author)
-}
-
 export const setTagsForPost = async (
     postId: number,
     tagIds: number[]
@@ -71,7 +64,15 @@ export const setTagsForPost = async (
             )
     })
 
-export const getPostBySlug = async (
+export const getPostRawBySlug = async (
     slug: string
-): Promise<PostRow | undefined> =>
+): Promise<PostRowRaw | undefined> =>
     (await db.knexTable("posts").where({ slug: slug }))[0]
+
+export const getPostEnrichedBySlug = async (
+    slug: string
+): Promise<PostRowEnriched | undefined> => {
+    const post = await getPostRawBySlug(slug)
+    if (!post) return undefined
+    return parsePostRow(post)
+}