✨ add formatting options column to wp posts table

baker/formatWordpressPost.tsx

@@ -16,6 +16,7 @@ import {
     JsonError,
     TocHeading,
     WP_BlockType,
+    parseKeyValueArgs,
 } from "@ourworldindata/utils"
 import { Footnote } from "../site/Footnote.js"
 import { LoadingIndicator } from "@ourworldindata/grapher"
@@ -28,7 +29,6 @@ import {
     extractDataValuesConfiguration,
     formatDataValue,
     formatImages,
-    parseKeyValueArgs,
 } from "./formatting.js"
 import { mathjax } from "mathjax-full/js/mathjax.js"
 import { TeX } from "mathjax-full/js/input/tex.js"

baker/formatting.test.ts

@@ -1,10 +1,11 @@
 #! /usr/bin/env jest
 
+import { extractDataValuesConfiguration } from "./formatting.js"
+
 import {
-    extractDataValuesConfiguration,
     parseFormattingOptions,
     parseKeyValueArgs,
-} from "./formatting.js"
+} from "@ourworldindata/utils"
 
 it("parses formatting options", () => {
     const formattingOptions =

baker/formatting.tsx

@@ -3,15 +3,14 @@ import {
     DataValueConfiguration,
     DataValueQueryArgs,
     FormattedPost,
-    FormattingOptions,
-    KeyValueProps,
     OwidVariableId,
     Country,
     OwidVariableDataMetadataDimensions,
     OwidChartDimensionInterface,
     OwidVariableDisplayConfigInterface,
     capitalize,
     Url,
+    parseKeyValueArgs,
 } from "@ourworldindata/utils"
 import { countryProfileDefaultCountryPlaceholder } from "../site/countryProfileProjects.js"
 import { BAKED_BASE_URL } from "../settings/serverSettings.js"
@@ -22,21 +21,6 @@ import path from "path"
 
 export const DEEP_LINK_CLASS = "deep-link"
 
-export const extractFormattingOptions = (html: string): FormattingOptions => {
-    const formattingOptionsMatch = html.match(
-        /<!--\s*formatting-options\s+(.*)\s*-->/
-    )
-    return formattingOptionsMatch
-        ? parseFormattingOptions(formattingOptionsMatch[1])
-        : {}
-}
-
-// Converts "toc:false raw somekey:somevalue" to { toc: false, raw: true, somekey: "somevalue" }
-// If only the key is specified, the value is assumed to be true (e.g. "raw" above)
-export const parseFormattingOptions = (text: string): FormattingOptions => {
-    return parseKeyValueArgs(text)
-}
-
 export const dataValueRegex = new RegExp(
     `{{\\s*${DATA_VALUE}\\s*(.+?)\\s*}}`,
     "g"
@@ -74,29 +58,6 @@ export const parseDataValueArgs = (
     )
 }
 
-export const parseKeyValueArgs = (text: string): KeyValueProps => {
-    const options: { [key: string]: string | boolean } = {}
-    text.split(/\s+/)
-        // filter out empty strings
-        .filter((s) => s && s.length > 0)
-        .forEach((option: string) => {
-            // using regex instead of split(":") to handle ":" in value
-            // e.g. {{LastUpdated timestampUrl:https://...}}
-            const optionRegex = /([^:]+):?(.*)/
-            const [, name, value] = option.match(optionRegex) as [
-                any,
-                string,
-                string,
-            ]
-            let parsedValue
-            if (value === "" || value === "true") parsedValue = true
-            else if (value === "false") parsedValue = false
-            else parsedValue = value
-            options[name] = parsedValue
-        })
-    return options
-}
-
 export const formatDataValue = (
     value: number,
     variableId: OwidVariableId,

baker/pageOverrides.test.ts

@@ -1,5 +1,8 @@
-import { FullPost, WP_PostType } from "@ourworldindata/utils"
-import { extractFormattingOptions } from "./formatting.js"
+import {
+    FullPost,
+    WP_PostType,
+    extractFormattingOptions,
+} from "@ourworldindata/utils"
 import * as pageOverrides from "./pageOverrides.js"
 
 import { jest } from "@jest/globals"

baker/siteRenderers.tsx

@@ -16,11 +16,7 @@ import OwidGdocPage from "../site/gdocs/OwidGdocPage.js"
 import React from "react"
 import ReactDOMServer from "react-dom/server.js"
 import * as lodash from "lodash"
-import {
-    extractFormattingOptions,
-    formatCountryProfile,
-    isCanonicalInternalUrl,
-} from "./formatting.js"
+import { formatCountryProfile, isCanonicalInternalUrl } from "./formatting.js"
 import {
     bakeGrapherUrls,
     getGrapherExportsByUrl,
@@ -59,6 +55,7 @@ import {
     IndexPost,
     mergePartialGrapherConfigs,
     OwidGdocType,
+    extractFormattingOptions,
 } from "@ourworldindata/utils"
 import { CountryProfileSpec } from "../site/countryProfileProjects.js"
 import { formatPost } from "./formatWordpressPost.js"

db/migration/1702473994107-PostsAddFormattingOptions.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class PostsAddFormattingOptions1702473994107
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        // add json column formattingOptions to posts mysql table
+        await queryRunner.query(
+            "ALTER TABLE posts ADD COLUMN formattingOptions JSON"
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // remove json column formattingOptions from posts mysql table
+        await queryRunner.query(
+            "ALTER TABLE posts DROP COLUMN formattingOptions"
+        )
+    }
+}

db/syncPostsToGrapher.ts

@@ -3,7 +3,13 @@
 
 import * as wpdb from "./wpdb.js"
 import * as db from "./db.js"
-import { excludeNullish, groupBy, keyBy, PostRow } from "@ourworldindata/utils"
+import {
+    excludeNullish,
+    extractFormattingOptions,
+    groupBy,
+    keyBy,
+    PostRow,
+} from "@ourworldindata/utils"
 import { postsTable, select } from "./model/Post.js"
 import { PostLink } from "./model/PostLink.js"
 
@@ -278,6 +284,7 @@ const syncPostsToGrapher = async (): Promise<void> => {
 
     const toInsert = rows.map((post: any) => {
         const content = post.post_content as string
+        const formattingOptions = extractFormattingOptions(content)
 
         return {
             id: post.ID,
@@ -299,6 +306,7 @@ const syncPostsToGrapher = async (): Promise<void> => {
             created_at_in_wordpress:
                 post.created_at === zeroDateString ? null : post.created_at,
             featured_image: post.featured_image || "",
+            formattingOptions: formattingOptions,
         }
     }) as PostRow[]
     const postLinks = await PostLink.find()
@@ -325,9 +333,17 @@ const syncPostsToGrapher = async (): Promise<void> => {
             await t.whereIn("id", toDelete).delete().from(postsTable)
 
         for (const row of toInsert) {
+            const rowForDb = {
+                ...row,
+                // TODO: it's not nice that we have to stringify this here
+                formattingOptions: JSON.stringify(row.formattingOptions),
+            }
             if (doesExistInGrapher[row.id])
-                await t.update(row).where("id", "=", row.id).into(postsTable)
-            else await t.insert(row).into(postsTable)
+                await t
+                    .update(rowForDb)
+                    .where("id", "=", rowForDb.id)
+                    .into(postsTable)
+            else await t.insert(rowForDb).into(postsTable)
         }
     })
 

packages/@ourworldindata/utils/src/index.ts

@@ -641,3 +641,9 @@ export {
     gdocUrlRegex,
     gdocIdRegex,
 } from "./GdocsConstants.js"
+
+export {
+    extractFormattingOptions,
+    parseFormattingOptions,
+    parseKeyValueArgs,
+} from "./wordpressUtils.js"

packages/@ourworldindata/utils/src/owidTypes.ts

@@ -183,6 +183,7 @@ export interface PostRow {
     excerpt: string
     created_at_in_wordpress: Date | null
     featured_image: string
+    formattingOptions: FormattingOptions
 }
 
 export interface PostRowWithGdocPublishStatus extends PostRow {

packages/@ourworldindata/utils/src/wordpressUtils.ts

@@ -0,0 +1,42 @@
+import { FormattingOptions, KeyValueProps } from "./owidTypes.js"
+
+export const extractFormattingOptions = (html: string): FormattingOptions => {
+    const formattingOptionsMatch = html.match(
+        /<!--\s*formatting-options(.*)-->/
+    )
+    const innerFormattingOptions = formattingOptionsMatch
+        ? formattingOptionsMatch[1].trim()
+        : ""
+    return formattingOptionsMatch
+        ? parseFormattingOptions(innerFormattingOptions)
+        : {}
+}
+
+// Converts "toc:false raw somekey:somevalue" to { toc: false, raw: true, somekey: "somevalue" }
+// If only the key is specified, the value is assumed to be true (e.g. "raw" above)
+export const parseFormattingOptions = (text: string): FormattingOptions => {
+    return parseKeyValueArgs(text)
+}
+
+export const parseKeyValueArgs = (text: string): KeyValueProps => {
+    const options: { [key: string]: string | boolean } = {}
+    text.split(/\s+/)
+        // filter out empty strings
+        .filter((s) => s && s.length > 0)
+        .forEach((option: string) => {
+            // using regex instead of split(":") to handle ":" in value
+            // e.g. {{LastUpdated timestampUrl:https://...}}
+            const optionRegex = /([^:]+):?(.*)/
+            const [, name, value] = option.match(optionRegex) as [
+                any,
+                string,
+                string,
+            ]
+            let parsedValue
+            if (value === "" || value === "true") parsedValue = true
+            else if (value === "false") parsedValue = false
+            else parsedValue = value
+            options[name] = parsedValue
+        })
+    return options
+}