✨ Switch all markdown rendering to react-remark

.prettierignore

@@ -14,3 +14,4 @@ coverage
 packages/@ourworldindata/*/dist/
 dist/
 grapherData/
+.vscode/

.vscode/launch.json

@@ -10,7 +10,9 @@
             "name": "Jest Tests",
             "program": "${workspaceFolder}/node_modules/jest/bin/jest.js",
             "request": "launch",
-            "skipFiles": ["<node_internals>/**"],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
             "type": "node"
         },
         {
@@ -19,7 +21,10 @@
             "request": "launch",
             "name": "Jest Test current file",
             "program": "${workspaceFolder}/node_modules/.bin/jest",
-            "args": ["${fileBasenameNoExtension}.js", "--watch"],
+            "args": [
+                "${fileBasenameNoExtension}.js",
+                "--watch"
+            ],
             "console": "integratedTerminal"
             // "internalConsoleOptions": "neverOpen"
         },
@@ -53,23 +58,32 @@
             "name": "Run migrate WP to ArchieML",
             "program": "${workspaceFolder}/itsJustJavascript/db/migrateWpPostsToArchieMl.js",
             "request": "launch",
-            "skipFiles": ["<node_internals>/**"],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
             "type": "node"
         },
         {
             "name": "Sync WP posts to grapher",
             "program": "${workspaceFolder}/itsJustJavascript/db/syncPostsToGrapher.js",
             "request": "launch",
-            "skipFiles": ["<node_internals>/**"],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
             "type": "node"
         },
         {
             "name": "Run SVGTester",
             "program": "${workspaceFolder}/itsJustJavascript/devTools/svgTester/verify-graphs.js",
             "request": "launch",
-            "skipFiles": ["<node_internals>/**"],
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
             "type": "node",
-            "args": ["-g", "367"]
+            "args": [
+                "-g",
+                "367"
+            ]
         },
         {
             "name": "Launch admin server",

adminSiteClient/ChartEditorPage.tsx

@@ -10,7 +10,6 @@ import {
     IReactionDisposer,
 } from "mobx"
 import { Prompt, Redirect } from "react-router-dom"
-import { extractDetailsFromSyntax } from "@ourworldindata/components"
 import {
     Bounds,
     capitalize,
@@ -19,6 +18,7 @@ import {
     get,
     set,
     groupBy,
+    extractDetailsFromSyntax,
 } from "@ourworldindata/utils"
 import { Grapher, Topic, GrapherInterface } from "@ourworldindata/grapher"
 import { Admin } from "./Admin.js"

baker/SiteBaker.tsx

@@ -45,8 +45,9 @@ import {
     OwidGdocPublished,
     clone,
     LinkedChart,
+    extractDetailsFromSyntax,
 } from "@ourworldindata/utils"
-import { extractDetailsFromSyntax } from "@ourworldindata/components"
+
 import { execWrapper } from "../db/execWrapper.js"
 import { countryProfileSpecs } from "../site/countryProfileProjects.js"
 import {

package.json

@@ -82,6 +82,7 @@
         "@types/jsonwebtoken": "^9.0.0",
         "@types/lodash": "^4.14.185",
         "@types/md5": "^2.3.2",
+        "@types/mdast": "^3.0",
         "@types/minimist": "^1.2.2",
         "@types/mousetrap": "^1.6.9",
         "@types/mysql": "^2.15.21",
@@ -154,6 +155,7 @@
         "lodash": "^4.17.20",
         "mathjax-full": "^3.1.0",
         "md5": "^2.3.0",
+        "mdast-util-from-markdown": "^0.8.0",
         "minimist": "^1.2.6",
         "mobx": "^5.15.7",
         "mobx-formatters": "^1.0.2",

packages/@ourworldindata/components/src/GdocsUtils.ts

@@ -1,18 +1,11 @@
 import {
     spansToUnformattedPlainText,
     gdocUrlRegex,
-    EnrichedBlockText,
     OwidGdocLinkJSON,
     Span,
     Url,
 } from "@ourworldindata/utils"
 import urlSlug from "url-slug"
-import {
-    EveryMarkdownNode,
-    MarkdownRoot,
-    mdParser,
-} from "./MarkdownTextWrap/parser.js"
-import { P, match } from "ts-pattern"
 
 export function getLinkType(urlString: string): OwidGdocLinkJSON["linkType"] {
     const url = Url.fromURL(urlString)
@@ -50,138 +43,3 @@ export function getUrlTarget(urlString: string): string {
 export function convertHeadingTextToId(headingText: Span[]): string {
     return urlSlug(spansToUnformattedPlainText(headingText))
 }
-
-const convertMarkdownNodeToSpan = (node: EveryMarkdownNode): Span[] => {
-    return match(node)
-        .with(
-            {
-                type: "text",
-            },
-            (n) => [
-                {
-                    spanType: "span-simple-text" as const,
-                    text: n.value,
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "textSegments",
-            },
-            (n) => n.children.flatMap(convertMarkdownNodeToSpan) as Span[]
-        )
-        .with(
-            {
-                type: "newline",
-            },
-            () => [
-                {
-                    spanType: "span-simple-text" as const,
-                    text: "\n",
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "whitespace",
-            },
-            () => [
-                {
-                    spanType: "span-simple-text" as const,
-                    text: " ",
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "detailOnDemand",
-            },
-            (n) => [
-                {
-                    spanType: "span-dod" as const,
-                    id: n.term,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "markdownLink",
-            },
-            (n) => [
-                {
-                    spanType: "span-link" as const,
-                    url: n.href,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "plainUrl",
-            },
-            (n) => [
-                {
-                    spanType: "span-link" as const,
-                    url: n.href,
-                    children: [
-                        {
-                            spanType: "span-simple-text" as const,
-                            text: n.href,
-                        },
-                    ],
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: P.union("italic", "plainItalic", "italicWithoutBold"),
-            },
-            (n) => [
-                {
-                    spanType: "span-italic" as const,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: P.union("bold", "plainBold", "boldWithoutItalic"),
-            },
-            (n) => [
-                {
-                    spanType: "span-bold" as const,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .exhaustive()
-    //.otherwise(() => ({ spanType: "span-simple-text" as const, text: "" }))
-}
-
-const convertMarkdownNodesToSpans = (nodes: MarkdownRoot): Span[] =>
-    nodes.children.flatMap(convertMarkdownNodeToSpan)
-
-export const markdownToEnrichedTextBlock = (
-    markdown: string
-): EnrichedBlockText => {
-    const parsedMarkdown = mdParser.markdown.parse(markdown)
-    if (parsedMarkdown.status) {
-        const spans = convertMarkdownNodesToSpans(parsedMarkdown.value)
-        return {
-            type: "text",
-            value: spans,
-            parseErrors: [],
-        }
-    } else
-        return {
-            type: "text",
-            value: [],
-            parseErrors: [
-                {
-                    message: `Failed to parse markdown - expected ${parsedMarkdown.expected} at ${parsedMarkdown.index}`,
-                    isWarning: false,
-                },
-            ],
-        }
-}

packages/@ourworldindata/components/src/MarkdownTextWrap/MarkdownTextWrap.stories.tsx

@@ -3,10 +3,9 @@ import { action, computed, observable } from "mobx"
 import { observer } from "mobx-react"
 import {
     MarkdownTextWrap,
-    parsimmonToTextTokens,
     IRToken,
+    convertMarkdownToIRTokens,
 } from "./MarkdownTextWrap"
-import { mdParser } from "./parser.js"
 import { TextWrap } from "../TextWrap/TextWrap.js"
 
 export default {
@@ -54,11 +53,8 @@ _THE END_
     }
 
     @computed get tokens(): IRToken[] {
-        const result = mdParser.markdown.parse(this.markdown)
-        if (result.status) {
-            return parsimmonToTextTokens(result.value.children)
-        }
-        return []
+        const result = convertMarkdownToIRTokens(this.markdown)
+        return result
     }
 
     render(): JSX.Element {

packages/@ourworldindata/components/src/MarkdownTextWrap/MarkdownTextWrap.test.ts

@@ -83,7 +83,7 @@ describe("MarkdownTextWrap", () => {
 
     it("should split on newline", () => {
         const element = new MarkdownTextWrap({
-            text: "_test\n**\nnewline\n**_test",
+            text: "_test\n**\nnewlineyarn \n**_test",
             fontSize: 10,
             lineHeight: 1,
         })