✨ replace our svg markdown parser with mdast parser

owid · Oct 30, 2023 · b1f2f96 · b1f2f96
1 parent 2baf6e7
commit b1f2f96
Show file tree

Hide file tree

Showing 8 changed files with 645 additions and 1,847 deletions.
diff --git a/package.json b/package.json
@@ -82,6 +82,7 @@
         "@types/jsonwebtoken": "^9.0.0",
         "@types/lodash": "^4.14.185",
         "@types/md5": "^2.3.2",
+        "@types/mdast": "^4.0.2",
         "@types/minimist": "^1.2.2",
         "@types/mousetrap": "^1.6.9",
         "@types/mysql": "^2.15.21",
@@ -154,6 +155,7 @@
         "lodash": "^4.17.20",
         "mathjax-full": "^3.1.0",
         "md5": "^2.3.0",
+        "mdast-util-from-markdown": "^2.0.0",
         "minimist": "^1.2.6",
         "mobx": "^5.15.7",
         "mobx-formatters": "^1.0.2",

diff --git a/packages/@ourworldindata/components/src/GdocsUtils.ts b/packages/@ourworldindata/components/src/GdocsUtils.ts
@@ -1,18 +1,11 @@
 import {
     spansToUnformattedPlainText,
     gdocUrlRegex,
-    EnrichedBlockText,
     OwidGdocLinkJSON,
     Span,
     Url,
 } from "@ourworldindata/utils"
 import urlSlug from "url-slug"
-import {
-    EveryMarkdownNode,
-    MarkdownRoot,
-    mdParser,
-} from "./MarkdownTextWrap/parser.js"
-import { P, match } from "ts-pattern"
 
 export function getLinkType(urlString: string): OwidGdocLinkJSON["linkType"] {
     const url = Url.fromURL(urlString)
@@ -50,149 +43,3 @@ export function getUrlTarget(urlString: string): string {
 export function convertHeadingTextToId(headingText: Span[]): string {
     return urlSlug(spansToUnformattedPlainText(headingText))
 }
-
-const convertMarkdownNodeToSpan = (node: EveryMarkdownNode): Span[] => {
-    return match(node)
-        .with(
-            {
-                type: "text",
-            },
-            (n) => [
-                {
-                    spanType: "span-simple-text" as const,
-                    text: n.value,
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "textSegments",
-            },
-            (n) => n.children.flatMap(convertMarkdownNodeToSpan) as Span[]
-        )
-        .with(
-            {
-                type: "newline",
-            },
-            () => [
-                {
-                    spanType: "span-simple-text" as const,
-                    text: "\n",
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "whitespace",
-            },
-            () => [
-                {
-                    spanType: "span-simple-text" as const,
-                    text: " ",
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "detailOnDemand",
-            },
-            (n) => [
-                {
-                    spanType: "span-dod" as const,
-                    id: n.term,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "markdownLink",
-            },
-            (n) => [
-                {
-                    spanType: "span-link" as const,
-                    url: n.href,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "plainUrl",
-            },
-            (n) => [
-                {
-                    spanType: "span-link" as const,
-                    url: n.href,
-                    children: [
-                        {
-                            spanType: "span-simple-text" as const,
-                            text: n.href,
-                        },
-                    ],
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: "bold",
-            },
-            (n) => [
-                {
-                    spanType: "span-bold" as const,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: P.union("italic", "plainItalic", "italicWithoutBold"),
-            },
-            (n) => [
-                {
-                    spanType: "span-italic" as const,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .with(
-            {
-                type: P.union("bold", "plainBold", "boldWithoutItalic"),
-            },
-            (n) => [
-                {
-                    spanType: "span-bold" as const,
-                    children: n.children.flatMap(convertMarkdownNodeToSpan),
-                } as Span,
-            ]
-        )
-        .exhaustive()
-    //.otherwise(() => ({ spanType: "span-simple-text" as const, text: "" }))
-}
-
-const convertMarkdownNodesToSpans = (nodes: MarkdownRoot): Span[] =>
-    nodes.children.flatMap(convertMarkdownNodeToSpan)
-
-export const markdownToEnrichedTextBlock = (
-    markdown: string
-): EnrichedBlockText => {
-    const parsedMarkdown = mdParser.markdown.parse(markdown)
-    if (parsedMarkdown.status) {
-        const spans = convertMarkdownNodesToSpans(parsedMarkdown.value)
-        return {
-            type: "text",
-            value: spans,
-            parseErrors: [],
-        }
-    } else
-        return {
-            type: "text",
-            value: [],
-            parseErrors: [
-                {
-                    message: `Failed to parse markdown - expected ${parsedMarkdown.expected} at ${parsedMarkdown.index}`,
-                    isWarning: false,
-                },
-            ],
-        }
-}
diff --git a/packages/@ourworldindata/components/src/MarkdownTextWrap/MarkdownTextWrap.stories.tsx b/packages/@ourworldindata/components/src/MarkdownTextWrap/MarkdownTextWrap.stories.tsx
@@ -3,10 +3,9 @@ import { action, computed, observable } from "mobx"
 import { observer } from "mobx-react"
 import {
     MarkdownTextWrap,
-    parsimmonToTextTokens,
     IRToken,
+    convertMarkdownToIRTokens,
 } from "./MarkdownTextWrap"
-import { mdParser } from "./parser.js"
 import { TextWrap } from "../TextWrap/TextWrap.js"
 
 export default {
@@ -54,11 +53,8 @@ _THE END_
     }
 
     @computed get tokens(): IRToken[] {
-        const result = mdParser.markdown.parse(this.markdown)
-        if (result.status) {
-            return parsimmonToTextTokens(result.value.children)
-        }
-        return []
+        const result = convertMarkdownToIRTokens(this.markdown)
+        return result
     }
 
     render(): JSX.Element {