Markdown: special handling of DoD links (#2873)

This PR adds a custom transform plugin to convert dod style links (those pointing to #dod-term). It walks the html AST (not the markdown one) and adds the attributes we need and removes the href target.

This plugin only applies to SimpleMarkdown, as here the html AST is used. In MarkdownTextWrap we don't go that far in the pipeline and just fetch the markdown AST - for that case we already have a special handling for dod style links in place when we pattern match from markdown AST to IRTokens

package.json

@@ -77,6 +77,7 @@
         "@types/express-rate-limit": "^5.1.0",
         "@types/fs-extra": "^11.0.1",
         "@types/fuzzyset": "^1.0.2",
+        "@types/hast": "2",
         "@types/html-to-text": "^8.1.1",
         "@types/js-cookie": "^3.0.2",
         "@types/jsonwebtoken": "^9.0.0",
@@ -213,6 +214,8 @@
         "typeorm": "^0.3.14",
         "typescript": "~5.2.2",
         "unidecode": "^0.1.8",
+        "unified": "^9.0",
+        "unist-util-visit": "^2.0.0",
         "url-join": "^4.0.0",
         "url-parse": "^1.5.10",
         "url-slug": "^3.0.2",

packages/@ourworldindata/components/src/SimpleMarkdownText.tsx

@@ -2,17 +2,50 @@ import React from "react"
 import { computed } from "mobx"
 import { Remark } from "react-remark"
 import { remarkPlainLinks } from "./markdown/remarkPlainLinks.js"
-
+import visit from "unist-util-visit"
 type SimpleMarkdownTextProps = {
     text: string
 }
 
+function transformDodLinks() {
+    // TODO: try to type this properly when we have upgraded to a recent version with ESM. This is
+    // going to be a HAST tree and with that typing it should be able to make the access in there typesafe.
+    return function (tree: any) {
+        visit(tree, "element", function (node: any) {
+            if (node.tagName === "a")
+                if (
+                    node.properties &&
+                    typeof node.properties.href === "string" &&
+                    node.properties.href.startsWith("#dod:")
+                ) {
+                    // use a regex to split the #dod: part from the term of the href. Use a named capture group
+                    // to capture the term
+                    const match = node.properties.href.match(/#dod:(?<term>.+)/)
+                    if (match) {
+                        node.properties.class = "dod-span"
+                        node.properties["data-id"] = match.groups?.term
+                        node.properties["aria-expanded"] = "false"
+                        delete node.properties.href
+                    }
+                    //node.children.push(
+                }
+        })
+    }
+}
+
 export class SimpleMarkdownText extends React.Component<SimpleMarkdownTextProps> {
     @computed get text(): string {
         return this.props.text
     }
 
     render(): JSX.Element | null {
-        return <Remark remarkPlugins={[remarkPlainLinks]}>{this.text}</Remark>
+        return (
+            <Remark
+                rehypePlugins={[transformDodLinks]}
+                remarkPlugins={[remarkPlainLinks]}
+            >
+                {this.text}
+            </Remark>
+        )
     }
 }

packages/@ourworldindata/utils/src/GdocsConstants.ts

@@ -23,12 +23,3 @@ export const gdocUrlRegex =
     /https:\/\/docs\.google\.com\/document(?:\/u\/\d)?\/d\/([-\w]+)\/?(edit)?#?/
 
 export const gdocIdRegex = /^[0-9A-Za-z\-_]{44}$/
-
-/** Works for:
- * #dod:text
- * #dod:text-hyphenated
- * #dod:text_underscored
- * #dod:text_underscored-and-hyphenated
- * Duplicated in parser.ts
- */
-export const detailOnDemandRegex = /#dod:([\w\-_]+)/

yarn.lock

@@ -4278,6 +4278,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/hast@npm:2":
+  version: 2.3.7
+  resolution: "@types/hast@npm:2.3.7"
+  dependencies:
+    "@types/unist": "npm:^2"
+  checksum: 3e63332825ed88117e7f355ba0cfd35367f1d951a1c381333b56188f7645947c3bbbe96abb4c8239324ba1317fd241d5afdb42e104a3654a424327340d49052c
+  languageName: node
+  linkType: hard
+
 "@types/history@npm:*":
   version: 4.7.2
   resolution: "@types/history@npm:4.7.2"
@@ -10425,6 +10434,7 @@ __metadata:
     "@types/fs-extra": "npm:^11.0.1"
     "@types/fuzzyset": "npm:^1.0.2"
     "@types/geojson": "npm:^7946.0.10"
+    "@types/hast": "npm:2"
     "@types/html-to-text": "npm:^8.1.1"
     "@types/indefinite": "npm:^2.3.2"
     "@types/jest": "npm:^29.5.5"
@@ -10582,6 +10592,8 @@ __metadata:
     typeorm: "npm:^0.3.14"
     typescript: "npm:~5.2.2"
     unidecode: "npm:^0.1.8"
+    unified: "npm:^9.0"
+    unist-util-visit: "npm:^2.0.0"
     url-join: "npm:^4.0.0"
     url-parse: "npm:^1.5.10"
     url-slug: "npm:^3.0.2"
@@ -19458,7 +19470,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"unified@npm:^9.0.0":
+"unified@npm:^9.0, unified@npm:^9.0.0":
   version: 9.2.2
   resolution: "unified@npm:9.2.2"
   dependencies: