diff --git a/db/migrateWpPostsToArchieMl.ts b/db/migrateWpPostsToArchieMl.ts index c25cc1fed7e..37f749b0238 100644 --- a/db/migrateWpPostsToArchieMl.ts +++ b/db/migrateWpPostsToArchieMl.ts @@ -17,6 +17,136 @@ import { adjustHeadingLevels, } from "./model/Gdoc/htmlToEnriched.js" +// Hard-coded slugs to avoid WP dependency +const entries = new Set([ + "population", + "population-change", + "age-structure", + "gender-ratio", + "life-and-death", + "life-expectancy", + "child-mortality", + "fertility-rate", + "distribution-of-the-world-population", + "urbanization", + "health", + "health-risks", + "air-pollution", + "outdoor-air-pollution", + "indoor-air-pollution", + "obesity", + "smoking", + "alcohol-consumption", + "infectious-diseases", + "monkeypox", + "coronavirus", + "hiv-aids", + "malaria", + "eradication-of-diseases", + "smallpox", + "polio", + "pneumonia", + "tetanus", + "health-institutions-and-interventions", + "financing-healthcare", + "vaccination", + "life-death-health", + "maternal-mortality", + "health-meta", + "causes-of-death", + "burden-of-disease", + "cancer", + "environment", + "nuclear-energy", + "energy-access", + "renewable-energy", + "fossil-fuels", + "waste", + "plastic-pollution", + "air-and-climate", + "co2-and-greenhouse-gas-emissions", + "climate-change", + "water", + "clean-water-sanitation", + "water-access", + "sanitation", + "water-use-stress", + "land-and-ecosystems", + "forests-and-deforestation", + "land-use", + "natural-disasters", + "food", + "nutrition", + "famines", + "food-supply", + "human-height", + "micronutrient-deficiency", + "diet-compositions", + "food-production", + "meat-production", + "agricultural-inputs", + "employment-in-agriculture", + "growth-inequality", + "public-sector", + "government-spending", + "taxation", + "military-personnel-spending", + "financing-education", + "poverty-and-prosperity", + "economic-inequality", + "poverty", + "economic-growth", + "economic-inequality-by-gender", + "labor", + "child-labor", + "working-hours", + "female-labor-supply", + "corruption", + "trade-migration", + "trade-and-globalization", + "tourism", + "education", + "educational-outcomes", + "global-education", + "literacy", + "pre-primary-education", + "primary-and-secondary-education", + "quality-of-education", + "tertiary-education", + "inputs-to-education", + "teachers-and-professors", + "media-education", + "technology", + "space-exploration-satellites", + "transport", + "work-life", + "culture", + "trust", + "housing", + "homelessness", + "time-use", + "relationships", + "marriages-and-divorces", + "social-connections-and-loneliness", + "happiness-wellbeing", + "happiness-and-life-satisfaction", + "human-development-index", + "politics", + "human-rights", + "lgbt-rights", + "women-rights", + "democracy", + "violence-rights", + "war-peace", + "biological-and-chemical-weapons", + "war-and-peace", + "terrorism", + "nuclear-weapons", + "violence", + "violence-against-rights-for-children", + "homicides", +]) + const migrate = async (): Promise => { const writeToFile = false const errors = [] @@ -33,7 +163,7 @@ const migrate = async (): Promise => { "excerpt", "created_at_in_wordpress", "updated_at" - ).from(db.knexTable(Post.postsTable)) //.where("id", "=", "22821")) + ).from(db.knexTable(Post.postsTable)) //.where("id", "=", "38189") for (const post of posts) { try { @@ -83,6 +213,7 @@ const migrate = async (): Promise => { slug: post.slug, content: { body: archieMlBodyElements, + toc: [], title: post.title, subtitle: post.excerpt, excerpt: post.excerpt, @@ -92,7 +223,9 @@ const migrate = async (): Promise => { dateline: dateline, // TODO: this discards block level elements - those might be needed? refs: undefined, - type: OwidGdocType.Article, + type: entries.has(post.slug) + ? OwidGdocType.TopicPage + : OwidGdocType.Article, }, published: false, createdAt: diff --git a/db/model/Gdoc/Gdoc.ts b/db/model/Gdoc/Gdoc.ts index 78af8035191..92c075c1ae0 100644 --- a/db/model/Gdoc/Gdoc.ts +++ b/db/model/Gdoc/Gdoc.ts @@ -605,6 +605,7 @@ export class Gdoc extends BaseEntity implements OwidGdocInterface { "aside", "callout", "expandable-paragraph", + "entry-summary", "gray-section", "heading", "horizontal-rule", diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index a80df713323..3d6e4ea9d07 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -32,6 +32,7 @@ import { EnrichedBlockResearchAndWritingLink, RawBlockResearchAndWritingLink, RawBlockAlign, + RawBlockEntrySummary, } from "@ourworldindata/utils" import { spanToHtmlString } from "./gdocUtils.js" import { match, P } from "ts-pattern" @@ -372,5 +373,13 @@ export function enrichedBlockToRawBlock( }, } }) + .with({ type: "entry-summary" }, (b): RawBlockEntrySummary => { + return { + type: b.type, + value: { + items: b.items, + }, + } + }) .exhaustive() } diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index d852460e6c3..b0de1a27493 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -418,4 +418,9 @@ export const enrichedBlockExamples: Record< content: [enrichedBlockText], parseErrors: [], }, + "entry-summary": { + type: "entry-summary", + items: [{ text: "Hello", slug: "#link-to-something" }], + parseErrors: [], + }, } diff --git a/db/model/Gdoc/htmlToEnriched.ts b/db/model/Gdoc/htmlToEnriched.ts index 6670031cc5d..3d5960ec890 100644 --- a/db/model/Gdoc/htmlToEnriched.ts +++ b/db/model/Gdoc/htmlToEnriched.ts @@ -26,9 +26,22 @@ import { EnrichedBlockProminentLink, BlockImageSize, detailOnDemandRegex, + EnrichedBlockEntrySummary, + EnrichedBlockEntrySummaryItem, + spansToUnformattedPlainText, + checkNodeIsSpanLink, + Url, + EnrichedBlockCallout, } from "@ourworldindata/utils" import { match, P } from "ts-pattern" -import { compact, flatten, isPlainObject, partition } from "lodash" +import { + compact, + flatten, + get, + isArray, + isPlainObject, + partition, +} from "lodash" import cheerio from "cheerio" import { spansToSimpleString } from "./gdocUtils.js" @@ -223,6 +236,10 @@ type ErrorNames = | "unhandled html tag found" | "prominent link missing title" | "prominent link missing url" + | "summary item isn't text" + | "summary item doesn't have link" + | "summary item has DataValue" + | "unknown content type inside summary block" interface BlockParseError { name: ErrorNames @@ -337,11 +354,12 @@ function isArchieMlComponent( export function convertAllWpComponentsToArchieMLBlocks( blocksOrComponents: ArchieBlockOrWpComponent[] ): OwidEnrichedGdocBlock[] { - return blocksOrComponents.flatMap((blockOrComponent) => { - if (isArchieMlComponent(blockOrComponent)) return [blockOrComponent] + return blocksOrComponents.flatMap((blockOrComponentOrToc) => { + if (isArchieMlComponent(blockOrComponentOrToc)) + return [blockOrComponentOrToc] else { return convertAllWpComponentsToArchieMLBlocks( - blockOrComponent.childrenResults + blockOrComponentOrToc.childrenResults ) } }) @@ -596,6 +614,92 @@ function finishWpComponent( } } else return { ...content, errors } }) + .with("owid/summary", () => { + // Summaries can either be lists of anchor links, or paragraphs of text + // If it's a paragraph of text, we want to turn it into a callout block + // If it's a list of anchor links, we want to turn it into a toc block + const contentIsAllText = + content.content.find( + (block) => "type" in block && block.type !== "text" + ) === undefined + + if (contentIsAllText) { + const callout: EnrichedBlockCallout = { + type: "callout", + title: "Summary", + text: content.content as EnrichedBlockText[], + parseErrors: [], + } + return { errors: [], content: [callout] } + } + + const contentIsList = + content.content.length === 1 && + "type" in content.content[0] && + content.content[0].type === "list" + if (contentIsList) { + const listItems = get(content, ["content", 0, "items"]) + const items: EnrichedBlockEntrySummaryItem[] = [] + const errors = content.errors + if (isArray(listItems)) { + listItems.forEach((item) => { + if (item.type === "text") { + const value = item.value[0] + if (checkNodeIsSpanLink(value)) { + const { hash } = Url.fromURL(value.url) + const text = spansToUnformattedPlainText( + value.children + ) + if (text.includes("DataValue")) { + errors.push({ + name: "summary item has DataValue", + details: text, + }) + } + items.push({ + // Remove "#" from the beginning of the slug + slug: hash.slice(1), + text: text, + }) + } else { + errors.push({ + name: "summary item doesn't have link", + details: value + ? `spanType is ${value.spanType}` + : "No item", + }) + } + } else { + errors.push({ + name: "summary item isn't text", + details: `item is type: ${item.type}`, + }) + } + }) + } + const toc: EnrichedBlockEntrySummary = { + type: "entry-summary", + items, + parseErrors: [], + } + return { errors: [], content: [toc] } + } + + const error: BlockParseError = { + name: "unknown content type inside summary block", + details: + "Unknown summary content: " + + content.content + .map((block) => + "type" in block ? block.type : block.tagName + ) + .join(", "), + } + return { + errors: [error], + content: [], + } + }) .otherwise(() => { return { errors: [ diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index 8be3b5aac28..ff937053c39 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -30,6 +30,7 @@ import { RawBlockTopicPageIntro, RawBlockExpandableParagraph, RawBlockAlign, + RawBlockEntrySummary, } from "@ourworldindata/utils" import { match } from "ts-pattern" @@ -522,6 +523,21 @@ function* rawBlockAlignToArchieMLString( yield "{}" } +function* rawBlockEntrySummaryToArchieMLString( + block: RawBlockEntrySummary +): Generator { + yield "{.entry-summary}" + yield "[.items]" + if (block.value.items) { + for (const item of block.value.items) { + yield* propertyToArchieMLString("text", item) + yield* propertyToArchieMLString("slug", item) + } + } + yield "[]" + yield "{}" +} + export function* OwidRawGdocBlockToArchieMLStringGenerator( block: OwidRawGdocBlock ): Generator { @@ -581,6 +597,7 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( rawResearchAndWritingToArchieMLString ) .with({ type: "align" }, rawBlockAlignToArchieMLString) + .with({ type: "entry-summary" }, rawBlockEntrySummaryToArchieMLString) .exhaustive() yield* content } diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index 41f57bf11e0..1c9c30383a2 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -95,6 +95,9 @@ import { RawBlockAlign, FaqDictionary, EnrichedFaq, + RawBlockEntrySummary, + EnrichedBlockEntrySummary, + EnrichedBlockEntrySummaryItem, } from "@ourworldindata/utils" import { extractUrl, @@ -173,6 +176,7 @@ export function parseRawBlocksToEnrichedBlocks( ) .with({ type: "expandable-paragraph" }, parseExpandableParagraph) .with({ type: "align" }, parseAlign) + .with({ type: "entry-summary" }, parseEntrySummary) .exhaustive() } @@ -1496,6 +1500,34 @@ function parseAlign(b: RawBlockAlign): EnrichedBlockAlign { } } +function parseEntrySummary( + raw: RawBlockEntrySummary +): EnrichedBlockEntrySummary { + const parseErrors: ParseError[] = [] + const items: EnrichedBlockEntrySummaryItem[] = [] + + if (raw.value.items) { + raw.value.items.forEach((item, i) => { + if (!item.text || !item.slug) { + parseErrors.push({ + message: `entry-summary item ${i} is not valid. It must have a text and a slug property`, + }) + } else { + items.push({ + text: item.text, + slug: item.slug, + }) + } + }) + } + + return { + type: "entry-summary", + items, + parseErrors, + } +} + export function parseRefs({ refs, refsByFirstAppearance, diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 59d1e7a458d..4cb9bcd8f79 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -161,6 +161,7 @@ import { EnrichedScrollerItem, EnrichedBlockKeyInsightsSlide, UserCountryInformation, + SpanLink, } from "./owidTypes.js" import { OwidVariableWithSource } from "./OwidVariable.js" import { PointVector } from "./PointVector.js" @@ -1621,7 +1622,8 @@ export function traverseEnrichedBlocks( "sdg-grid", "sdg-toc", "topic-page-intro", - "all-charts" + "all-charts", + "entry-summary" ), }, callback @@ -1633,6 +1635,10 @@ export function checkNodeIsSpan(node: NodeWithUrl): node is Span { return "spanType" in node } +export function checkNodeIsSpanLink(node: unknown): node is SpanLink { + return isObject(node) && "spanType" in node && node.spanType === "span-link" +} + export function spansToUnformattedPlainText(spans: Span[]): string { return spans .map((span) => diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index 71d7dc2a93b..4e088544093 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -73,6 +73,8 @@ export { type EnrichedRecircLink, type EnrichedScrollerItem, type EnrichedSDGGridItem, + type EnrichedBlockEntrySummary, + type EnrichedBlockEntrySummaryItem, type EntryMeta, type EntryNode, EPOCH_DATE, @@ -153,6 +155,8 @@ export { type RawRecircLink, type RawDetail, type RawSDGGridItem, + type RawBlockEntrySummary, + type RawBlockEntrySummaryItem, type RelatedChart, type Ref, type RefDictionary, @@ -320,6 +324,7 @@ export { recursivelyMapArticleContent, traverseEnrichedBlocks, checkNodeIsSpan, + checkNodeIsSpanLink, spansToUnformattedPlainText, findDuplicates, checkIsOwidGdocType, diff --git a/packages/@ourworldindata/utils/src/owidTypes.ts b/packages/@ourworldindata/utils/src/owidTypes.ts index 6371dcc1f3e..969e1e65a0f 100644 --- a/packages/@ourworldindata/utils/src/owidTypes.ts +++ b/packages/@ourworldindata/utils/src/owidTypes.ts @@ -1052,6 +1052,32 @@ export type EnrichedBlockAlign = { content: OwidEnrichedGdocBlock[] } & EnrichedBlockWithParseErrors +export type RawBlockEntrySummaryItem = { + text?: string + slug?: string +} + +// This block renders via the TableOfContents component, same as the sdg-toc block. +// Because the summary headings can differ from the actual headings in the document, +// we need to serialize the text and slug explicitly, instead of programmatically generating them +// by analyzing the document (like we do for the sdg-toc block) +export type RawBlockEntrySummary = { + type: "entry-summary" + value: { + items?: RawBlockEntrySummaryItem[] + } +} + +export type EnrichedBlockEntrySummaryItem = { + text: string + slug: string +} + +export type EnrichedBlockEntrySummary = { + type: "entry-summary" + items: EnrichedBlockEntrySummaryItem[] +} & EnrichedBlockWithParseErrors + export type Ref = { id: string // Can be -1 @@ -1096,6 +1122,7 @@ export type OwidRawGdocBlock = | RawBlockTopicPageIntro | RawBlockKeyInsights | RawBlockAlign + | RawBlockEntrySummary export type OwidEnrichedGdocBlock = | EnrichedBlockAllCharts @@ -1129,6 +1156,7 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockKeyInsights | EnrichedBlockResearchAndWriting | EnrichedBlockAlign + | EnrichedBlockEntrySummary export enum OwidGdocPublicationContext { unlisted = "unlisted", diff --git a/site/gdocs/ArticleBlock.tsx b/site/gdocs/ArticleBlock.tsx index 9f9f282cf85..37678bd53e2 100644 --- a/site/gdocs/ArticleBlock.tsx +++ b/site/gdocs/ArticleBlock.tsx @@ -22,7 +22,7 @@ import { BlockErrorBoundary, BlockErrorFallback } from "./BlockErrorBoundary.js" import { match } from "ts-pattern" import { renderSpans } from "./utils.js" import Paragraph from "./Paragraph.js" -import SDGTableOfContents from "./SDGTableOfContents.js" +import TableOfContents from "./TableOfContents.js" import urlSlug from "url-slug" import { MissingData } from "./MissingData.js" import { AdditionalCharts } from "./AdditionalCharts.js" @@ -76,7 +76,7 @@ const layouts: { [key in Container]: Layouts} = { ["research-and-writing"]: "col-start-2 span-cols-12", ["scroller"]: "grid span-cols-12 col-start-2", ["sdg-grid"]: "grid col-start-2 span-cols-12 col-lg-start-3 span-lg-cols-10 span-sm-cols-12 col-sm-start-2", - ["sdg-toc"]: "grid grid-cols-8 col-start-4 span-cols-8 grid-md-cols-10 col-md-start-3 span-md-cols-10 grid-sm-cols-12 span-sm-cols-12 col-sm-start-2", + ["toc"]: "grid grid-cols-8 col-start-4 span-cols-8 grid-md-cols-10 col-md-start-3 span-md-cols-10 grid-sm-cols-12 span-sm-cols-12 col-sm-start-2", ["side-by-side"]: "grid span-cols-12 col-start-2", ["sticky-left-left-column"]: "grid grid-cols-7 span-cols-7 span-md-cols-12 grid-md-cols-12", ["sticky-left-right-column"]: "grid grid-cols-5 span-cols-5 span-md-cols-12 grid-md-cols-12", @@ -495,12 +495,26 @@ export default function ArticleBlock({ )) .with({ type: "sdg-toc" }, () => { return toc ? ( - ) : null }) + .with({ type: "entry-summary" }, (block) => { + return ( + ({ + ...item, + title: item.text, + isSubheading: false, + }))} + className={getLayout("toc", containerType)} + /> + ) + }) .with({ type: "missing-data" }, () => ( )) diff --git a/site/gdocs/SDGTableOfContents.scss b/site/gdocs/TableOfContents.scss similarity index 95% rename from site/gdocs/SDGTableOfContents.scss rename to site/gdocs/TableOfContents.scss index 6d484b5b5be..9fa51c2c8e8 100644 --- a/site/gdocs/SDGTableOfContents.scss +++ b/site/gdocs/TableOfContents.scss @@ -1,4 +1,4 @@ -.sdg-toc { +.toc { padding: 40px 0; margin: 32px 0; background-color: $beige; @@ -12,7 +12,7 @@ margin-right: var(--grid-gap); } - .sdg-toc-toggle { + .toc-toggle { @include h2-bold; margin: 0; padding: 0; @@ -25,8 +25,8 @@ } // Have to hard-code this because the span-cols-x overrides col-start-x - .sdg-toc-toggle, - .sdg-toc-content { + .toc-toggle, + .toc-content { grid-column-start: 2; } diff --git a/site/gdocs/SDGTableOfContents.tsx b/site/gdocs/TableOfContents.tsx similarity index 78% rename from site/gdocs/SDGTableOfContents.tsx rename to site/gdocs/TableOfContents.tsx index fbaa5e726e0..addba3d5e46 100644 --- a/site/gdocs/SDGTableOfContents.tsx +++ b/site/gdocs/TableOfContents.tsx @@ -7,12 +7,14 @@ import AnimateHeight from "react-animate-height" // See ARIA roles: https://w3c.github.io/aria-practices/examples/menu-button/menu-button-links.html -export default function SDGTableOfContents({ +export default function TableOfContents({ toc, className = "", + title, }: { toc: TocHeadingWithTitleSupertitle[] className?: string + title: string }) { const [height, setHeight] = useState<"auto" | 0>(0) const [isOpen, setIsOpen] = useState(false) @@ -23,28 +25,28 @@ export default function SDGTableOfContents({ return (