From 8936e11597db518e5ca634c5cea9b744adadfd8e Mon Sep 17 00:00:00 2001 From: sophiamersmann Date: Wed, 17 Jan 2024 09:23:07 +0000 Subject: [PATCH] :construction: (gdoc) add key indicator component prototype --- adminSiteClient/gdocsDeploy.ts | 6 +- baker/GrapherBaker.tsx | 84 ++++++------------- baker/SiteBaker.tsx | 76 ++++++++++++----- db/model/Gdoc/GdocBase.ts | 49 ++++++++++- db/model/Gdoc/GdocPost.ts | 22 ++++- db/model/Gdoc/enrichedToMarkdown.ts | 11 +++ db/model/Gdoc/enrichedToRaw.ts | 16 +++- db/model/Gdoc/exampleEnrichedBlocks.ts | 6 ++ db/model/Gdoc/rawToArchie.ts | 24 +++++- db/model/Gdoc/rawToEnriched.ts | 77 ++++++++++++++--- db/model/Variable.ts | 67 +++++++++++++-- .../types/src/gdocTypes/ArchieMlComponents.ts | 20 +++++ .../types/src/gdocTypes/Datapage.ts | 6 +- .../types/src/gdocTypes/Gdoc.ts | 7 ++ packages/@ourworldindata/types/src/index.ts | 3 + packages/@ourworldindata/utils/src/Util.ts | 3 +- site/DataInsightsIndexPageContent.tsx | 1 + site/DataPageV2Content.tsx | 2 + site/gdocs/OwidGdoc.tsx | 9 +- site/gdocs/components/ArticleBlock.tsx | 5 ++ site/gdocs/components/Chart.scss | 4 + site/gdocs/components/KeyIndicator.tsx | 46 ++++++++++ site/gdocs/utils.tsx | 22 ++++- 23 files changed, 453 insertions(+), 113 deletions(-) create mode 100644 site/gdocs/components/KeyIndicator.tsx diff --git a/adminSiteClient/gdocsDeploy.ts b/adminSiteClient/gdocsDeploy.ts index e441d69af37..98dc19d2b2a 100644 --- a/adminSiteClient/gdocsDeploy.ts +++ b/adminSiteClient/gdocsDeploy.ts @@ -1,12 +1,11 @@ +import { isEqual, omit } from "@ourworldindata/utils" import { OwidGdoc, OwidGdocBaseInterface, OwidGdocPostContent, - isEqual, - omit, OwidGdocDataInsightContent, OwidGdocType, -} from "@ourworldindata/utils" +} from "@ourworldindata/types" import { GDOC_DIFF_OMITTABLE_PROPERTIES } from "./GdocsDiff.js" import { GDOCS_DETAILS_ON_DEMAND_ID } from "../settings/clientSettings.js" @@ -52,6 +51,7 @@ export const checkIsLightningUpdate = ( breadcrumbs: true, errors: true, linkedCharts: true, + linkedIndicators: true, linkedDocuments: true, relatedCharts: true, revisionId: true, diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index 37d52ff0ecf..30f85be28de 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -7,22 +7,12 @@ import { urlToSlug, without, deserializeJSONFromHTML, - OwidVariableDataMetadataDimensions, uniq, - JsonError, keyBy, - DimensionProperty, - OwidVariableWithSource, mergePartialGrapherConfigs, - OwidChartDimensionInterface, compact, - OwidGdocPostInterface, merge, - EnrichedFaq, - FaqEntryData, - FaqDictionary, partition, - ImageMetadata, } from "@ourworldindata/utils" import { getRelatedArticles, @@ -45,13 +35,25 @@ import * as db from "../db/db.js" import { glob } from "glob" import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js" import { getPostEnrichedBySlug } from "../db/model/Post.js" -import { ChartTypeName, GrapherInterface } from "@ourworldindata/types" +import { + JsonError, + GrapherInterface, + OwidVariableDataMetadataDimensions, + DimensionProperty, + OwidVariableWithSource, + OwidChartDimensionInterface, + OwidGdocPostInterface, + EnrichedFaq, + FaqEntryData, + FaqDictionary, + ImageMetadata, +} from "@ourworldindata/types" import workerpool from "workerpool" import ProgressBar from "progress" import { getVariableData, - getVariableMetadata, getMergedGrapherConfigForVariable, + getVariableOfDatapageIfApplicable, } from "../db/model/Variable.js" import { getDatapageDataV2, getDatapageGdoc } from "../datapage/Datapage.js" import { ExplorerProgram } from "../explorer/ExplorerProgram.js" @@ -61,7 +63,6 @@ import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" import { getShortPageCitation } from "../site/gdocs/utils.js" -import { isEmpty } from "lodash" import { getSlugForTopicTag, getTagToSlugMap } from "./GrapherBakingUtils.js" const renderDatapageIfApplicable = async ( @@ -70,50 +71,19 @@ const renderDatapageIfApplicable = async ( publishedExplorersBySlug?: Record, imageMetadataDictionary?: Record ) => { - // If we have a single Y variable and that one has a schema version >= 2, - // meaning it has the metadata to render a datapage, AND if the metadata includes - // text for at least one of the description* fields or titlePublic, then we show the datapage - // based on this information. - const yVariableIds = grapher - .dimensions!.filter((d) => d.property === DimensionProperty.y) - .map((d) => d.variableId) - const xVariableIds = grapher - .dimensions!.filter((d) => d.property === DimensionProperty.x) - .map((d) => d.variableId) - // Make a data page for single indicator indicator charts. - // For scatter plots we want to only show a data page if it has no X variable mapped, which - // is a special case where time is the X axis. Marimekko charts are the other chart that uses - // the X dimension but there we usually map population on X which should not prevent us from - // showing a data page. - if ( - yVariableIds.length === 1 && - (grapher.type !== ChartTypeName.ScatterPlot || - xVariableIds.length === 0) - ) { - const variableId = yVariableIds[0] - const variableMetadata = await getVariableMetadata(variableId) - - if ( - variableMetadata.schemaVersion !== undefined && - variableMetadata.schemaVersion >= 2 && - (!isEmpty(variableMetadata.descriptionShort) || - !isEmpty(variableMetadata.descriptionProcessing) || - !isEmpty(variableMetadata.descriptionKey) || - !isEmpty(variableMetadata.descriptionFromProducer) || - !isEmpty(variableMetadata.presentation?.titlePublic)) - ) { - return await renderDataPageV2({ - variableId, - variableMetadata, - isPreviewing: isPreviewing, - useIndicatorGrapherConfigs: false, - pageGrapher: grapher, - publishedExplorersBySlug, - imageMetadataDictionary, - }) - } - } - return undefined + const variable = await getVariableOfDatapageIfApplicable(grapher) + + if (!variable) return undefined + + return await renderDataPageV2({ + variableId: variable.id, + variableMetadata: variable.metadata, + isPreviewing: isPreviewing, + useIndicatorGrapherConfigs: false, + pageGrapher: grapher, + publishedExplorersBySlug, + imageMetadataDictionary, + }) } /** diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 035d69245cd..b016e6455e4 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -47,6 +47,7 @@ import { countries, FullPost, LinkedChart, + LinkedIndicator, extractDetailsFromSyntax, OwidGdocErrorMessageType, ImageMetadata, @@ -54,6 +55,7 @@ import { OwidGdocPostInterface, OwidGdocType, DATA_INSIGHTS_INDEX_PAGE_SIZE, + excludeUndefined, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" @@ -83,6 +85,10 @@ import { } from "../settings/clientSettings.js" import pMap from "p-map" import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js" +import { + getVariableMetadata, + getVariableOfDatapageIfApplicable, +} from "../db/model/Variable.js" type PrefetchedAttachments = { linkedDocuments: Record @@ -91,6 +97,7 @@ type PrefetchedAttachments = { graphers: Record explorers: Record } + linkedIndicators: Record } // These aren't all "wordpress" steps @@ -282,7 +289,7 @@ export class SiteBaker { return without(existingSlugs, ...postSlugsFromDb) } - // Prefetches all linkedDocuments, imageMetadata, and linkedCharts instead of having to fetch them + // Prefetches all linkedDocuments, imageMetadata, linkedCharts, and linkedIndicators instead of having to fetch them // for each individual gdoc. Optionally takes a tuple of string arrays to pick from the prefetched // dictionaries. _prefetchedAttachmentsCache: PrefetchedAttachments | undefined = undefined @@ -308,23 +315,38 @@ export class SiteBaker { `${BAKED_BASE_URL}/default-thumbnail.jpg`, })) ) + // Includes redirects - const publishedChartsBySlug = await Chart.mapSlugsToConfigs().then( - (results) => - results.reduce( - (acc, cur) => ({ - ...acc, - [cur.slug]: { - originalSlug: cur.slug, - resolvedUrl: `${BAKED_GRAPHER_URL}/${cur.config.slug}`, - queryString: "", - title: cur.config.title || "", - thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${cur.config.slug}.svg`, - }, - }), - {} as Record - ) + const publishedChartsRaw = await Chart.mapSlugsToConfigs() + const publishedCharts: LinkedChart[] = await Promise.all( + publishedChartsRaw.map(async (chart) => { + const datapageIndicator = + await getVariableOfDatapageIfApplicable(chart.config) + return { + originalSlug: chart.slug, + resolvedUrl: `${BAKED_GRAPHER_URL}/${chart.config.slug}`, + queryString: "", + title: chart.config.title || "", + thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${chart.config.slug}.svg`, + indicatorId: datapageIndicator?.id, + } + }) + ) + const publishedChartsBySlug = keyBy(publishedCharts, "originalSlug") + + const datapageIndicatorIds = excludeUndefined( + publishedCharts.map((chart) => chart.indicatorId) + ) + const datapageIndicators: LinkedIndicator[] = await Promise.all( + datapageIndicatorIds.map(async (indicatorId: number) => { + const metadata = await getVariableMetadata(indicatorId) + return { + id: indicatorId, + titlePublic: metadata.presentation?.titlePublic, + } + }) ) + const datapageIndicatorsById = keyBy(datapageIndicators, "id") const prefetchedAttachments = { linkedDocuments: publishedGdocsDictionary, @@ -333,6 +355,7 @@ export class SiteBaker { explorers: publishedExplorersBySlug, graphers: publishedChartsBySlug, }, + linkedIndicators: datapageIndicatorsById, } this._prefetchedAttachmentsCache = prefetchedAttachments } @@ -353,6 +376,16 @@ export class SiteBaker { .map((gdoc) => gdoc.content["featured-image"]) .filter((filename): filename is string => !!filename) + const linkedGrapherCharts = pick( + this._prefetchedAttachmentsCache.linkedCharts.graphers, + linkedGrapherSlugs + ) + const linkedIndicatorIds = excludeUndefined( + Object.values(linkedGrapherCharts).map( + (chart) => chart.indicatorId + ) + ) + return { linkedDocuments, imageMetadata: pick( @@ -361,11 +394,7 @@ export class SiteBaker { ), linkedCharts: { graphers: { - ...pick( - this._prefetchedAttachmentsCache.linkedCharts - .graphers, - linkedGrapherSlugs - ), + ...linkedGrapherCharts, }, explorers: { ...pick( @@ -375,6 +404,10 @@ export class SiteBaker { ), }, }, + linkedIndicators: pick( + this._prefetchedAttachmentsCache.linkedIndicators, + linkedIndicatorIds + ), } } return this._prefetchedAttachmentsCache @@ -460,6 +493,7 @@ export class SiteBaker { ...attachments.linkedCharts.graphers, ...attachments.linkedCharts.explorers, } + publishedGdoc.linkedIndicators = attachments.linkedIndicators // this is a no-op if the gdoc doesn't have an all-chart block await publishedGdoc.loadRelatedCharts() diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 80d60e357ac..dcd30e8d8e9 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -12,6 +12,7 @@ import { import { getUrlTarget } from "@ourworldindata/components" import { LinkedChart, + LinkedIndicator, keyBy, excludeNull, ImageMetadata, @@ -34,6 +35,7 @@ import { MinimalDataInsightInterface, getFeaturedImageFilename, OwidGdoc, + urlToSlug, } from "@ourworldindata/utils" import { BAKED_GRAPHER_URL } from "../../../settings/serverSettings.js" import { google } from "googleapis" @@ -55,6 +57,10 @@ import { } from "./gdocUtils.js" import { OwidGoogleAuth } from "../../OwidGoogleAuth.js" import { enrichedBlocksToMarkdown } from "./enrichedToMarkdown.js" +import { + getVariableMetadata, + getVariableOfDatapageIfApplicable, +} from "../Variable.js" @Entity("tags") export class Tag extends BaseEntity implements TagInterface { @@ -98,6 +104,7 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { errors: OwidGdocErrorMessage[] = [] imageMetadata: Record = {} linkedCharts: Record = {} + linkedIndicators: Record = {} linkedDocuments: Record = {} latestDataInsights: MinimalDataInsightInterface[] = [] @@ -268,6 +275,20 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { return [...this.filenames, ...featuredImages] } + get linkedKeyIndicatorSlugs(): string[] { + const slugs = new Set() + for (const enrichedBlockSource of this.enrichedBlockSources) { + for (const block of enrichedBlockSource) { + traverseEnrichedBlocks(block, (block) => { + if (block.type === "key-indicator") { + slugs.add(urlToSlug(block.datapageUrl)) + } + }) + } + } + return [...slugs] + } + get linkedChartSlugs(): { grapher: string[]; explorer: string[] } { const { grapher, explorer } = this.links.reduce( (slugsByLinkType, { linkType, target }) => { @@ -282,6 +303,8 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { } ) + this.linkedKeyIndicatorSlugs.forEach((slug) => grapher.add(slug)) + return { grapher: [...grapher], explorer: [...explorer] } } @@ -508,7 +531,8 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { "sticky-left", "sticky-right", "table", - "text" + "text", + "key-indicator" ), }, () => [] @@ -549,11 +573,14 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { if (!chart) return const resolvedSlug = chart.config.slug ?? "" const resolvedTitle = chart.config.title ?? "" + const datapageIndicator = + await getVariableOfDatapageIfApplicable(chart.config) const linkedChart: LinkedChart = { originalSlug, title: resolvedTitle, resolvedUrl: `${BAKED_GRAPHER_URL}/${resolvedSlug}`, thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${resolvedSlug}.svg`, + indicatorId: datapageIndicator?.id, } return linkedChart } @@ -581,6 +608,25 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { ) } + async loadLinkedIndicators(): Promise { + const linkedIndicators = await Promise.all( + this.linkedKeyIndicatorSlugs.map(async (originalSlug) => { + const linkedChart = this.linkedCharts[originalSlug] + if (!linkedChart || !linkedChart.indicatorId) return + const metadata = await getVariableMetadata( + linkedChart.indicatorId + ) + const linkedIndicator: LinkedIndicator = { + id: linkedChart.indicatorId, + titlePublic: metadata.presentation?.titlePublic, + } + return linkedIndicator + }) + ).then(excludeNullish) + + this.linkedIndicators = keyBy(linkedIndicators, "id") + } + async loadLinkedDocuments(): Promise { const linkedDocuments = await Promise.all( this.linkedDocumentIds.map(async (target) => { @@ -704,6 +750,7 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { await this.loadLinkedDocuments() await this.loadImageMetadata() await this.loadLinkedCharts(publishedExplorersBySlug) + await this.loadLinkedIndicators() // depends on linked charts await this._loadSubclassAttachments() await this.validate(publishedExplorersBySlug) } diff --git a/db/model/Gdoc/GdocPost.ts b/db/model/Gdoc/GdocPost.ts index fa4e0e9810f..f6ee0e0229b 100644 --- a/db/model/Gdoc/GdocPost.ts +++ b/db/model/Gdoc/GdocPost.ts @@ -11,7 +11,8 @@ import { OwidEnrichedGdocBlock, RawBlockText, RelatedChart, -} from "@ourworldindata/utils" +} from "@ourworldindata/types" +import { traverseEnrichedBlocks, urlToSlug } from "@ourworldindata/utils" import { GDOCS_DETAILS_ON_DEMAND_ID } from "../../../settings/serverSettings.js" import { formatCitation, @@ -143,6 +144,25 @@ export class GdocPost extends GdocBase implements OwidGdocPostInterface { } } + // Validate that charts referenced in key-indicator blocks render a datapage + for (const enrichedBlockSource of this.enrichedBlockSources) { + enrichedBlockSource.forEach((block) => + traverseEnrichedBlocks(block, (block) => { + if (block.type === "key-indicator" && block.datapageUrl) { + const slug = urlToSlug(block.datapageUrl) + const linkedChart = this.linkedCharts?.[slug] + if (!linkedChart?.indicatorId) { + errors.push({ + property: "body", + type: OwidGdocErrorMessageType.Error, + message: `Grapher chart with slug ${slug} is not a datapage`, + }) + } + } + }) + ) + } + return errors } diff --git a/db/model/Gdoc/enrichedToMarkdown.ts b/db/model/Gdoc/enrichedToMarkdown.ts index 1106b9d75af..e9ef6cb4f57 100644 --- a/db/model/Gdoc/enrichedToMarkdown.ts +++ b/db/model/Gdoc/enrichedToMarkdown.ts @@ -271,5 +271,16 @@ ${links}` ).join("\n\n> ") return `> ${text}` + b.citation ? `\n-- ${b.citation}` : "" }) + .with({ type: "key-indicator" }, (b): string | undefined => + markdownComponent( + "KeyIndicator", + { + datapageUrl: b.datapageUrl, + title: b.title, + // blurb ignored + }, + exportComponents + ) + ) .exhaustive() } diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index 5eef3e6bb02..2c619fbed27 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -36,7 +36,8 @@ import { RawBlockVideo, RawBlockTable, RawBlockBlockquote, -} from "@ourworldindata/utils" + RawBlockKeyIndicator, +} from "@ourworldindata/types" import { spanToHtmlString } from "./gdocUtils.js" import { match, P } from "ts-pattern" @@ -433,5 +434,18 @@ export function enrichedBlockToRawBlock( }, } }) + .with({ type: "key-indicator" }, (b): RawBlockKeyIndicator => { + return { + type: "key-indicator", + value: { + datapageUrl: b.datapageUrl, + blurb: b.blurb.map((enriched) => ({ + type: "text", + value: spansToHtmlText(enriched.value), + })), + title: b.title, + }, + } + }) .exhaustive() } diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index a164992f511..6316bba6dab 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -535,4 +535,10 @@ export const enrichedBlockExamples: Record< citation: "Max Roser", parseErrors: [], }, + "key-indicator": { + type: "key-indicator", + datapageUrl: "https://ourworldindata.org/grapher/life-expectancy", + blurb: [enrichedBlockText], + parseErrors: [], + }, } diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index 28210b86be4..cbec65b58cf 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -32,11 +32,12 @@ import { RawBlockExpandableParagraph, RawBlockAlign, RawBlockEntrySummary, - isArray, RawBlockTable, RawBlockTableRow, RawBlockBlockquote, -} from "@ourworldindata/utils" + RawBlockKeyIndicator, +} from "@ourworldindata/types" +import { isArray } from "@ourworldindata/utils" import { match } from "ts-pattern" export function appendDotEndIfMultiline( @@ -620,6 +621,24 @@ function* rawBlockTableToArchieMLString( yield "{}" } +function* rawBlockKeyIndicatorToArchieMLString( + block: RawBlockKeyIndicator +): Generator { + yield "{.key-indicator}" + if (typeof block.value !== "string") { + yield* propertyToArchieMLString("datapageUrl", block.value) + yield* propertyToArchieMLString("title", block.value) + if (block.value.blurb) { + yield "[.+blurb]" + for (const textBlock of block.value.blurb) { + yield* OwidRawGdocBlockToArchieMLStringGenerator(textBlock) + } + yield "[]" + } + } + yield "{}" +} + export function* OwidRawGdocBlockToArchieMLStringGenerator( block: OwidRawGdocBlock | RawBlockTableRow ): Generator { @@ -684,6 +703,7 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( .with({ type: "table" }, rawBlockTableToArchieMLString) .with({ type: "table-row" }, rawBlockRowToArchieMLString) .with({ type: "blockquote" }, rawBlockBlockquoteToArchieMLString) + .with({ type: "key-indicator" }, rawBlockKeyIndicatorToArchieMLString) .exhaustive() yield* content } diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index bd497e098bd..3b5633e2748 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -3,7 +3,6 @@ import { ChartPositionChoice, ChartControlKeyword, ChartTabKeyword, - compact, EnrichedBlockAside, EnrichedBlockCallout, EnrichedBlockChart, @@ -33,12 +32,11 @@ import { EnrichedRecircLink, EnrichedScrollerItem, EnrichedSDGGridItem, - isArray, + EnrichedBlockKeyIndicator, OwidEnrichedGdocBlock, OwidRawGdocBlock, OwidGdocErrorMessage, ParseError, - partition, RawBlockAdditionalCharts, RawBlockAside, RawBlockCallout, @@ -61,23 +59,18 @@ import { RawBlockStickyLeftContainer, RawBlockStickyRightContainer, RawBlockText, + RawBlockKeyIndicator, Span, SpanSimpleText, - omitUndefinedValues, EnrichedBlockSimpleText, BlockImageSize, checkIsBlockImageSize, RawBlockTopicPageIntro, EnrichedBlockTopicPageIntro, - Url, EnrichedTopicPageIntroRelatedTopic, DetailDictionary, EnrichedDetail, - checkIsPlainObjectWithGuard, EnrichedBlockKeyInsightsSlide, - keyBy, - filterValidStringValues, - uniq, RawBlockResearchAndWriting, RawBlockResearchAndWritingLink, EnrichedBlockResearchAndWriting, @@ -89,7 +82,6 @@ import { EnrichedBlockAllCharts, RefDictionary, OwidGdocErrorMessageType, - excludeNullish, RawBlockResearchAndWritingRow, EnrichedBlockAlign, HorizontalAlign, @@ -109,7 +101,19 @@ import { tableTemplates, RawBlockBlockquote, EnrichedBlockBlockquote, +} from "@ourworldindata/types" +import { traverseEnrichedSpan, + keyBy, + filterValidStringValues, + uniq, + excludeNullish, + checkIsPlainObjectWithGuard, + omitUndefinedValues, + Url, + isArray, + partition, + compact, } from "@ourworldindata/utils" import { checkIsInternalLink } from "@ourworldindata/components" import { @@ -194,6 +198,7 @@ export function parseRawBlocksToEnrichedBlocks( .with({ type: "align" }, parseAlign) .with({ type: "entry-summary" }, parseEntrySummary) .with({ type: "table" }, parseTable) + .with({ type: "key-indicator" }, parseKeyIndicator) .exhaustive() } @@ -1879,3 +1884,55 @@ export function parseRefs({ return { definitions: parsedRefs, errors: refErrors } } + +const parseKeyIndicator = ( + raw: RawBlockKeyIndicator +): EnrichedBlockKeyIndicator => { + const createError = ( + error: ParseError, + datapageUrl: string + ): EnrichedBlockKeyIndicator => ({ + type: "key-indicator", + datapageUrl, + blurb: [], + parseErrors: [error], + }) + + const val = raw.value + + if (typeof val === "string") + return createError( + { + message: "Value is a string, not an object with properties", + }, + "" + ) + + if (!val.datapageUrl) + return createError( + { message: "datapageUrl property is missing or empty" }, + "" + ) + + const url = extractUrl(val.datapageUrl) + + if (!isArray(val.blurb)) + return createError( + { + message: + "Blurb is not a freeform array. Make sure you've written [.+blurb]", + }, + url + ) + + const parsedBlurb = val.blurb.map(parseText) + const parsedBlurbErrors = parsedBlurb.flatMap((block) => block.parseErrors) + + return omitUndefinedValues({ + type: "key-indicator", + datapageUrl: url, + blurb: parsedBlurb, + title: val.title, + parseErrors: parsedBlurbErrors, + }) as EnrichedBlockKeyIndicator +} diff --git a/db/model/Variable.ts b/db/model/Variable.ts index 3514268ac80..b5fd936acf5 100644 --- a/db/model/Variable.ts +++ b/db/model/Variable.ts @@ -1,6 +1,14 @@ import _ from "lodash" import { Writable } from "stream" import * as db from "../db.js" +import { retryPromise, isEmpty } from "@ourworldindata/utils" +import { + getVariableDataRoute, + getVariableMetadataRoute, +} from "@ourworldindata/grapher" +import pl from "nodejs-polars" +import { DATA_API_URL } from "../../settings/serverSettings.js" +import { escape } from "mysql" import { OwidChartDimensionInterface, OwidVariableDisplayConfigInterface, @@ -11,18 +19,12 @@ import { DataValueResult, OwidVariableWithSourceAndDimension, OwidVariableId, - retryPromise, + ChartTypeName, + DimensionProperty, OwidLicense, GrapherInterface, OwidProcessingLevel, -} from "@ourworldindata/utils" -import { - getVariableDataRoute, - getVariableMetadataRoute, -} from "@ourworldindata/grapher" -import pl from "nodejs-polars" -import { DATA_API_URL } from "../../settings/serverSettings.js" -import { escape } from "mysql" +} from "@ourworldindata/types" export interface VariableRow { id: number @@ -468,6 +470,53 @@ export const readSQLasDF = async ( return createDataFrame(await db.queryMysql(sql, params)) } +export async function getVariableOfDatapageIfApplicable( + grapher: GrapherInterface +): Promise< + | { + id: number + metadata: OwidVariableWithSourceAndDimension + } + | undefined +> { + // If we have a single Y variable and that one has a schema version >= 2, + // meaning it has the metadata to render a datapage, AND if the metadata includes + // text for at least one of the description* fields or titlePublic, then we show the datapage + // based on this information. + const yVariableIds = grapher + .dimensions!.filter((d) => d.property === DimensionProperty.y) + .map((d) => d.variableId) + const xVariableIds = grapher + .dimensions!.filter((d) => d.property === DimensionProperty.x) + .map((d) => d.variableId) + // Make a data page for single indicator indicator charts. + // For scatter plots we want to only show a data page if it has no X variable mapped, which + // is a special case where time is the X axis. Marimekko charts are the other chart that uses + // the X dimension but there we usually map population on X which should not prevent us from + // showing a data page. + if ( + yVariableIds.length === 1 && + (grapher.type !== ChartTypeName.ScatterPlot || + xVariableIds.length === 0) + ) { + const variableId = yVariableIds[0] + const variableMetadata = await getVariableMetadata(variableId) + + if ( + variableMetadata.schemaVersion !== undefined && + variableMetadata.schemaVersion >= 2 && + (!isEmpty(variableMetadata.descriptionShort) || + !isEmpty(variableMetadata.descriptionProcessing) || + !isEmpty(variableMetadata.descriptionKey) || + !isEmpty(variableMetadata.descriptionFromProducer) || + !isEmpty(variableMetadata.presentation?.titlePublic)) + ) { + return { id: variableId, metadata: variableMetadata } + } + } + return undefined +} + /** * Perform regex search over the variables table. */ diff --git a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts index ef32a56a413..9d5bfb12147 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts @@ -86,6 +86,24 @@ export type EnrichedBlockChart = { tabs?: ChartTabKeyword[] } & EnrichedBlockWithParseErrors +export type RawBlockKeyIndicatorValue = { + datapageUrl?: string + title?: string + blurb?: RawBlockText[] +} + +export type RawBlockKeyIndicator = { + type: "key-indicator" + value: RawBlockKeyIndicatorValue | ArchieMLUnexpectedNonObjectValue +} + +export type EnrichedBlockKeyIndicator = { + type: "key-indicator" + datapageUrl: string + blurb: EnrichedBlockText[] + title?: string +} & EnrichedBlockWithParseErrors + export type RawBlockScroller = { type: "scroller" value: OwidRawGdocBlock[] | ArchieMLUnexpectedNonObjectValue @@ -723,6 +741,7 @@ export type OwidRawGdocBlock = | RawBlockEntrySummary | RawBlockTable | RawBlockBlockquote + | RawBlockKeyIndicator export type OwidEnrichedGdocBlock = | EnrichedBlockAllCharts @@ -760,3 +779,4 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockEntrySummary | EnrichedBlockTable | EnrichedBlockBlockquote + | EnrichedBlockKeyIndicator diff --git a/packages/@ourworldindata/types/src/gdocTypes/Datapage.ts b/packages/@ourworldindata/types/src/gdocTypes/Datapage.ts index 59aca6cfa0e..3ffa2a226d9 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Datapage.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Datapage.ts @@ -146,7 +146,11 @@ export type DataPageParseError = { message: string; path?: string } export type FaqEntryData = Pick< OwidGdocPostInterface, - "linkedCharts" | "linkedDocuments" | "relatedCharts" | "imageMetadata" + | "linkedCharts" + | "linkedIndicators" + | "linkedDocuments" + | "relatedCharts" + | "imageMetadata" > & { faqs: OwidEnrichedGdocBlock[] } diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 887c5b3f86c..7bba258f9bb 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -23,6 +23,12 @@ export interface LinkedChart { resolvedUrl: string title: string thumbnail?: string + indicatorId?: number // in case of a datapage +} + +export interface LinkedIndicator { + id: number + titlePublic?: string } export enum OwidGdocType { @@ -46,6 +52,7 @@ export interface OwidGdocBaseInterface { breadcrumbs?: BreadcrumbItem[] | null linkedDocuments?: Record linkedCharts?: Record + linkedIndicators?: Record imageMetadata?: Record relatedCharts?: RelatedChart[] tags?: Tag[] diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 4c348d2d1af..ba7c6ddf51a 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -192,6 +192,7 @@ export { type RawBlockText, type RawBlockTopicPageIntro, type RawBlockUrl, + type RawBlockKeyIndicator, tableTemplates, type TableTemplate, tableSizes, @@ -249,6 +250,7 @@ export { type EnrichedBlockTable, type EnrichedBlockTableRow, type EnrichedBlockTableCell, + type EnrichedBlockKeyIndicator, type RawBlockResearchAndWritingRow, } from "./gdocTypes/ArchieMlComponents.js" export { @@ -276,6 +278,7 @@ export { GdocsContentSource, type OwidArticleBackportingStatistics, type LinkedChart, + type LinkedIndicator, DYNAMIC_COLLECTION_PAGE_CONTAINER_ID, } from "./gdocTypes/Gdoc.js" diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index b3e306c5ccd..bf1fe046df7 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -1643,7 +1643,8 @@ export function traverseEnrichedBlocks( "sdg-toc", "topic-page-intro", "all-charts", - "entry-summary" + "entry-summary", + "key-indicator" ), }, callback diff --git a/site/DataInsightsIndexPageContent.tsx b/site/DataInsightsIndexPageContent.tsx index f9f4205bce2..501473cf07c 100644 --- a/site/DataInsightsIndexPageContent.tsx +++ b/site/DataInsightsIndexPageContent.tsx @@ -103,6 +103,7 @@ export const DataInsightsIndexPageContent = ( imageMetadata, linkedCharts, linkedDocuments, + linkedIndicators: {}, // not needed for data insights relatedCharts: [], // not needed for the index page latestDataInsights: [], // not needed for the index page }} diff --git a/site/DataPageV2Content.tsx b/site/DataPageV2Content.tsx index 64eb84413ca..e10f31f2bff 100644 --- a/site/DataPageV2Content.tsx +++ b/site/DataPageV2Content.tsx @@ -163,6 +163,7 @@ export const DataPageV2Content = ({ linkedDocuments = {}, imageMetadata = {}, linkedCharts = {}, + linkedIndicators = {}, relatedCharts = [], } = faqEntries ?? {} @@ -235,6 +236,7 @@ export const DataPageV2Content = ({ linkedDocuments, imageMetadata, linkedCharts, + linkedIndicators, relatedCharts, }} > diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index d0ee5896664..ffa9fc7dd0f 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -2,15 +2,15 @@ import React, { createContext } from "react" import ReactDOM from "react-dom" import { LinkedChart, + LinkedIndicator, OwidGdocPostInterface, - getOwidGdocFromJSON, ImageMetadata, RelatedChart, - get, OwidGdocType, OwidGdoc as OwidGdocInterface, MinimalDataInsightInterface, -} from "@ourworldindata/utils" +} from "@ourworldindata/types" +import { get, getOwidGdocFromJSON } from "@ourworldindata/utils" import { DebugProvider } from "./DebugContext.js" import { match, P } from "ts-pattern" import { GdocPost } from "./pages/GdocPost.js" @@ -19,6 +19,7 @@ import { Fragment } from "./pages/Fragment.js" export const AttachmentsContext = createContext<{ linkedCharts: Record + linkedIndicators: Record linkedDocuments: Record imageMetadata: Record relatedCharts: RelatedChart[] @@ -27,6 +28,7 @@ export const AttachmentsContext = createContext<{ linkedDocuments: {}, imageMetadata: {}, linkedCharts: {}, + linkedIndicators: {}, relatedCharts: [], latestDataInsights: [], }) @@ -97,6 +99,7 @@ export function OwidGdoc({ linkedDocuments: get(props, "linkedDocuments", {}), imageMetadata: get(props, "imageMetadata", {}), linkedCharts: get(props, "linkedCharts", {}), + linkedIndicators: get(props, "linkedIndicators", {}), relatedCharts: get(props, "relatedCharts", []), latestDataInsights: get(props, "latestDataInsights", []), }} diff --git a/site/gdocs/components/ArticleBlock.tsx b/site/gdocs/components/ArticleBlock.tsx index 7de53f5cf1a..dbd8506547f 100644 --- a/site/gdocs/components/ArticleBlock.tsx +++ b/site/gdocs/components/ArticleBlock.tsx @@ -34,6 +34,7 @@ import { ResearchAndWriting } from "./ResearchAndWriting.js" import { AllCharts } from "./AllCharts.js" import Video from "./Video.js" import { Table } from "./Table.js" +import KeyIndicator from "./KeyIndicator.js" export type Container = | "default" @@ -69,6 +70,7 @@ const layouts: { [key in Container]: Layouts} = { ["image--narrow"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 col-sm-start-2 span-sm-cols-12", ["image--wide"]: "col-start-4 span-cols-8 col-md-start-2 span-md-cols-12", ["image-caption"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", + ["key-indicator"]: "col-start-5 span-cols-6", ["key-insights"]: "col-start-2 span-cols-12", ["list"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", ["numbered-list"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", @@ -623,6 +625,9 @@ export default function ArticleBlock({ ) }) + .with({ type: "key-indicator" }, (block) => ( + + )) .exhaustive() return ( diff --git a/site/gdocs/components/Chart.scss b/site/gdocs/components/Chart.scss index d6f10aad371..9221e0c71bc 100644 --- a/site/gdocs/components/Chart.scss +++ b/site/gdocs/components/Chart.scss @@ -11,3 +11,7 @@ figure.chart { figure.explorer { height: 700px; } + +div.margin-0 figure { + margin: 0; +} diff --git a/site/gdocs/components/KeyIndicator.tsx b/site/gdocs/components/KeyIndicator.tsx new file mode 100644 index 00000000000..0c574a2c1ed --- /dev/null +++ b/site/gdocs/components/KeyIndicator.tsx @@ -0,0 +1,46 @@ +import React from "react" +import { EnrichedBlockKeyIndicator } from "@ourworldindata/types" +import Chart from "./Chart.js" +import Paragraph from "./Paragraph.js" +import { useLinkedChart, useLinkedIndicator } from "../utils.js" + +export default function KeyIndicator({ + d, + className, +}: { + d: EnrichedBlockKeyIndicator + className?: string +}) { + const { linkedChart } = useLinkedChart(d.datapageUrl) + const { linkedIndicator } = useLinkedIndicator( + linkedChart?.indicatorId ?? 0 + ) + + if (!linkedChart) return null + if (!linkedIndicator) return null + + return ( +
+
+ Custom title: {d.title} +
+
+ Default title: {linkedChart?.title} +
+
+ Blurb: + {d.blurb.map((textBlock, i) => ( + + ))} +
+ +
+ Linked indicator with metadata: + {JSON.stringify(linkedIndicator, null, 2)} +
+
+ ) +} diff --git a/site/gdocs/utils.tsx b/site/gdocs/utils.tsx index 9333c3564e0..f326cf791fb 100644 --- a/site/gdocs/utils.tsx +++ b/site/gdocs/utils.tsx @@ -7,10 +7,10 @@ import { OwidGdocPostInterface, ImageMetadata, LinkedChart, - Url, OwidGdocPostContent, - formatAuthors, -} from "@ourworldindata/utils" + LinkedIndicator, +} from "@ourworldindata/types" +import { formatAuthors, Url } from "@ourworldindata/utils" import { match } from "ts-pattern" import { AttachmentsContext } from "./OwidGdoc.js" @@ -103,6 +103,22 @@ export const useLinkedChart = ( } } +export const useLinkedIndicator = ( + id: number +): { linkedIndicator?: LinkedIndicator; errorMessage?: string } => { + const { linkedIndicators } = useContext(AttachmentsContext) + + const linkedIndicator = linkedIndicators?.[id] + + if (!linkedIndicator) { + return { + errorMessage: `Indicator with id ${id} not found`, + } + } + + return { linkedIndicator } +} + export const useImage = ( filename: string | undefined ): ImageMetadata | undefined => {