From 8ae533ee0fe64fc3c3a5a2865a87c8c77268c88d Mon Sep 17 00:00:00 2001 From: Ike Saunders Date: Fri, 5 Jan 2024 16:59:29 -0500 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=94=A8=20use=20a=20minimal=20represen?= =?UTF-8?q?tation=20of=20gdocs=20to=20link=20to=20instead=20of=20whole=20m?= =?UTF-8?q?odel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- baker/SiteBaker.tsx | 11 +++++--- db/model/Gdoc/GdocBase.ts | 25 +++++++++++-------- db/model/Gdoc/GdocDataInsight.ts | 4 +-- db/model/Gdoc/GdocPost.ts | 3 ++- db/model/Gdoc/gdocUtils.ts | 21 ++++++++++++++++ .../types/src/gdocTypes/Gdoc.ts | 20 ++++++++++++--- packages/@ourworldindata/types/src/index.ts | 1 + site/DataInsightsIndexPageContent.tsx | 7 ++++-- site/gdocs/OwidGdoc.tsx | 4 +-- site/gdocs/components/ProminentLink.tsx | 6 ++--- site/gdocs/components/Recirc.tsx | 8 +++--- site/gdocs/components/ResearchAndWriting.tsx | 8 +++--- site/gdocs/components/TopicPageIntro.tsx | 2 +- site/gdocs/utils.tsx | 13 ++++++---- 14 files changed, 90 insertions(+), 43 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 035d69245cd..3f80efb48d3 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -51,9 +51,9 @@ import { OwidGdocErrorMessageType, ImageMetadata, OwidGdoc, - OwidGdocPostInterface, OwidGdocType, DATA_INSIGHTS_INDEX_PAGE_SIZE, + OwidGdocMinimalPostInterface, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" @@ -83,9 +83,10 @@ import { } from "../settings/clientSettings.js" import pMap from "p-map" import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js" +import { fullGdocToMinimalGdoc } from "../db/model/Gdoc/gdocUtils.js" type PrefetchedAttachments = { - linkedDocuments: Record + linkedDocuments: Record imageMetadata: Record linkedCharts: { graphers: Record @@ -290,7 +291,9 @@ export class SiteBaker { picks?: [string[], string[], string[], string[]] ): Promise { if (!this._prefetchedAttachmentsCache) { - const publishedGdocs = await GdocPost.getPublishedGdocs() + const publishedGdocs = await GdocPost.getPublishedGdocs().then( + (fullGdocs) => fullGdocs.map(fullGdocToMinimalGdoc) + ) const publishedGdocsDictionary = keyBy(publishedGdocs, "id") const imageMetadataDictionary: Record = @@ -350,7 +353,7 @@ export class SiteBaker { // Gdoc.linkedImageFilenames normally gets featuredImages, but it relies on linkedDocuments already being populated, // which is isn't when we're prefetching attachments. So we have to do it manually here. const featuredImages = Object.values(linkedDocuments) - .map((gdoc) => gdoc.content["featured-image"]) + .map((gdoc) => gdoc["featured-image"]) .filter((filename): filename is string => !!filename) return { diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 80d60e357ac..8b7d9708cb8 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -32,8 +32,7 @@ import { OwidGdocPublicationContext, BreadcrumbItem, MinimalDataInsightInterface, - getFeaturedImageFilename, - OwidGdoc, + OwidGdocMinimalPostInterface, } from "@ourworldindata/utils" import { BAKED_GRAPHER_URL } from "../../../settings/serverSettings.js" import { google } from "googleapis" @@ -50,6 +49,7 @@ import { EXPLORERS_ROUTE_FOLDER } from "../../../explorer/ExplorerConstants.js" import { match, P } from "ts-pattern" import { extractUrl, + fullGdocToMinimalGdoc, getAllLinksFromResearchAndWritingBlock, spansToSimpleString, } from "./gdocUtils.js" @@ -98,7 +98,7 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { errors: OwidGdocErrorMessage[] = [] imageMetadata: Record = {} linkedCharts: Record = {} - linkedDocuments: Record = {} + linkedDocuments: Record = {} latestDataInsights: MinimalDataInsightInterface[] = [] _getSubclassEnrichedBlocks: (gdoc: typeof this) => OwidEnrichedGdocBlock[] = @@ -262,7 +262,7 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { // but we try (and then filter nulls) because we need featured images if we're using prominent links // even if this method is being called on a GdocFaq (for example) const featuredImages = Object.values(this.linkedDocuments) - .map((d) => getFeaturedImageFilename(d as OwidGdoc)) + .map((d) => d["featured-image"]) .filter((filename?: string): filename is string => !!filename) return [...this.filenames, ...featuredImages] @@ -582,14 +582,17 @@ export class GdocBase extends BaseEntity implements OwidGdocBaseInterface { } async loadLinkedDocuments(): Promise { - const linkedDocuments = await Promise.all( - this.linkedDocumentIds.map(async (target) => { - const linkedDocument = await GdocBase.findOneBy({ - id: target, + const linkedDocuments: OwidGdocMinimalPostInterface[] = + await Promise.all( + this.linkedDocumentIds.map(async (target) => { + const linkedDocument = await GdocBase.findOneBy({ + id: target, + }) + return linkedDocument }) - return linkedDocument - }) - ).then(excludeNull) + ) + .then(excludeNull) + .then((fullGdocs) => fullGdocs.map(fullGdocToMinimalGdoc)) this.linkedDocuments = keyBy(linkedDocuments, "id") } diff --git a/db/model/Gdoc/GdocDataInsight.ts b/db/model/Gdoc/GdocDataInsight.ts index b12f58fae29..b8d4d711276 100644 --- a/db/model/Gdoc/GdocDataInsight.ts +++ b/db/model/Gdoc/GdocDataInsight.ts @@ -4,10 +4,10 @@ import { OwidGdocErrorMessageType, OwidGdocDataInsightContent, OwidGdocDataInsightInterface, - OwidGdocPostInterface, MinimalDataInsightInterface, OwidGdocType, DATA_INSIGHTS_INDEX_PAGE_SIZE, + OwidGdocMinimalPostInterface, } from "@ourworldindata/utils" import { GdocBase } from "./GdocBase.js" import { getConnection } from "../../../db/db.js" @@ -27,7 +27,7 @@ export class GdocDataInsight } } - linkedDocuments: Record = {} + linkedDocuments: Record = {} latestDataInsights: MinimalDataInsightInterface[] = [] // TODO: support query parameters in grapher urls so we can track country selections _urlProperties: string[] = ["grapher-url"] diff --git a/db/model/Gdoc/GdocPost.ts b/db/model/Gdoc/GdocPost.ts index fa4e0e9810f..2a74cd05424 100644 --- a/db/model/Gdoc/GdocPost.ts +++ b/db/model/Gdoc/GdocPost.ts @@ -11,6 +11,7 @@ import { OwidEnrichedGdocBlock, RawBlockText, RelatedChart, + OwidGdocMinimalPostInterface, } from "@ourworldindata/utils" import { GDOCS_DETAILS_ON_DEMAND_ID } from "../../../settings/serverSettings.js" import { @@ -35,7 +36,7 @@ export class GdocPost extends GdocBase implements OwidGdocPostInterface { } } - linkedDocuments: Record = {} + linkedDocuments: Record = {} relatedCharts: RelatedChart[] = [] _filenameProperties = ["cover-image", "featured-image"] diff --git a/db/model/Gdoc/gdocUtils.ts b/db/model/Gdoc/gdocUtils.ts index e6509e9db44..a5489181e86 100644 --- a/db/model/Gdoc/gdocUtils.ts +++ b/db/model/Gdoc/gdocUtils.ts @@ -3,6 +3,10 @@ import { Span, excludeNullish, EnrichedBlockResearchAndWritingLink, + OwidGdocMinimalPostInterface, + OwidGdocType, + formatDate, + OwidGdocBaseInterface, } from "@ourworldindata/utils" import { match, P } from "ts-pattern" import cheerio from "cheerio" @@ -151,3 +155,20 @@ export function parseAuthors(authors?: string): string[] { .split(",") .map((author: string) => author.trim()) } + +export function fullGdocToMinimalGdoc( + gdoc: OwidGdocBaseInterface +): OwidGdocMinimalPostInterface { + return { + id: gdoc.id, + title: gdoc.content.title || "", + slug: gdoc.slug, + authors: gdoc.content.authors, + publishedAt: gdoc.publishedAt ? formatDate(gdoc.publishedAt) : "", + published: gdoc.published, + subtitle: gdoc.content.subtitle || "", + excerpt: gdoc.content.excerpt || "", + type: gdoc.content.type || OwidGdocType.Article, + "featured-image": gdoc.content["featured-image"], + } +} diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 887c5b3f86c..b87872d8707 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -44,7 +44,7 @@ export interface OwidGdocBaseInterface { revisionId: string | null publicationContext: OwidGdocPublicationContext breadcrumbs?: BreadcrumbItem[] | null - linkedDocuments?: Record + linkedDocuments?: Record linkedCharts?: Record imageMetadata?: Record relatedCharts?: RelatedChart[] @@ -54,7 +54,21 @@ export interface OwidGdocBaseInterface { export interface OwidGdocPostInterface extends OwidGdocBaseInterface { content: OwidGdocPostContent - linkedDocuments?: Record + linkedDocuments?: Record +} + +// Used for linkedDocuments attachments, instead of attaching the entire gdoc model +export interface OwidGdocMinimalPostInterface { + id: string + title: string + slug: string + authors: string[] + publishedAt: string + published: boolean + subtitle: string + excerpt: string + type: OwidGdocType + "featured-image"?: string } export interface OwidGdocDataInsightContent { @@ -70,7 +84,7 @@ export const DATA_INSIGHTS_INDEX_PAGE_SIZE = 20 export interface OwidGdocDataInsightInterface extends OwidGdocBaseInterface { content: OwidGdocDataInsightContent - linkedDocuments?: Record + linkedDocuments?: Record latestDataInsights?: MinimalDataInsightInterface[] tags?: Tag[] } diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 0ae9db4c12d..5dcdd1531d9 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -261,6 +261,7 @@ export { type OwidGdocBaseInterface, type OwidGdocPostContent, type OwidGdocPostInterface, + type OwidGdocMinimalPostInterface, type OwidGdocDataInsightContent, type OwidGdocDataInsightInterface, type MinimalDataInsightInterface, diff --git a/site/DataInsightsIndexPageContent.tsx b/site/DataInsightsIndexPageContent.tsx index f9f4205bce2..481e9a222b8 100644 --- a/site/DataInsightsIndexPageContent.tsx +++ b/site/DataInsightsIndexPageContent.tsx @@ -4,7 +4,7 @@ import ReactDOM from "react-dom" import { ImageMetadata, LinkedChart, - OwidGdocPostInterface, + OwidGdocMinimalPostInterface, merge, } from "@ourworldindata/utils" import { DataInsightBody, indexToIdMap } from "./gdocs/pages/DataInsight.js" @@ -93,7 +93,10 @@ export const DataInsightsIndexPageContent = ( { imageMetadata: {} as Record, linkedCharts: {} as Record, - linkedDocuments: {} as Record, + linkedDocuments: {} as Record< + string, + OwidGdocMinimalPostInterface + >, } ) return ( diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index d0ee5896664..7adacef2069 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -2,7 +2,6 @@ import React, { createContext } from "react" import ReactDOM from "react-dom" import { LinkedChart, - OwidGdocPostInterface, getOwidGdocFromJSON, ImageMetadata, RelatedChart, @@ -10,6 +9,7 @@ import { OwidGdocType, OwidGdoc as OwidGdocInterface, MinimalDataInsightInterface, + OwidGdocMinimalPostInterface, } from "@ourworldindata/utils" import { DebugProvider } from "./DebugContext.js" import { match, P } from "ts-pattern" @@ -19,7 +19,7 @@ import { Fragment } from "./pages/Fragment.js" export const AttachmentsContext = createContext<{ linkedCharts: Record - linkedDocuments: Record + linkedDocuments: Record imageMetadata: Record relatedCharts: RelatedChart[] latestDataInsights?: MinimalDataInsightInterface[] diff --git a/site/gdocs/components/ProminentLink.tsx b/site/gdocs/components/ProminentLink.tsx index 57017dec518..dd088085f15 100644 --- a/site/gdocs/components/ProminentLink.tsx +++ b/site/gdocs/components/ProminentLink.tsx @@ -47,9 +47,9 @@ export const ProminentLink = (props: { let thumbnail: string | undefined = props.thumbnail if (linkType === "gdoc") { href = `/${linkedDocument?.slug}` - title = title ?? linkedDocument?.content.title - description = description ?? linkedDocument?.content.excerpt - thumbnail = thumbnail ?? linkedDocument?.content["featured-image"] + title = title ?? linkedDocument?.title + description = description ?? linkedDocument?.excerpt + thumbnail = thumbnail ?? linkedDocument?.["featured-image"] } else if (linkType === "grapher" || linkType === "explorer") { href = `${linkedChart?.resolvedUrl}` title = title ?? linkedChart?.title diff --git a/site/gdocs/components/Recirc.tsx b/site/gdocs/components/Recirc.tsx index d6b8b256a4f..91708182343 100644 --- a/site/gdocs/components/Recirc.tsx +++ b/site/gdocs/components/Recirc.tsx @@ -10,14 +10,12 @@ function RecircItem({ url }: { url: string }) { return (