From efdb77e13f4cd77e4d90c1d3b30de778569ab276 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Wed, 13 Mar 2024 12:35:28 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20introduce=20compact=20OwidGdocIn?= =?UTF-8?q?dexItem=20for=20gdocs=20overview=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteClient/GdocsIndexPage.tsx | 31 +++++++++---------- adminSiteClient/GdocsPreviewPage.tsx | 1 + adminSiteClient/GdocsStore.tsx | 16 +++++++--- adminSiteServer/apiRouter.ts | 14 +++------ db/model/Gdoc/GdocBase.ts | 2 +- db/model/Gdoc/GdocFactory.ts | 30 ++++++++++++++++++ .../types/src/gdocTypes/Gdoc.ts | 21 +++++++++++++ packages/@ourworldindata/types/src/index.ts | 2 ++ packages/@ourworldindata/utils/src/Util.ts | 1 + 9 files changed, 87 insertions(+), 31 deletions(-) diff --git a/adminSiteClient/GdocsIndexPage.tsx b/adminSiteClient/GdocsIndexPage.tsx index 48d448b1c95..ede8a2fee27 100644 --- a/adminSiteClient/GdocsIndexPage.tsx +++ b/adminSiteClient/GdocsIndexPage.tsx @@ -23,6 +23,7 @@ import { spansToUnformattedPlainText, OwidGdoc, checkIsGdocPost, + OwidGdocIndexItem, } from "@ourworldindata/utils" import { Route, RouteComponentProps } from "react-router-dom" import { Link } from "./Link.js" @@ -139,7 +140,7 @@ export class GdocsIndexPage extends React.Component { return this.context?.availableTags || [] } - @computed get allGdocsToShow(): OwidGdoc[] { + @computed get allGdocsToShow(): OwidGdocIndexItem[] { const { searchWords, context } = this if (!context) return [] @@ -152,18 +153,18 @@ export class GdocsIndexPage extends React.Component { ? context.gdocs.filter( (gdoc) => // don't filter docs with no type set - !gdoc.content.type || !!this.filters[gdoc.content.type] + !gdoc.type || !!this.filters[gdoc.type] ) : context.gdocs if (searchWords.length > 0) { const filterFn = filterFunctionForSearchWords( searchWords, - (gdoc: OwidGdoc) => { + (gdoc: OwidGdocIndexItem) => { const properties = [ - gdoc.content.title, + gdoc.title, gdoc.slug, - gdoc.content.authors?.join(" "), + gdoc.authors?.join(" "), gdoc.tags?.map(({ name }) => name).join(" "), gdoc.id, ] @@ -228,17 +229,16 @@ export class GdocsIndexPage extends React.Component {
- {gdoc.content.type ? ( + {gdoc.type ? ( - {iconGdocTypeMap[gdoc.content.type]} + {iconGdocTypeMap[gdoc.type]} ) : null} { className="gdoc-index-item__title" title="Preview article" > - {gdoc.content.title || "Untitled"} + {gdoc.title || "Untitled"}

- {gdoc.content.authors?.join(", ")} + {gdoc.authors?.join(", ")}

- {gdoc.content.type && + {gdoc.type && ![ OwidGdocType.Fragment, OwidGdocType.AboutPage, - ].includes(gdoc.content.type) && + ].includes(gdoc.type) && gdoc.tags ? ( { : undefined } href={ - gdoc.content.type !== - OwidGdocType.Fragment + gdoc.type !== OwidGdocType.Fragment ? `${BAKED_BASE_URL}/${gdoc.slug}` : undefined } diff --git a/adminSiteClient/GdocsPreviewPage.tsx b/adminSiteClient/GdocsPreviewPage.tsx index c2948685ac5..489b519a8a1 100644 --- a/adminSiteClient/GdocsPreviewPage.tsx +++ b/adminSiteClient/GdocsPreviewPage.tsx @@ -40,6 +40,7 @@ import { openSuccessNotification } from "./gdocsNotifications.js" import { GdocsDiffButton } from "./GdocsDiffButton.js" import { GdocsDiff } from "./GdocsDiff.js" import { BAKED_BASE_URL } from "../settings/clientSettings.js" +import { extractGdocIndexItem } from "@ourworldindata/types/dist/gdocTypes/Gdoc.js" export const GdocsPreviewPage = ({ match, history }: GdocsMatchProps) => { const { id } = match.params diff --git a/adminSiteClient/GdocsStore.tsx b/adminSiteClient/GdocsStore.tsx index afb28666069..9f90ac04b54 100644 --- a/adminSiteClient/GdocsStore.tsx +++ b/adminSiteClient/GdocsStore.tsx @@ -6,9 +6,11 @@ import { DbChartTagJoin, OwidGdoc, DbPlainTag, + OwidGdocIndexItem, } from "@ourworldindata/utils" import { AdminAppContext } from "./AdminAppContext.js" import { Admin } from "./Admin.js" +import { extractGdocIndexItem } from "@ourworldindata/types" /** * This was originally a MobX data domain store (see @@ -18,7 +20,7 @@ import { Admin } from "./Admin.js" * Today, this store acts as CRUD proxy for requests to API endpoints. */ export class GdocsStore { - @observable gdocs: OwidGdoc[] = [] + @observable gdocs: OwidGdocIndexItem[] = [] @observable availableTags: DbChartTagJoin[] = [] admin: Admin @@ -33,9 +35,13 @@ export class GdocsStore { @action async update(gdoc: OwidGdoc): Promise { - return this.admin + const item: OwidGdoc = await this.admin .requestJSON(`/api/gdocs/${gdoc.id}`, gdoc, "PUT") .then(getOwidGdocFromJSON) + const indexItem = extractGdocIndexItem(gdoc) + const gdocToUpdateIndex = this.gdocs.findIndex((g) => g.id === gdoc.id) + if (gdocToUpdateIndex >= 0) this.gdocs[gdocToUpdateIndex] = indexItem + return item } @action @@ -62,7 +68,9 @@ export class GdocsStore { @action async fetchGdocs() { - const gdocs = (await this.admin.getJSON("/api/gdocs")) as OwidGdoc[] + const gdocs = (await this.admin.getJSON( + "/api/gdocs" + )) as OwidGdocIndexItem[] this.gdocs = gdocs } @@ -73,7 +81,7 @@ export class GdocsStore { } @action - async updateTags(gdoc: OwidGdoc, tags: DbPlainTag[]) { + async updateTags(gdoc: OwidGdocIndexItem, tags: DbPlainTag[]) { const json = await this.admin.requestJSON( `/api/gdocs/${gdoc.id}/setTags`, { tagIds: tags.map((t) => t.id) }, diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 62d8c4080f8..4c960434016 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -126,6 +126,7 @@ import { getPublishedLinksTo } from "../db/model/Link.js" import { createGdocAndInsertIntoDb, gdocFromJSON, + getAllGdocIndexItemsOrderedByUpdatedAt, getAndLoadGdocById, getGdocBaseObjectById, loadGdocFromGdocBase, @@ -2540,16 +2541,9 @@ apiRouter.put("/deploy", async (req, res) => { // 2024-03-10 Daniel: This route seems a bit insane - returning all gdocs in full would be transmitting something like 40MB by now. // I assume this is a leftover that can be deleted. -// apiRouter.get("/gdocs", async () => { -// // orderBy was leading to a sort buffer overflow (ER_OUT_OF_SORTMEMORY) with MySQL's default sort_buffer_size -// // when the posts_gdocs table got larger than 9MB, so we sort in memory -// return GdocPost.find({ relations: ["tags"] }).then((gdocs) => -// gdocs.sort((a, b) => { -// if (!a.updatedAt || !b.updatedAt) return 0 -// return b.updatedAt.getTime() - a.updatedAt.getTime() -// }) -// ) -// }) +getRouteWithROTransaction(apiRouter, "/gdocs", async (req, res, trx) => { + return getAllGdocIndexItemsOrderedByUpdatedAt(trx) +}) getRouteNonIdempotentWithRWTransaction( apiRouter, diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 4aa94a19d61..3e258eebb9d 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -868,7 +868,7 @@ export async function getMinimalGdocBaseObjectsByIds( content ->> '$.type' as type, content ->> '$."featured-image"' as "featured-image" FROM posts_gdocs - WHERE id in :ids`, + WHERE id in (:ids)`, { ids } ) return rows.map((row) => { diff --git a/db/model/Gdoc/GdocFactory.ts b/db/model/Gdoc/GdocFactory.ts index 5a368c5441b..5875ae79dff 100644 --- a/db/model/Gdoc/GdocFactory.ts +++ b/db/model/Gdoc/GdocFactory.ts @@ -9,12 +9,14 @@ import { OwidEnrichedGdocBlock, OwidGdoc, OwidGdocBaseInterface, + OwidGdocIndexItem, OwidGdocMinimalPostInterface, OwidGdocPublicationContext, OwidGdocType, PostsGdocsTableName, PostsGdocsXTagsTableName, checkIsOwidGdocType, + extractGdocIndexItem, formatDate, parsePostsGdocsRow, serializePostsGdocsRow, @@ -463,3 +465,31 @@ export async function upsertGdoc( .onConflict("id") .merge() } + +// TODO: +export async function getAllGdocIndexItemsOrderedByUpdatedAt( + knex: KnexReadonlyTransaction +): Promise { + // Old note from Ike for somewhat different code that might still be relevant: + // orderBy was leading to a sort buffer overflow (ER_OUT_OF_SORTMEMORY) with MySQL's default sort_buffer_size + // when the posts_gdocs table got larger than 9MB, so we sort in memory + const gdocs: DbRawPostGdoc[] = await knex + .table(PostsGdocsTableName) + .orderBy("updatedAt", "desc") + const tagsForGdocs = await knexRaw>( + knex, + `-- sql + SELECT gt.gdocId as gdocId, tags.* + FROM tags + JOIN posts_gdocs_x_tags gt ON gt.tagId = tags.id + WHERE gt.gdocId in (:ids)`, + { ids: gdocs.map((gdoc) => gdoc.id) } + ) + const groupedTags = groupBy(tagsForGdocs, "gdocId") + return gdocs.map((gdoc) => + extractGdocIndexItem({ + ...parsePostsGdocsRow(gdoc), + tags: groupedTags[gdoc.id] ? groupedTags[gdoc.id] : null, + }) + ) +} diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 19d57d7ddf2..b8e72060f0f 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -92,6 +92,27 @@ export interface OwidGdocMinimalPostInterface { "featured-image"?: string // used in prominent links and research & writing block } +export type OwidGdocIndexItem = Pick< + OwidGdocBaseInterface, + "id" | "slug" | "tags" | "published" | "publishedAt" +> & + Pick + +export function extractGdocIndexItem( + gdoc: OwidGdocBaseInterface +): OwidGdocIndexItem { + return { + id: gdoc.id, + slug: gdoc.slug, + tags: gdoc.tags ?? [], + published: gdoc.published, + publishedAt: gdoc.publishedAt, + title: gdoc.content.title ?? "", + authors: gdoc.content.authors, + type: gdoc.content.type, + } +} + export interface OwidGdocDataInsightContent { title: string authors: string[] diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index b3ff8e6b069..28afedfa90b 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -293,6 +293,8 @@ export { type LinkedIndicator, DYNAMIC_COLLECTION_PAGE_CONTAINER_ID, type OwidGdocContent, + type OwidGdocIndexItem, + extractGdocIndexItem, } from "./gdocTypes/Gdoc.js" export { diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 7c43a24fa08..db0145d97f8 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -169,6 +169,7 @@ import { UserCountryInformation, Time, TimeBound, + OwidGdocIndexItem, } from "@ourworldindata/types" import { PointVector } from "./PointVector.js" import React from "react"