🔨 introduce compact OwidGdocIndexItem for gdocs overview page

adminSiteClient/GdocsIndexPage.tsx

@@ -23,6 +23,7 @@ import {
     spansToUnformattedPlainText,
     OwidGdoc,
     checkIsGdocPost,
+    OwidGdocIndexItem,
 } from "@ourworldindata/utils"
 import { Route, RouteComponentProps } from "react-router-dom"
 import { Link } from "./Link.js"
@@ -139,7 +140,7 @@ export class GdocsIndexPage extends React.Component<GdocsMatchProps> {
         return this.context?.availableTags || []
     }
 
-    @computed get allGdocsToShow(): OwidGdoc[] {
+    @computed get allGdocsToShow(): OwidGdocIndexItem[] {
         const { searchWords, context } = this
         if (!context) return []
 
@@ -152,18 +153,18 @@ export class GdocsIndexPage extends React.Component<GdocsMatchProps> {
             ? context.gdocs.filter(
                   (gdoc) =>
                       // don't filter docs with no type set
-                      !gdoc.content.type || !!this.filters[gdoc.content.type]
+                      !gdoc.type || !!this.filters[gdoc.type]
               )
             : context.gdocs
 
         if (searchWords.length > 0) {
             const filterFn = filterFunctionForSearchWords(
                 searchWords,
-                (gdoc: OwidGdoc) => {
+                (gdoc: OwidGdocIndexItem) => {
                     const properties = [
-                        gdoc.content.title,
+                        gdoc.title,
                         gdoc.slug,
-                        gdoc.content.authors?.join(" "),
+                        gdoc.authors?.join(" "),
                         gdoc.tags?.map(({ name }) => name).join(" "),
                         gdoc.id,
                     ]
@@ -228,17 +229,16 @@ export class GdocsIndexPage extends React.Component<GdocsMatchProps> {
                         <div
                             key={gdoc.id}
                             className={cx(`gdoc-index-item`, {
-                                [`gdoc-index-item__${gdoc.content.type}`]:
-                                    gdoc.content.type,
+                                [`gdoc-index-item__${gdoc.type}`]: gdoc.type,
                             })}
                         >
                             <div className="gdoc-index-item__content">
-                                {gdoc.content.type ? (
+                                {gdoc.type ? (
                                     <span
                                         className="gdoc-index-item__type-icon"
-                                        title={gdoc.content.type}
+                                        title={gdoc.type}
                                     >
-                                        {iconGdocTypeMap[gdoc.content.type]}
+                                        {iconGdocTypeMap[gdoc.type]}
                                     </span>
                                 ) : null}
                                 <Link
@@ -248,19 +248,19 @@ export class GdocsIndexPage extends React.Component<GdocsMatchProps> {
                                         className="gdoc-index-item__title"
                                         title="Preview article"
                                     >
-                                        {gdoc.content.title || "Untitled"}
+                                        {gdoc.title || "Untitled"}
                                     </h5>
                                 </Link>
                                 <GdocsEditLink gdocId={gdoc.id} />
                                 <p className="gdoc-index-item__byline">
-                                    {gdoc.content.authors?.join(", ")}
+                                    {gdoc.authors?.join(", ")}
                                 </p>
                                 <span className="gdoc-index-item__tags">
-                                    {gdoc.content.type &&
+                                    {gdoc.type &&
                                     ![
                                         OwidGdocType.Fragment,
                                         OwidGdocType.AboutPage,
-                                    ].includes(gdoc.content.type) &&
+                                    ].includes(gdoc.type) &&
                                     gdoc.tags ? (
                                         <EditableTags
                                             tags={gdoc.tags}
@@ -286,8 +286,7 @@ export class GdocsIndexPage extends React.Component<GdocsMatchProps> {
                                                 : undefined
                                         }
                                         href={
-                                            gdoc.content.type !==
-                                            OwidGdocType.Fragment
+                                            gdoc.type !== OwidGdocType.Fragment
                                                 ? `${BAKED_BASE_URL}/${gdoc.slug}`
                                                 : undefined
                                         }

adminSiteClient/GdocsPreviewPage.tsx

@@ -40,6 +40,7 @@ import { openSuccessNotification } from "./gdocsNotifications.js"
 import { GdocsDiffButton } from "./GdocsDiffButton.js"
 import { GdocsDiff } from "./GdocsDiff.js"
 import { BAKED_BASE_URL } from "../settings/clientSettings.js"
+import { extractGdocIndexItem } from "@ourworldindata/types/dist/gdocTypes/Gdoc.js"
 
 export const GdocsPreviewPage = ({ match, history }: GdocsMatchProps) => {
     const { id } = match.params

adminSiteClient/GdocsStore.tsx

@@ -6,9 +6,11 @@ import {
     DbChartTagJoin,
     OwidGdoc,
     DbPlainTag,
+    OwidGdocIndexItem,
 } from "@ourworldindata/utils"
 import { AdminAppContext } from "./AdminAppContext.js"
 import { Admin } from "./Admin.js"
+import { extractGdocIndexItem } from "@ourworldindata/types"
 
 /**
  * This was originally a MobX data domain store (see
@@ -18,7 +20,7 @@ import { Admin } from "./Admin.js"
  * Today, this store acts as CRUD proxy for requests to API endpoints.
  */
 export class GdocsStore {
-    @observable gdocs: OwidGdoc[] = []
+    @observable gdocs: OwidGdocIndexItem[] = []
     @observable availableTags: DbChartTagJoin[] = []
     admin: Admin
 
@@ -33,9 +35,13 @@ export class GdocsStore {
 
     @action
     async update(gdoc: OwidGdoc): Promise<OwidGdoc> {
-        return this.admin
+        const item: OwidGdoc = await this.admin
             .requestJSON<OwidGdocJSON>(`/api/gdocs/${gdoc.id}`, gdoc, "PUT")
             .then(getOwidGdocFromJSON)
+        const indexItem = extractGdocIndexItem(gdoc)
+        const gdocToUpdateIndex = this.gdocs.findIndex((g) => g.id === gdoc.id)
+        if (gdocToUpdateIndex >= 0) this.gdocs[gdocToUpdateIndex] = indexItem
+        return item
     }
 
     @action
@@ -62,7 +68,9 @@ export class GdocsStore {
 
     @action
     async fetchGdocs() {
-        const gdocs = (await this.admin.getJSON("/api/gdocs")) as OwidGdoc[]
+        const gdocs = (await this.admin.getJSON(
+            "/api/gdocs"
+        )) as OwidGdocIndexItem[]
         this.gdocs = gdocs
     }
 
@@ -73,7 +81,7 @@ export class GdocsStore {
     }
 
     @action
-    async updateTags(gdoc: OwidGdoc, tags: DbPlainTag[]) {
+    async updateTags(gdoc: OwidGdocIndexItem, tags: DbPlainTag[]) {
         const json = await this.admin.requestJSON(
             `/api/gdocs/${gdoc.id}/setTags`,
             { tagIds: tags.map((t) => t.id) },

adminSiteServer/apiRouter.ts

@@ -126,6 +126,7 @@ import { getPublishedLinksTo } from "../db/model/Link.js"
 import {
     createGdocAndInsertIntoDb,
     gdocFromJSON,
+    getAllGdocIndexItemsOrderedByUpdatedAt,
     getAndLoadGdocById,
     getGdocBaseObjectById,
     loadGdocFromGdocBase,
@@ -2540,16 +2541,9 @@ apiRouter.put("/deploy", async (req, res) => {
 
 // 2024-03-10 Daniel: This route seems a bit insane - returning all gdocs in full would be transmitting something like 40MB by now.
 //                    I assume this is a leftover that can be deleted.
-// apiRouter.get("/gdocs", async () => {
-//     // orderBy was leading to a sort buffer overflow (ER_OUT_OF_SORTMEMORY) with MySQL's default sort_buffer_size
-//     // when the posts_gdocs table got larger than 9MB, so we sort in memory
-//     return GdocPost.find({ relations: ["tags"] }).then((gdocs) =>
-//         gdocs.sort((a, b) => {
-//             if (!a.updatedAt || !b.updatedAt) return 0
-//             return b.updatedAt.getTime() - a.updatedAt.getTime()
-//         })
-//     )
-// })
+getRouteWithROTransaction(apiRouter, "/gdocs", async (req, res, trx) => {
+    return getAllGdocIndexItemsOrderedByUpdatedAt(trx)
+})
 
 getRouteNonIdempotentWithRWTransaction(
     apiRouter,

db/model/Gdoc/GdocBase.ts

@@ -868,7 +868,7 @@ export async function getMinimalGdocBaseObjectsByIds(
                 content ->> '$.type' as type,
                 content ->> '$."featured-image"' as "featured-image"
             FROM posts_gdocs
-            WHERE id in :ids`,
+            WHERE id in (:ids)`,
         { ids }
     )
     return rows.map((row) => {

db/model/Gdoc/GdocFactory.ts

@@ -9,12 +9,14 @@ import {
     OwidEnrichedGdocBlock,
     OwidGdoc,
     OwidGdocBaseInterface,
+    OwidGdocIndexItem,
     OwidGdocMinimalPostInterface,
     OwidGdocPublicationContext,
     OwidGdocType,
     PostsGdocsTableName,
     PostsGdocsXTagsTableName,
     checkIsOwidGdocType,
+    extractGdocIndexItem,
     formatDate,
     parsePostsGdocsRow,
     serializePostsGdocsRow,
@@ -463,3 +465,31 @@ export async function upsertGdoc(
         .onConflict("id")
         .merge()
 }
+
+// TODO:
+export async function getAllGdocIndexItemsOrderedByUpdatedAt(
+    knex: KnexReadonlyTransaction
+): Promise<OwidGdocIndexItem[]> {
+    // Old note from Ike for somewhat different code that might still be relevant:
+    // orderBy was leading to a sort buffer overflow (ER_OUT_OF_SORTMEMORY) with MySQL's default sort_buffer_size
+    // when the posts_gdocs table got larger than 9MB, so we sort in memory
+    const gdocs: DbRawPostGdoc[] = await knex
+        .table<DbRawPostGdoc>(PostsGdocsTableName)
+        .orderBy("updatedAt", "desc")
+    const tagsForGdocs = await knexRaw<DbPlainTag & Pick<DbRawPostGdoc, "id">>(
+        knex,
+        `-- sql
+        SELECT gt.gdocId as gdocId, tags.*
+        FROM tags
+        JOIN posts_gdocs_x_tags gt ON gt.tagId = tags.id
+        WHERE gt.gdocId in (:ids)`,
+        { ids: gdocs.map((gdoc) => gdoc.id) }
+    )
+    const groupedTags = groupBy(tagsForGdocs, "gdocId")
+    return gdocs.map((gdoc) =>
+        extractGdocIndexItem({
+            ...parsePostsGdocsRow(gdoc),
+            tags: groupedTags[gdoc.id] ? groupedTags[gdoc.id] : null,
+        })
+    )
+}

packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts

@@ -92,6 +92,27 @@ export interface OwidGdocMinimalPostInterface {
     "featured-image"?: string // used in prominent links and research & writing block
 }
 
+export type OwidGdocIndexItem = Pick<
+    OwidGdocBaseInterface,
+    "id" | "slug" | "tags" | "published" | "publishedAt"
+> &
+    Pick<OwidGdocContent, "title" | "authors" | "type">
+
+export function extractGdocIndexItem(
+    gdoc: OwidGdocBaseInterface
+): OwidGdocIndexItem {
+    return {
+        id: gdoc.id,
+        slug: gdoc.slug,
+        tags: gdoc.tags ?? [],
+        published: gdoc.published,
+        publishedAt: gdoc.publishedAt,
+        title: gdoc.content.title ?? "",
+        authors: gdoc.content.authors,
+        type: gdoc.content.type,
+    }
+}
+
 export interface OwidGdocDataInsightContent {
     title: string
     authors: string[]

packages/@ourworldindata/types/src/index.ts

@@ -293,6 +293,8 @@ export {
     type LinkedIndicator,
     DYNAMIC_COLLECTION_PAGE_CONTAINER_ID,
     type OwidGdocContent,
+    type OwidGdocIndexItem,
+    extractGdocIndexItem,
 } from "./gdocTypes/Gdoc.js"
 
 export {

packages/@ourworldindata/utils/src/Util.ts

@@ -169,6 +169,7 @@ import {
     UserCountryInformation,
     Time,
     TimeBound,
+    OwidGdocIndexItem,
 } from "@ourworldindata/types"
 import { PointVector } from "./PointVector.js"
 import React from "react"