Revert "Merge pull request #3496 from owid/image-store-caching"

This reverts commit 041cdd2, reversing
changes made to 1976d05.

adminSiteServer/apiRouter.ts

@@ -135,7 +135,7 @@ import {
     getGdocBaseObjectById,
     setLinksForGdoc,
     setTagsForGdoc,
-    addImagesToContentGraph,
+    syncImagesAndAddToContentGraph,
     updateGdocContentOnly,
     upsertGdoc,
 } from "../db/model/Gdoc/GdocFactory.js"
@@ -2276,7 +2276,6 @@ getRouteNonIdempotentWithRWTransaction(
             | undefined
 
         try {
-            // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published
             const gdoc = await getAndLoadGdocById(trx, id, contentSource)
 
             if (!gdoc.published) {
@@ -2368,7 +2367,7 @@ putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => {
     const nextGdoc = gdocFromJSON(req.body)
     await nextGdoc.loadState(trx)
 
-    await addImagesToContentGraph(trx, nextGdoc)
+    await syncImagesAndAddToContentGraph(trx, nextGdoc)
 
     await setLinksForGdoc(
         trx,

baker/SiteBaker.tsx

@@ -1,7 +1,7 @@
 import fs from "fs-extra"
 import path from "path"
 import { glob } from "glob"
-import { keyBy, without, uniq, mapValues, pick, chunk } from "lodash"
+import { keyBy, without, uniq, mapValues, pick } from "lodash"
 import ProgressBar from "progress"
 import * as wpdb from "../db/wpdb.js"
 import * as db from "../db/db.js"
@@ -100,10 +100,7 @@ import {
     getVariableMetadata,
     getVariableOfDatapageIfApplicable,
 } from "../db/model/Variable.js"
-import {
-    gdocFromJSON,
-    getAllMinimalGdocBaseObjects,
-} from "../db/model/Gdoc/GdocFactory.js"
+import { getAllMinimalGdocBaseObjects } from "../db/model/Gdoc/GdocFactory.js"
 import { getBakePath } from "@ourworldindata/components"
 import { GdocAuthor } from "../db/model/Gdoc/GdocAuthor.js"
 import { DATA_INSIGHTS_ATOM_FEED_NAME } from "../site/gdocs/utils.js"
@@ -307,7 +304,6 @@ export class SiteBaker {
         picks?: [string[], string[], string[], string[]]
     ): Promise<PrefetchedAttachments> {
         if (!this._prefetchedAttachmentsCache) {
-            console.log("Prefetching attachments")
             const publishedGdocs = await getAllMinimalGdocBaseObjects(knex)
             const publishedGdocsDictionary = keyBy(publishedGdocs, "id")
 
@@ -332,31 +328,23 @@ export class SiteBaker {
 
             // Includes redirects
             const publishedChartsRaw = await mapSlugsToConfigs(knex)
-            const publishedCharts: LinkedChart[] = []
-
-            for (const publishedChartsRawChunk of chunk(
-                publishedChartsRaw,
-                20
-            )) {
-                await Promise.all(
-                    publishedChartsRawChunk.map(async (chart) => {
-                        const tab = chart.config.tab ?? GrapherTabOption.chart
-                        const datapageIndicator =
-                            await getVariableOfDatapageIfApplicable(
-                                chart.config
-                            )
-                        publishedCharts.push({
-                            originalSlug: chart.slug,
-                            resolvedUrl: `${BAKED_GRAPHER_URL}/${chart.config.slug}`,
-                            tab,
-                            title: chart.config.title || "",
-                            thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${chart.config.slug}.svg`,
-                            indicatorId: datapageIndicator?.id,
-                            tags: [],
-                        })
-                    })
-                )
-            }
+            const publishedCharts: LinkedChart[] = await Promise.all(
+                publishedChartsRaw.map(async (chart) => {
+                    const tab = chart.config.tab ?? GrapherTabOption.chart
+                    const datapageIndicator =
+                        await getVariableOfDatapageIfApplicable(chart.config)
+                    return {
+                        originalSlug: chart.slug,
+                        resolvedUrl: `${BAKED_GRAPHER_URL}/${chart.config.slug}`,
+                        tab,
+                        queryString: "",
+                        title: chart.config.title || "",
+                        thumbnail: `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${chart.config.slug}.svg`,
+                        indicatorId: datapageIndicator?.id,
+                        tags: [],
+                    }
+                })
+            )
             const publishedChartsBySlug = keyBy(publishedCharts, "originalSlug")
 
             const publishedChartsWithIndicatorIds = publishedCharts.filter(
@@ -498,10 +486,7 @@ export class SiteBaker {
     // TODO: this transaction is only RW because somewhere inside it we fetch images
     async bakeGDocPosts(knex: db.KnexReadWriteTransaction, slugs?: string[]) {
         if (!this.bakeSteps.has("gdocPosts")) return
-        // We don't need to load these as we prefetch all attachments
-        const publishedGdocs = await db
-            .getPublishedGdocPosts(knex)
-            .then((gdocs) => gdocs.map(gdocFromJSON))
+        const publishedGdocs = await GdocPost.getPublishedGdocPosts(knex)
 
         const gdocsToBake =
             slugs !== undefined
@@ -534,9 +519,7 @@ export class SiteBaker {
             publishedGdoc.linkedIndicators = attachments.linkedIndicators
 
             // this is a no-op if the gdoc doesn't have an all-chart block
-            if ("loadRelatedCharts" in publishedGdoc) {
-                await publishedGdoc.loadRelatedCharts(knex)
-            }
+            await publishedGdoc.loadRelatedCharts(knex)
 
             await publishedGdoc.validate(knex)
             if (

baker/algolia/algoliaUtils.tsx

@@ -23,6 +23,7 @@ import {
     SearchIndexName,
 } from "../../site/search/searchTypes.js"
 import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js"
+import { GdocPost } from "../../db/model/Gdoc/GdocPost.js"
 import { ArticleBlocks } from "../../site/gdocs/components/ArticleBlocks.js"
 import React from "react"
 import {
@@ -33,8 +34,6 @@ import {
 import { getIndexName } from "../../site/search/searchClient.js"
 import { ObjectWithObjectID } from "@algolia/client-search"
 import { SearchIndex } from "algoliasearch"
-import { match, P } from "ts-pattern"
-import { gdocFromJSON } from "../../db/model/Gdoc/GdocFactory.js"
 
 interface TypeAndImportance {
     type: PageType
@@ -147,27 +146,20 @@ function generateGdocRecords(
     const getPostTypeAndImportance = (
         gdoc: OwidGdocPostInterface
     ): TypeAndImportance => {
-        return match(gdoc.content.type)
-            .with(OwidGdocType.Article, () => ({
-                type: "article" as const,
-                importance: 0,
-            }))
-            .with(OwidGdocType.AboutPage, () => ({
-                type: "about" as const,
-                importance: 1,
-            }))
-            .with(
-                P.union(OwidGdocType.TopicPage, OwidGdocType.LinearTopicPage),
-                () => ({
-                    type: "topic" as const,
-                    importance: 3,
-                })
-            )
-            .with(P.union(OwidGdocType.Fragment, undefined), () => ({
-                type: "other" as const,
-                importance: 0,
-            }))
-            .exhaustive()
+        switch (gdoc.content.type) {
+            case OwidGdocType.TopicPage:
+                return { type: "topic", importance: 3 }
+            case OwidGdocType.LinearTopicPage:
+                return { type: "topic", importance: 3 }
+            case OwidGdocType.Fragment:
+                // this should not happen because we filter out fragments; but we want to have an exhaustive switch/case so we include it
+                return { type: "other", importance: 0 }
+            case OwidGdocType.AboutPage:
+                return { type: "about", importance: 0 }
+            case OwidGdocType.Article:
+            case undefined:
+                return { type: "article", importance: 0 }
+        }
     }
 
     const records: PageRecord[] = []
@@ -210,11 +202,9 @@ function generateGdocRecords(
 // Generate records for countries, WP posts (not including posts that have been succeeded by Gdocs equivalents), and Gdocs
 export const getPagesRecords = async (knex: db.KnexReadWriteTransaction) => {
     const pageviews = await getAnalyticsPageviewsByUrlObj(knex)
-    const gdocs = await db
-        .getPublishedGdocPosts(knex)
-        .then((gdocs) => gdocs.map(gdocFromJSON) as OwidGdocPostInterface[])
-
+    const gdocs = await GdocPost.getPublishedGdocPosts(knex)
     const publishedGdocsBySlug = keyBy(gdocs, "slug")
+    // TODO: the knex instance should be handed down as a parameter
     const slugsWithPublishedGdocsSuccessors =
         await db.getSlugsWithPublishedGdocsSuccessors(knex)
     const postsApi = await getPostsFromSnapshots(knex, undefined, (post) => {

baker/algolia/indexToAlgolia.tsx

@@ -25,8 +25,6 @@ const indexToAlgolia = async () => {
     await index.replaceAllObjects(records)
 
     await wpdb.singleton.end()
-
-    process.exit(0)
 }
 
 process.on("unhandledRejection", (e) => {

baker/sitemap.ts

@@ -15,6 +15,7 @@ import { countryProfileSpecs } from "../site/countryProfileProjects.js"
 import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js"
 import { EXPLORERS_ROUTE_FOLDER } from "../explorer/ExplorerConstants.js"
 import { ExplorerProgram } from "../explorer/ExplorerProgram.js"
+import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
 import { getPostsFromSnapshots } from "../db/model/Post.js"
 import { calculateDataInsightIndexPageCount } from "../db/model/Gdoc/gdocUtils.js"
 
@@ -73,7 +74,7 @@ export const makeSitemap = async (
         undefined,
         (postrow) => !alreadyPublishedViaGdocsSlugsSet.has(postrow.slug)
     )
-    const gdocPosts = await db.getPublishedGdocPosts(knex)
+    const gdocPosts = await GdocPost.getPublishedGdocPosts(knex)
 
     const publishedDataInsights = await db.getPublishedDataInsights(knex)
     const dataInsightFeedPageCount = calculateDataInsightIndexPageCount(

db/db.ts

@@ -10,12 +10,8 @@ import { registerExitHandler } from "./cleanup.js"
 import { keyBy } from "@ourworldindata/utils"
 import {
     DbChartTagJoin,
-    DbEnrichedPostGdoc,
-    DbRawPostGdoc,
-    ImageMetadata,
     MinimalDataInsightInterface,
     OwidGdocType,
-    parsePostsGdocsRow,
 } from "@ourworldindata/types"
 
 // Return the first match from a mysql query
@@ -314,50 +310,3 @@ export const getHomepageId = (
             AND published = TRUE`
     ).then((result) => result?.id)
 }
-
-export const getImageMetadataByFilenames = async (
-    knex: KnexReadonlyTransaction,
-    filenames: string[]
-): Promise<Record<string, ImageMetadata & { id: number }>> => {
-    if (filenames.length === 0) return {}
-    const rows = await knexRaw<ImageMetadata & { id: number }>(
-        knex,
-        `-- sql
-        SELECT
-            id,
-            googleId,
-            filename,
-            defaultAlt,
-            updatedAt,
-            originalWidth,
-            originalHeight
-        FROM
-            images
-        WHERE filename IN (?)`,
-        [filenames]
-    )
-    return keyBy(rows, "filename")
-}
-
-export const getPublishedGdocPosts = async (
-    knex: KnexReadonlyTransaction
-): Promise<DbEnrichedPostGdoc[]> => {
-    return knexRaw<DbRawPostGdoc>(
-        knex,
-        `-- sql
-            SELECT *
-            FROM posts_gdocs
-            WHERE published = 1
-            AND content ->> '$.type' IN (:types)
-            AND publishedAt <= NOW()
-            ORDER BY publishedAt DESC`,
-        {
-            types: [
-                OwidGdocType.Article,
-                OwidGdocType.LinearTopicPage,
-                OwidGdocType.TopicPage,
-                OwidGdocType.AboutPage,
-            ],
-        }
-    ).then((rows) => rows.map(parsePostsGdocsRow))
-}

db/model/Gdoc/GdocAuthor.ts

@@ -43,14 +43,16 @@ export class GdocAuthor extends GdocBase implements OwidGdocAuthorInterface {
         return blocks
     }
 
+    // TODO: this transaction is only RW because somewhere inside it we fetch images
     _loadSubclassAttachments = (
-        knex: db.KnexReadonlyTransaction
+        knex: db.KnexReadWriteTransaction
     ): Promise<void> => {
         return this.loadLatestWorkImages(knex)
     }
 
+    // TODO: this transaction is only RW because somewhere inside it we fetch images
     loadLatestWorkImages = async (
-        knex: db.KnexReadonlyTransaction
+        knex: db.KnexReadWriteTransaction
     ): Promise<void> => {
         if (!this.content.title) return
 
@@ -75,7 +77,7 @@ export class GdocAuthor extends GdocBase implements OwidGdocAuthorInterface {
         // Load the image metadata for the latest work images, including the
         // default featured image which is used as a fallback in the entire
         // research and writing block
-        return super.loadImageMetadataFromDB(knex, [
+        return super.loadImageMetadata(knex, [
             ...latestWorkImageFilenames,
             DEFAULT_GDOC_FEATURED_IMAGE,
         ])