✨ image store caching polish

baker/algolia/algoliaUtils.tsx

@@ -164,17 +164,10 @@ function generateGdocRecords(
                     importance: 3,
                 })
             )
-            .with(OwidGdocType.Fragment, () => ({
+            .with(P.union(OwidGdocType.Fragment, undefined), () => ({
                 type: "other" as const,
                 importance: 0,
             }))
-            .with(undefined, () => {
-                console.error(
-                    `Failed indexing gdoc post ${gdoc.id} (No type)`,
-                    gdoc
-                )
-                return { type: "other" as const, importance: 0 }
-            })
             .exhaustive()
     }
 

db/db.ts

@@ -322,7 +322,7 @@ export const getHomepageId = (
     ).then((result) => result?.id)
 }
 
-export const getImagesMetadataByFilenames = async (
+export const getImageMetadataByFilenames = async (
     knex: KnexReadonlyTransaction,
     filenames: string[]
 ): Promise<Record<string, ImageMetadata>> => {

db/model/Gdoc/GdocBase.ts

@@ -691,7 +691,7 @@ export class GdocBase implements OwidGdocBaseInterface {
 
         if (!imagesFilenames.length) return
 
-        const imageMetadata = await db.getImagesMetadataByFilenames(
+        const imageMetadata = await db.getImageMetadataByFilenames(
             knex,
             imagesFilenames
         )

db/model/Gdoc/GdocFactory.ts

@@ -36,7 +36,7 @@ import {
     KnexReadonlyTransaction,
     knexRaw,
     KnexReadWriteTransaction,
-    getImagesMetadataByFilenames,
+    getImageMetadataByFilenames,
     getPublishedGdocPosts,
 } from "../../db.js"
 import { enrichedBlocksToMarkdown } from "./enrichedToMarkdown.js"
@@ -347,10 +347,7 @@ export async function loadGdocFromGdocBase(
         await fetchImagesFromDriveAndSyncToS3(knex, gdoc.filenames)
     }
 
-    const start = Date.now()
     await gdoc.loadState(knex)
-    const end = Date.now()
-    console.log(`Loaded state for ${gdoc.id} in ${end - start}ms`)
 
     return gdoc
 }
@@ -604,7 +601,7 @@ export async function addImagesToContentGraph(
     // Includes fragments so that images in data pages are
     // synced to S3 and ultimately baked in bakeDriveImages().
     if (filenames.length && gdoc.published) {
-        const images = await getImagesMetadataByFilenames(trx, filenames)
+        const images = await getImageMetadataByFilenames(trx, filenames)
         const gdocXImagesToInsert: DbInsertPostGdocXImage[] = []
         for (const filename in images) {
             const image = images[filename] as DbEnrichedImage

db/model/Image.ts

@@ -19,7 +19,6 @@ import {
     serializeImageRow,
     ImagesTableName,
     merge,
-    pick,
     excludeUndefined,
 } from "@ourworldindata/utils"
 import { OwidGoogleAuth } from "../OwidGoogleAuth.js"
@@ -34,22 +33,17 @@ import {
 } from "../../settings/serverSettings.js"
 import { KnexReadWriteTransaction, KnexReadonlyTransaction } from "../db.js"
 import { at } from "lodash"
+import { ENV } from "../../settings/clientSettings.js"
 
 class ImageStore {
     images: Record<string, ImageMetadata> | undefined
     hasInitialized: boolean = false
     isFetchingFromGoogleDrive: boolean = false
-
-    constructor() {
-        setInterval(
-            async () => {
-                await this.fetchImageMetadata([])
-            },
-            60 * 60 * 1000
-        )
-    }
+    lastFetchTime: number | undefined
 
     async init(): Promise<void> {
+        // don't prefetch every image in development mode, wait for a specific request first
+        if (ENV === "development") return
         // only one init at a time
         while (this.isFetchingFromGoogleDrive) {
             await new Promise((resolve) => setTimeout(resolve, 100))
@@ -65,6 +59,16 @@ class ImageStore {
     async fetchImageMetadata(
         filenames: string[]
     ): Promise<Record<string, ImageMetadata | undefined>> {
+        if (
+            this.images &&
+            this.lastFetchTime &&
+            Date.now() - this.lastFetchTime < 60 * 1000
+        ) {
+            console.log(
+                `Skipping image metadata refresh since last fetch was less than a minute ago`
+            )
+            return this.images
+        }
         this.isFetchingFromGoogleDrive = true
         try {
             console.log(
@@ -157,6 +161,7 @@ class ImageStore {
             throw error
         } finally {
             this.isFetchingFromGoogleDrive = false
+            this.lastFetchTime = Date.now()
         }
     }
 
@@ -173,11 +178,11 @@ class ImageStore {
     }
 }
 
-export const imageStore = new ImageStore()
+const _imageStore = new ImageStore()
 
 export async function getImageStore(): Promise<ImageStore> {
-    await imageStore.init()
-    return imageStore
+    await _imageStore.init()
+    return _imageStore
 }
 
 export const s3Client = new S3Client({
@@ -236,12 +241,14 @@ export class Image implements ImageMetadata {
                 if (
                     stored.updatedAt !== fresh.updatedAt ||
                     stored.defaultAlt !== fresh.defaultAlt ||
-                    stored.originalWidth !== fresh.originalWidth
+                    stored.originalWidth !== fresh.originalWidth ||
+                    stored.originalHeight !== fresh.originalHeight
                 ) {
                     await fresh.fetchFromDriveAndUploadToS3()
                     stored.updatedAt = fresh.updatedAt
                     stored.defaultAlt = fresh.defaultAlt
                     stored.originalWidth = fresh.originalWidth
+                    stored.originalHeight = fresh.originalHeight
                     await updateImage(knex, stored.id, {
                         updatedAt: fresh.updatedAt,
                         defaultAlt: fresh.defaultAlt,

devTools/updateImageHeights/update-image-heights.ts

@@ -1,9 +1,10 @@
-import { imageStore } from "../../db/model/Image.js"
+import { getImageStore } from "../../db/model/Image.js"
 import * as db from "../../db/db.js"
 import * as lodash from "lodash"
 import { exit } from "../../db/cleanup.js"
 
 async function updateImageHeights() {
+    const imageStore = await getImageStore()
     const transaction = await db.knexInstance().transaction()
     const filenames = await db
         .knexRaw<{ filename: string }>(