🔨 render thumbnails in CF workers using chart configs from R2

.gitignore

@@ -43,3 +43,4 @@ dist/
 .nx/workspace-data
 .dev.vars
 **/tsup.config.bundled*.mjs
+cfstorage

adminSiteServer/apiRouter.ts

@@ -85,6 +85,7 @@ import {
     DbInsertUser,
     FlatTagGraph,
     DbRawChartConfig,
+    R2GrapherConfigDirectory,
 } from "@ourworldindata/types"
 import { uuidv7 } from "uuidv7"
 import {
@@ -158,7 +159,6 @@ import path from "path"
 import {
     deleteGrapherConfigFromR2,
     deleteGrapherConfigFromR2ByUUID,
-    R2GrapherConfigDirectory,
     saveGrapherConfigToR2,
     saveGrapherConfigToR2ByUUID,
     getMd5HashBase64,

adminSiteServer/chartConfigR2Helpers.ts

@@ -14,6 +14,7 @@ import {
     S3Client,
 } from "@aws-sdk/client-s3"
 import { Base64String, JsonError } from "@ourworldindata/utils"
+import { R2GrapherConfigDirectory } from "@ourworldindata/types"
 import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
 import { createHash } from "crypto"
 
@@ -25,10 +26,6 @@ export function getMd5HashBase64(data: string): Base64String {
         .update(data, "utf-8")
         .digest("base64") as Base64String
 }
-export enum R2GrapherConfigDirectory {
-    byUUID = "config/by-uuid",
-    publishedGrapherBySlug = "grapher/by-slug",
-}
 
 let s3Client: S3Client | undefined = undefined
 

devTools/syncGraphersToR2/syncGraphersToR2.ts

@@ -23,7 +23,6 @@ import {
     KnexReadonlyTransaction,
     knexReadonlyTransaction,
 } from "../../db/db.js"
-import { R2GrapherConfigDirectory } from "../../adminSiteServer/chartConfigR2Helpers.js"
 import {
     base64ToBytes,
     bytesToBase64,
@@ -32,6 +31,7 @@ import {
     excludeUndefined,
     HexString,
     hexToBytes,
+    R2GrapherConfigDirectory,
 } from "@ourworldindata/utils"
 import { string } from "ts-pattern/dist/patterns.js"
 import { chunk, take } from "lodash"

functions/_common/grapherRenderer.ts

@@ -1,5 +1,10 @@
-import { Grapher, GrapherInterface } from "@ourworldindata/grapher"
-import { Bounds, deserializeJSONFromHTML } from "@ourworldindata/utils"
+import { Grapher } from "@ourworldindata/grapher"
+import {
+    Bounds,
+    excludeUndefined,
+    GrapherInterface,
+    R2GrapherConfigDirectory,
+} from "@ourworldindata/utils"
 import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm"
 import { TimeLogger } from "./timeLogger"
 import { png } from "itty-router"
@@ -143,19 +148,33 @@ async function fetchAndRenderGrapherToSvg({
 }) {
     const grapherLogger = new TimeLogger("grapher")
 
-    // Fetch grapher config and extract it from the HTML
-    const grapherConfig: GrapherInterface = await env.ASSETS.fetch(
-        new URL(`/grapher/${slug}`, env.url)
-    )
-        .then((r) => (r.ok ? r : Promise.reject("Failed to load grapher page")))
-        .then((r) => r.text())
-        .then((html) => deserializeJSONFromHTML(html))
+    const url = new URL(`/grapher/${slug}`, env.url)
+    const slugOnly = url.pathname.split("/").pop()
 
-    if (!grapherConfig) {
-        throw new Error("Could not find grapher config")
+    // The top level directory is either the bucket path (should be set in dev environments and production)
+    // or the branch name on preview staging environments
+    console.log("branch", env.CF_PAGES_BRANCH)
+    const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_PATH
+        ? [env.GRAPHER_CONFIG_R2_BUCKET_PATH]
+        : ["by-branch", env.CF_PAGES_BRANCH]
+
+    const key = excludeUndefined([
+        ...topLevelDirectory,
+        R2GrapherConfigDirectory.publishedGrapherBySlug,
+        `${slugOnly}.json`,
+    ]).join("/")
+
+    console.log("fetching grapher config from this key", key)
+
+    // Fetch grapher config
+    const fetchResponse = await env.r2ChartConfigs.get(key)
+
+    if (!fetchResponse) {
+        return null
     }
 
-    grapherLogger.log("fetchGrapherConfig")
+    const grapherConfig: GrapherInterface = await fetchResponse.json()
+    console.log("grapher title", grapherConfig.title)
 
     const bounds = new Bounds(0, 0, options.svgWidth, options.svgHeight)
     const grapher = new Grapher({
@@ -206,6 +225,10 @@ export const fetchAndRenderGrapher = async (
         env,
     })
 
+    if (!svg) {
+        return new Response("Not found", { status: 404 })
+    }
+
     switch (outType) {
         case "png":
             return png(await renderSvgToPng(svg, options))

functions/grapher/thumbnail/[slug].ts

@@ -5,7 +5,12 @@ export interface Env {
     ASSETS: {
         fetch: typeof fetch
     }
+    r2ChartConfigs: {
+        get: (url: string) => Promise<R2ObjectBody>
+    }
     url: URL
+    GRAPHER_CONFIG_R2_BUCKET_PATH: string
+    CF_PAGES_BRANCH: string
     ENV: string
 }
 

package.json

@@ -26,7 +26,7 @@
         "revertLastDbMigration": "tsx --tsconfig tsconfig.tsx.json node_modules/typeorm/cli.js migration:revert -d db/dataSource.ts",
         "startAdminServer": "node --enable-source-maps ./itsJustJavascript/adminSiteServer/app.js",
         "startAdminDevServer": "tsx watch --ignore '**.mjs' --tsconfig tsconfig.tsx.json adminSiteServer/app.tsx",
-        "startLocalCloudflareFunctions": "wrangler pages dev",
+        "startLocalCloudflareFunctions": "wrangler pages dev --local --persist-to ./cfstorage",
         "startDeployQueueServer": "node --enable-source-maps ./itsJustJavascript/baker/startDeployQueueServer.js",
         "startLernaWatcher": "lerna watch --scope '@ourworldindata/*' -- lerna run build --scope=\\$LERNA_PACKAGE_NAME --include-dependents",
         "startTmuxServer": "node_modules/tmex/tmex dev \"yarn startLernaWatcher\" \"yarn startAdminDevServer\" \"yarn startViteServer\"",

packages/@ourworldindata/types/src/domainTypes/Various.ts

@@ -65,3 +65,8 @@ export class JsonError extends Error {
 export interface QueryParams {
     [key: string]: string | undefined
 }
+
+export enum R2GrapherConfigDirectory {
+    byUUID = "config/by-uuid",
+    publishedGrapherBySlug = "grapher/by-slug",
+}

packages/@ourworldindata/types/src/index.ts

@@ -18,6 +18,7 @@ export {
     type RawPageview,
     type UserCountryInformation,
     type QueryParams,
+    R2GrapherConfigDirectory,
 } from "./domainTypes/Various.js"
 export { type BreadcrumbItem, type KeyValueProps } from "./domainTypes/Site.js"
 export {

wrangler.toml

@@ -13,11 +13,26 @@ MAILGUN_DOMAIN = "mg.ourworldindata.org"
 SLACK_ERROR_CHANNEL_ID = "C016H0BNNB1"
 ENV = "preview"
 
+[[r2_buckets]]
+binding = "r2ChartConfigs"
+bucket_name = "owid-grapher-configs-staging"
+
 # Overrides for CF production deployment
 [env.production]
 compatibility_date = "2024-04-29"
 
+[[env.production.r2_buckets]]
+binding = "r2ChartConfigs"
+bucket_name = "owid-grapher-configs"
+
 [env.production.vars]
 ENV = "production"
 MAILGUN_DOMAIN = "mg.ourworldindata.org"
 SLACK_ERROR_CHANNEL_ID = "C5JJW19PS"
+GRAPHER_CONFIG_R2_BUCKET_PATH = "v1"
+
+
+[[env.preview.r2_buckets]]
+binding = "r2ChartConfigs"
+bucket_name = "owid-grapher-configs-staging"
+