✨ Render thumbnails for graphers by uuid (#3880)

This PR duplicates the thumbnail rendering logic into the top level so that we can render pngs at `/grapher/SLUG.png` and svgs at `/grapher/SLUG.svg`. It also adds thumbnail rendering at `/grapher/by-uuid/UUID.png` and `.svg`. 

The HTML rewriting (of og images) is changed to point to the new `/grapher/SLUG.png` route. 

The old thumbnail rendering at `/grapher/thumbnail/SLUG` is kept for now so that existing fetches will still work until the various cached versions of the HTML have expired.

.env.example-full

@@ -36,7 +36,7 @@ GRAPHER_CONFIG_R2_BUCKET_PATH= # optional - for local dev set it to "devs/YOURNA
 
 OPENAI_API_KEY=
 
-GRAPHER_DYNAMIC_THUMBNAIL_URL=  # optional; can set this to https://ourworldindata.org/grapher/thumbnail to use the live thumbnail worker
+GRAPHER_DYNAMIC_THUMBNAIL_URL=  # optional; can set this to https://ourworldindata.org/grapher to use the live thumbnail worker
 
 # enable search (readonly)
 ALGOLIA_ID=   # optional

functions/_common/grapherRenderer.ts

@@ -23,6 +23,8 @@ declare global {
     var window: any
 }
 
+export type Etag = string
+
 const grapherBaseUrl = "https://ourworldindata.org/grapher"
 
 // Lots of defaults; these are mostly the same as they are in owid-grapher.
@@ -166,17 +168,17 @@ interface FetchGrapherConfigResult {
     etag: string | undefined
 }
 
-interface GrapherSlug {
+export interface GrapherSlug {
     type: "slug"
     id: string
 }
 
-interface GrapherUuid {
+export interface GrapherUuid {
     type: "uuid"
     id: string
 }
 
-type GrapherIdentifier = GrapherSlug | GrapherUuid
+export type GrapherIdentifier = GrapherSlug | GrapherUuid
 
 export async function fetchUnparsedGrapherConfig(
     identifier: GrapherIdentifier,
@@ -267,17 +269,14 @@ export async function fetchGrapherConfig(
 }
 
 async function fetchAndRenderGrapherToSvg(
-    slug: string,
+    id: GrapherIdentifier,
     options: ImageOptions,
     searchParams: URLSearchParams,
     env: Env
 ): Promise<string> {
     const grapherLogger = new TimeLogger("grapher")
 
-    const grapherConfigResponse = await fetchGrapherConfig(
-        { type: "slug", id: slug },
-        env
-    )
+    const grapherConfigResponse = await fetchGrapherConfig(id, env)
 
     if (grapherConfigResponse.status === 404) {
         // we throw 404 errors instad of returning a 404 response so that the router
@@ -320,20 +319,15 @@ async function fetchAndRenderGrapherToSvg(
 }
 
 export const fetchAndRenderGrapher = async (
-    slug: string,
+    id: GrapherIdentifier,
     searchParams: URLSearchParams,
     outType: "png" | "svg",
     env: Env
 ) => {
     const options = extractOptions(searchParams)
 
-    console.log("Rendering", slug, outType, options)
-    const svg = await fetchAndRenderGrapherToSvg(
-        slug,
-        options,
-        searchParams,
-        env
-    )
+    console.log("Rendering", id.id, outType, options)
+    const svg = await fetchAndRenderGrapherToSvg(id, options, searchParams, env)
     console.log("fetched svg")
 
     switch (outType) {

functions/_common/reusableHandlers.ts

@@ -0,0 +1,37 @@
+import { Env } from "./env.js"
+import {
+    Etag,
+    GrapherIdentifier,
+    fetchAndRenderGrapher,
+} from "./grapherRenderer.js"
+
+export async function handleThumbnailRequest(
+    id: GrapherIdentifier,
+    searchParams: URLSearchParams,
+    env: Env,
+    _etag: Etag,
+    ctx: EventContext<unknown, any, Record<string, unknown>>,
+    extension: "png" | "svg"
+) {
+    const url = new URL(env.url)
+    const shouldCache = !url.searchParams.has("nocache")
+
+    const cache = caches.default
+    console.log("Handling", env.url, ctx.request.headers.get("User-Agent"))
+    if (shouldCache) {
+        console.log("Checking cache")
+        const maybeCached = await cache.match(ctx.request)
+        console.log("Cache check result", maybeCached ? "hit" : "miss")
+        if (maybeCached) return maybeCached
+    }
+    const resp = await fetchAndRenderGrapher(id, searchParams, extension, env)
+    if (shouldCache) {
+        resp.headers.set("Cache-Control", "public, s-maxage=3600, max-age=3600")
+        ctx.waitUntil(caches.default.put(ctx.request, resp.clone()))
+    } else
+        resp.headers.set(
+            "Cache-Control",
+            "public, s-maxage=0, max-age=0, must-revalidate"
+        )
+    return resp
+}

functions/grapher/[slug].ts

@@ -2,18 +2,27 @@ import { Env } from "../_common/env.js"
 import {
     getOptionalRedirectForSlug,
     createRedirectResponse,
+    Etag,
     fetchUnparsedGrapherConfig,
 } from "../_common/grapherRenderer.js"
 import { IRequestStrict, Router, StatusError, error, cors } from "itty-router"
+import { handleThumbnailRequest } from "../_common/reusableHandlers.js"
 
 const { preflight, corsify } = cors({
     allowMethods: ["GET", "OPTIONS", "HEAD"],
 })
-const extensions = {
+// We collect the possible extensions here so we can easily take them into account
+// when handling redirects
+export const extensions = {
     configJson: ".config.json",
+    png: ".png",
+    svg: ".svg",
 }
 
-const router = Router<IRequestStrict, [URL, Env, string]>({
+const router = Router<
+    IRequestStrict,
+    [URL, Env, Etag, EventContext<unknown, any, Record<string, unknown>>]
+>({
     before: [preflight],
     finally: [corsify],
 })
@@ -23,6 +32,30 @@ router
         async ({ params: { slug } }, { searchParams }, env, etag) =>
             handleConfigRequest(slug, searchParams, env, etag)
     )
+    .get(
+        `/grapher/:slug${extensions.png}`,
+        async ({ params: { slug } }, { searchParams }, env, etag, ctx) =>
+            handleThumbnailRequest(
+                { type: "slug", id: slug },
+                searchParams,
+                env,
+                etag,
+                ctx,
+                "png"
+            )
+    )
+    .get(
+        `/grapher/:slug${extensions.svg}`,
+        async ({ params: { slug } }, { searchParams }, env, etag, ctx) =>
+            handleThumbnailRequest(
+                { type: "slug", id: slug },
+                searchParams,
+                env,
+                etag,
+                ctx,
+                "svg"
+            )
+    )
     .get(
         "/grapher/:slug",
         async ({ params: { slug } }, { searchParams }, env) =>
@@ -42,7 +75,8 @@ export const onRequest: PagesFunction = async (context) => {
             request,
             url,
             { ...env, url },
-            request.headers.get("if-none-match")
+            request.headers.get("if-none-match"),
+            context
         )
         .catch(async (e) => {
             // Here we do a unified after the fact handling of 404s to check
@@ -119,10 +153,10 @@ async function handleHtmlPageRequest(
     // In the case of the redirect, the browser will then request the new URL which will again be handled by this worker.
     if (grapherPageResp.status !== 200) return grapherPageResp
 
-    const openGraphThumbnailUrl = `/grapher/thumbnail/${slug}.png?imType=og${
+    const openGraphThumbnailUrl = `/grapher/${slug}.png?imType=og${
         url.search ? "&" + url.search.slice(1) : ""
     }`
-    const twitterThumbnailUrl = `/grapher/thumbnail/${slug}.png?imType=twitter${
+    const twitterThumbnailUrl = `/grapher/${slug}.png?imType=twitter${
         url.search ? "&" + url.search.slice(1) : ""
     }`
 

functions/grapher/by-uuid/[uuid].ts

@@ -1,14 +1,40 @@
 import { Env } from "../../_common/env.js"
 import { fetchGrapherConfig } from "../../_common/grapherRenderer.js"
 import { IRequestStrict, Router, error, StatusError } from "itty-router"
+import { handleThumbnailRequest } from "../../_common/reusableHandlers.js"
+import { extensions } from "../[slug].js"
 
 const router = Router<IRequestStrict, [URL, Env, string]>()
 router
     .get(
-        "/grapher/by-uuid/:uuid.config.json",
+        `/grapher/by-uuid/:uuid${extensions.configJson}`,
         async ({ params: { uuid } }, { searchParams }, env, etag) =>
             handleConfigRequest(uuid, searchParams, env, etag)
     )
+    .get(
+        `/grapher/by-uuid/:uuid${extensions.png}`,
+        async ({ params: { uuid } }, { searchParams }, env, etag, ctx) =>
+            handleThumbnailRequest(
+                { type: "uuid", id: uuid },
+                searchParams,
+                env,
+                etag,
+                ctx,
+                "png"
+            )
+    )
+    .get(
+        `/grapher/by-uuid/:uuid${extensions.svg}`,
+        async ({ params: { uuid } }, { searchParams }, env, etag, ctx) =>
+            handleThumbnailRequest(
+                { type: "uuid", id: uuid },
+                searchParams,
+                env,
+                etag,
+                ctx,
+                "svg"
+            )
+    )
     .all("*", () => error(404, "Route not defined"))
 
 export const onRequest: PagesFunction = async (context) => {
@@ -20,7 +46,8 @@ export const onRequest: PagesFunction = async (context) => {
             request,
             url,
             { ...env, url },
-            request.headers.get("if-none-match")
+            request.headers.get("if-none-match"),
+            context
         )
         .catch((e) => {
             if (e instanceof StatusError) {
@@ -56,7 +83,7 @@ async function handleConfigRequest(
         ? "public, s-maxage=3600, max-age=0, must-revalidate"
         : "public, s-maxage=0, max-age=0, must-revalidate"
 
-    return new Response(JSON.stringify(grapherPageResp.grapherConfig), {
+    return Response.json(grapherPageResp.grapherConfig, {
         headers: {
             "content-type": "application/json",
             "Cache-Control": cacheControl,

functions/grapher/thumbnail/[slug].ts

@@ -2,22 +2,39 @@ import { Env } from "../../_common/env.js"
 import { fetchAndRenderGrapher } from "../../_common/grapherRenderer.js"
 import { IRequestStrict, Router, error } from "itty-router"
 
+// TODO: remove the /grapher/thumbnail route two weeks or so after the change to use /grapher/:slug.png is deployed
+// We keep this around for another two weeks so that cached html pages etc can still fetch the correct thumbnail
 const router = Router<IRequestStrict, [URL, Env, ExecutionContext]>()
 router
     .get(
         "/grapher/thumbnail/:slug.png",
         async ({ params: { slug } }, { searchParams }, env) =>
-            fetchAndRenderGrapher(slug, searchParams, "png", env)
+            fetchAndRenderGrapher(
+                { type: "slug", id: slug },
+                searchParams,
+                "png",
+                env
+            )
     )
     .get(
         "/grapher/thumbnail/:slug.svg",
         async ({ params: { slug } }, { searchParams }, env) =>
-            fetchAndRenderGrapher(slug, searchParams, "svg", env)
+            fetchAndRenderGrapher(
+                { type: "slug", id: slug },
+                searchParams,
+                "svg",
+                env
+            )
     )
     .get(
         "/grapher/thumbnail/:slug",
         async ({ params: { slug } }, { searchParams }, env) =>
-            fetchAndRenderGrapher(slug, searchParams, "svg", env)
+            fetchAndRenderGrapher(
+                { type: "slug", id: slug },
+                searchParams,
+                "svg",
+                env
+            )
     )
     .all("*", () => error(404, "Route not defined"))
 

settings/clientSettings.ts

@@ -35,8 +35,7 @@ export const BAKED_SITE_EXPORTS_BASE_URL: string =
     process.env.BAKED_SITE_EXPORTS_BASE_URL ?? `${BAKED_BASE_URL}/exports`
 
 export const GRAPHER_DYNAMIC_THUMBNAIL_URL: string =
-    process.env.GRAPHER_DYNAMIC_THUMBNAIL_URL ??
-    `${BAKED_GRAPHER_URL}/thumbnail`
+    process.env.GRAPHER_DYNAMIC_THUMBNAIL_URL ?? `${BAKED_GRAPHER_URL}`
 
 export const ADMIN_BASE_URL: string =
     process.env.ADMIN_BASE_URL ??

site/search/SearchPanel.tsx

@@ -159,7 +159,7 @@ function ChartHit({
     )
     const queryStr = useMemo(() => getEntityQueryStr(entities), [entities])
     const previewUrl = queryStr
-        ? `${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${hit.slug}${queryStr}`
+        ? `${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${hit.slug}.svg${queryStr}`
         : `${BAKED_GRAPHER_EXPORTS_BASE_URL}/${hit.slug}.svg`
 
     useEffect(() => {