From d6aa96e58186560903973507fa155f6245a371ed Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Thu, 15 Aug 2024 15:21:10 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20switch=20from=20R2=20bindings=20?= =?UTF-8?q?to=20using=20fetch=20against=20a=20public=20bucket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dev.vars.example | 2 + functions/_common/grapherRenderer.ts | 75 +++++++++++++++---- functions/grapher/thumbnail/[slug].ts | 6 +- package.json | 2 +- .../types/src/domainTypes/Various.ts | 2 +- wrangler.toml | 22 +++--- 6 files changed, 76 insertions(+), 33 deletions(-) diff --git a/.dev.vars.example b/.dev.vars.example index 5e3ac2ec579..384f1dc15a7 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -20,3 +20,5 @@ MAILGUN_SENDING_KEY= # optional SLACK_BOT_OAUTH_TOKEN= SLACK_ERROR_CHANNEL_ID=C016H0BNNB1 #bot-testing channel + +GRAPHER_CONFIG_R2_BUCKET_PATH=devs/YOURNAME diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index ffe054581dc..b18a5551b30 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -135,17 +135,36 @@ const extractOptions = (params: URLSearchParams): ImageOptions => { return options as ImageOptions } -async function fetchAndRenderGrapherToSvg({ - slug, - options, - searchParams, - env, -}: { - slug: string - options: ImageOptions - searchParams: URLSearchParams - env: Env -}) { +const WORKER_CACHE_TIME_IN_SECONDS = 60 + +async function fetchFromR2( + url: URL, + etag: string | undefined, + fallbackUrl?: URL +) { + const headers = new Headers() + if (etag) headers.set("If-None-Match", etag) + const init = { + cf: { + cacheEverything: true, + cacheTtl: WORKER_CACHE_TIME_IN_SECONDS, + }, + headers, + } + const primaryResponse = await fetch(url.toString(), init) + if (primaryResponse.status === 404 && fallbackUrl) { + return fetch(fallbackUrl.toString(), init) + } + return primaryResponse +} + +async function fetchAndRenderGrapherToSvg( + slug: string, + options: ImageOptions, + searchParams: URLSearchParams, + env: Env, + etag?: string +) { const grapherLogger = new TimeLogger("grapher") const url = new URL(`/grapher/${slug}`, env.url) @@ -166,10 +185,31 @@ async function fetchAndRenderGrapherToSvg({ console.log("fetching grapher config from this key", key) + const requestUrl = new URL(key, env.GRAPHER_CONFIG_R2_BUCKET_URL) + + let fallbackUrl + + if ( + env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL && + env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH + ) { + const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH + const fallbackKey = excludeUndefined([ + ...topLevelDirectory, + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${slugOnly}.json`, + ]).join("/") + fallbackUrl = new URL( + fallbackKey, + env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL + ) + } + // Fetch grapher config - const fetchResponse = await env.r2ChartConfigs.get(key) + const fetchResponse = await fetchFromR2(requestUrl, etag, fallbackUrl) - if (!fetchResponse) { + if (fetchResponse.status !== 200) { + console.log("Failed to fetch grapher config", fetchResponse.status) return null } @@ -213,17 +253,20 @@ export const fetchAndRenderGrapher = async ( slug: string, searchParams: URLSearchParams, outType: "png" | "svg", - env: Env + env: Env, + etag?: string ) => { const options = extractOptions(searchParams) console.log("Rendering", slug, outType, options) - const svg = await fetchAndRenderGrapherToSvg({ + const svg = await fetchAndRenderGrapherToSvg( slug, options, searchParams, env, - }) + etag + ) + console.log("fetched svg") if (!svg) { return new Response("Not found", { status: 404 }) diff --git a/functions/grapher/thumbnail/[slug].ts b/functions/grapher/thumbnail/[slug].ts index a62cb8c8d17..e49e32508fc 100644 --- a/functions/grapher/thumbnail/[slug].ts +++ b/functions/grapher/thumbnail/[slug].ts @@ -5,11 +5,11 @@ export interface Env { ASSETS: { fetch: typeof fetch } - r2ChartConfigs: { - get: (url: string) => Promise - } url: URL + GRAPHER_CONFIG_R2_BUCKET_URL: string + GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL: string GRAPHER_CONFIG_R2_BUCKET_PATH: string + GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH: string CF_PAGES_BRANCH: string ENV: string } diff --git a/package.json b/package.json index 7a5c7142f9b..20f8e503d94 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "revertLastDbMigration": "tsx --tsconfig tsconfig.tsx.json node_modules/typeorm/cli.js migration:revert -d db/dataSource.ts", "startAdminServer": "node --enable-source-maps ./itsJustJavascript/adminSiteServer/app.js", "startAdminDevServer": "tsx watch --ignore '**.mjs' --tsconfig tsconfig.tsx.json adminSiteServer/app.tsx", - "startLocalCloudflareFunctions": "wrangler pages dev --local --persist-to ./cfstorage", + "startLocalCloudflareFunctions": "wrangler pages dev", "startDeployQueueServer": "node --enable-source-maps ./itsJustJavascript/baker/startDeployQueueServer.js", "startLernaWatcher": "lerna watch --scope '@ourworldindata/*' -- lerna run build --scope=\\$LERNA_PACKAGE_NAME --include-dependents", "startTmuxServer": "node_modules/tmex/tmex dev \"yarn startLernaWatcher\" \"yarn startAdminDevServer\" \"yarn startViteServer\"", diff --git a/packages/@ourworldindata/types/src/domainTypes/Various.ts b/packages/@ourworldindata/types/src/domainTypes/Various.ts index bc23e990f9d..a440821591c 100644 --- a/packages/@ourworldindata/types/src/domainTypes/Various.ts +++ b/packages/@ourworldindata/types/src/domainTypes/Various.ts @@ -68,5 +68,5 @@ export interface QueryParams { export enum R2GrapherConfigDirectory { byUUID = "config/by-uuid", - publishedGrapherBySlug = "grapher/by-slug", + publishedGrapherBySlug = "config/by-slug-published", } diff --git a/wrangler.toml b/wrangler.toml index ab8b57941aa..0c232fd44b9 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -6,33 +6,31 @@ pages_build_output_dir = "./localBake" # Vars that should be available in all envs, including local dev [vars] ENV = "development" +GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs-staging.ourworldindata.org" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "https://grapher-configs.ourworldindata.org" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "v1" + # Overrides for CF preview deployments [env.preview.vars] MAILGUN_DOMAIN = "mg.ourworldindata.org" SLACK_ERROR_CHANNEL_ID = "C016H0BNNB1" ENV = "preview" - -[[r2_buckets]] -binding = "r2ChartConfigs" -bucket_name = "owid-grapher-configs-staging" +GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs-staging.ourworldindata.org" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "https://grapher-configs.ourworldindata.org" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "v1" # Overrides for CF production deployment [env.production] compatibility_date = "2024-04-29" -[[env.production.r2_buckets]] -binding = "r2ChartConfigs" -bucket_name = "owid-grapher-configs" - [env.production.vars] ENV = "production" MAILGUN_DOMAIN = "mg.ourworldindata.org" SLACK_ERROR_CHANNEL_ID = "C5JJW19PS" +GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs.ourworldindata.org" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "" GRAPHER_CONFIG_R2_BUCKET_PATH = "v1" -[[env.preview.r2_buckets]] -binding = "r2ChartConfigs" -bucket_name = "owid-grapher-configs-staging" -