diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 8b7ae33f20f..a60f472803d 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -85,6 +85,7 @@ import { DbInsertUser, FlatTagGraph, DbRawChartConfig, + R2GrapherConfigDirectory, } from "@ourworldindata/types" import { uuidv7 } from "uuidv7" import { @@ -158,7 +159,6 @@ import path from "path" import { deleteGrapherConfigFromR2, deleteGrapherConfigFromR2ByUUID, - R2GrapherConfigDirectory, saveGrapherConfigToR2, saveGrapherConfigToR2ByUUID, getMd5HashBase64, diff --git a/adminSiteServer/chartConfigR2Helpers.ts b/adminSiteServer/chartConfigR2Helpers.ts index 781fbd233fb..eb670bcb46e 100644 --- a/adminSiteServer/chartConfigR2Helpers.ts +++ b/adminSiteServer/chartConfigR2Helpers.ts @@ -14,6 +14,7 @@ import { S3Client, } from "@aws-sdk/client-s3" import { Base64String, JsonError } from "@ourworldindata/utils" +import { R2GrapherConfigDirectory } from "@ourworldindata/types" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" import { createHash } from "crypto" @@ -25,10 +26,6 @@ export function getMd5HashBase64(data: string): Base64String { .update(data, "utf-8") .digest("base64") as Base64String } -export enum R2GrapherConfigDirectory { - byUUID = "config/by-uuid", - publishedGrapherBySlug = "grapher/by-slug", -} let s3Client: S3Client | undefined = undefined diff --git a/devTools/syncGraphersToR2/syncGraphersToR2.ts b/devTools/syncGraphersToR2/syncGraphersToR2.ts index bedbc1d722d..081d83adc6c 100644 --- a/devTools/syncGraphersToR2/syncGraphersToR2.ts +++ b/devTools/syncGraphersToR2/syncGraphersToR2.ts @@ -23,7 +23,6 @@ import { KnexReadonlyTransaction, knexReadonlyTransaction, } from "../../db/db.js" -import { R2GrapherConfigDirectory } from "../../adminSiteServer/chartConfigR2Helpers.js" import { base64ToBytes, bytesToBase64, @@ -32,6 +31,7 @@ import { excludeUndefined, HexString, hexToBytes, + R2GrapherConfigDirectory, } from "@ourworldindata/utils" import { string } from "ts-pattern/dist/patterns.js" import { chunk, take } from "lodash" diff --git a/functions/README.md b/functions/README.md index caaa61334a0..6fdf5ca8bf4 100644 --- a/functions/README.md +++ b/functions/README.md @@ -10,6 +10,8 @@ Pages Functions are very similar to Cloudflare Workers; however they will always Pages Functions use file-based routing, which means that the file `grapher/[slug].ts` will serve routes like `/grapher/child-mortality`. In addition, there's a [`_routes.json`](../_routes.json) file that specifies which routes are to be served dynamically. +Inside a file-based route we sometimes use an instance of itty-router to decide on the exact functionality to provide (e.g. png vs svg generation) + ## Development 1. Copy `.dev.vars.example` to `.dev.vars` and fill in the required variables. @@ -28,7 +30,9 @@ Note: compatibility dates between local development, production and preview envi ## Testing on Fondation staging sites vs Cloudfare previews -`yarn deployContentPreview` deploys the staging `bakedSite` to a Cloudflare preview at https://[PREVIEW_BRANCH].owid-staging.pages.dev. This is the recommended way to test functions in a production-like environment. See [../ops/buildkite/deploy-content-preview](../ops/buildkite/deploy-content-preview) for more details. +We have two cloudflare projects set up that you can deploy previews to. `owid` which is also where our production deployment runs, and `owid-staging`. Currently, `owid` is configured to require authentication while `owid-staging` is accessible from the internet without any kind of auth. + +`yarn deployContentPreview` deploys the staging `bakedSite` to a Cloudflare preview at https://[PREVIEW_BRANCH].[PROJECT].pages.dev. This is the recommended way to test functions in a production-like environment. See [../ops/buildkite/deploy-content-preview](../ops/buildkite/deploy-content-preview) for more details. ### Rationale @@ -36,7 +40,7 @@ A custom staging site is available at http://staging-site-[BRANCH] upon pushing When it comes to testing functions in a production-like environment, Cloudflare previews are recommended. -Cloudflare previews are served by Cloudflare (as opposed to `wrangler` on staging sites) and are available at https://[RANDOM_ID].owid-staging.pages.dev. Cloudflare previews do not rely on the `wrangler` CLI and its `.dev.vars` file. Instead, they use the [Cloudflare dashboard to configure environment variables](https://dash.cloudflare.com/078fcdfed9955087315dd86792e71a7e/pages/view/owid/settings/environment-variables), in the same way and place as the production site. +Cloudflare previews are served by Cloudflare (as opposed to `wrangler` on staging sites) and are available at https://[RANDOM_ID].[PROJECT].pages.dev. Cloudflare previews do not rely on the `wrangler` CLI and its `.dev.vars` file, but they do take the `wrangler.toml` file into account for environment variables. For secrets, they use the [values set via the Cloudflare dashboard](https://dash.cloudflare.com/078fcdfed9955087315dd86792e71a7e/pages/view/owid/settings/environment-variables), in the same way and place as the production site. This proximity of configurations in the Cloudflare dashboard makes spotting differences between production and preview environments easier - and is one of the reason of using Cloudflare previews in the same project (owid) over using a new project specific to staging. diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 249488b75bf..ffe054581dc 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -1,5 +1,10 @@ -import { Grapher, GrapherInterface } from "@ourworldindata/grapher" -import { Bounds, deserializeJSONFromHTML } from "@ourworldindata/utils" +import { Grapher } from "@ourworldindata/grapher" +import { + Bounds, + excludeUndefined, + GrapherInterface, + R2GrapherConfigDirectory, +} from "@ourworldindata/utils" import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm" import { TimeLogger } from "./timeLogger" import { png } from "itty-router" @@ -143,19 +148,33 @@ async function fetchAndRenderGrapherToSvg({ }) { const grapherLogger = new TimeLogger("grapher") - // Fetch grapher config and extract it from the HTML - const grapherConfig: GrapherInterface = await env.ASSETS.fetch( - new URL(`/grapher/${slug}`, env.url) - ) - .then((r) => (r.ok ? r : Promise.reject("Failed to load grapher page"))) - .then((r) => r.text()) - .then((html) => deserializeJSONFromHTML(html)) + const url = new URL(`/grapher/${slug}`, env.url) + const slugOnly = url.pathname.split("/").pop() - if (!grapherConfig) { - throw new Error("Could not find grapher config") + // The top level directory is either the bucket path (should be set in dev environments and production) + // or the branch name on preview staging environments + console.log("branch", env.CF_PAGES_BRANCH) + const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_PATH + ? [env.GRAPHER_CONFIG_R2_BUCKET_PATH] + : ["by-branch", env.CF_PAGES_BRANCH] + + const key = excludeUndefined([ + ...topLevelDirectory, + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${slugOnly}.json`, + ]).join("/") + + console.log("fetching grapher config from this key", key) + + // Fetch grapher config + const fetchResponse = await env.r2ChartConfigs.get(key) + + if (!fetchResponse) { + return null } - grapherLogger.log("fetchGrapherConfig") + const grapherConfig: GrapherInterface = await fetchResponse.json() + console.log("grapher title", grapherConfig.title) const bounds = new Bounds(0, 0, options.svgWidth, options.svgHeight) const grapher = new Grapher({ @@ -206,6 +225,10 @@ export const fetchAndRenderGrapher = async ( env, }) + if (!svg) { + return new Response("Not found", { status: 404 }) + } + switch (outType) { case "png": return png(await renderSvgToPng(svg, options)) diff --git a/functions/grapher/thumbnail/[slug].ts b/functions/grapher/thumbnail/[slug].ts index b5efae2ac13..a62cb8c8d17 100644 --- a/functions/grapher/thumbnail/[slug].ts +++ b/functions/grapher/thumbnail/[slug].ts @@ -5,7 +5,12 @@ export interface Env { ASSETS: { fetch: typeof fetch } + r2ChartConfigs: { + get: (url: string) => Promise + } url: URL + GRAPHER_CONFIG_R2_BUCKET_PATH: string + CF_PAGES_BRANCH: string ENV: string } diff --git a/package.json b/package.json index 20f8e503d94..7a5c7142f9b 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "revertLastDbMigration": "tsx --tsconfig tsconfig.tsx.json node_modules/typeorm/cli.js migration:revert -d db/dataSource.ts", "startAdminServer": "node --enable-source-maps ./itsJustJavascript/adminSiteServer/app.js", "startAdminDevServer": "tsx watch --ignore '**.mjs' --tsconfig tsconfig.tsx.json adminSiteServer/app.tsx", - "startLocalCloudflareFunctions": "wrangler pages dev", + "startLocalCloudflareFunctions": "wrangler pages dev --local --persist-to ./cfstorage", "startDeployQueueServer": "node --enable-source-maps ./itsJustJavascript/baker/startDeployQueueServer.js", "startLernaWatcher": "lerna watch --scope '@ourworldindata/*' -- lerna run build --scope=\\$LERNA_PACKAGE_NAME --include-dependents", "startTmuxServer": "node_modules/tmex/tmex dev \"yarn startLernaWatcher\" \"yarn startAdminDevServer\" \"yarn startViteServer\"", diff --git a/packages/@ourworldindata/types/src/domainTypes/Various.ts b/packages/@ourworldindata/types/src/domainTypes/Various.ts index 946339baa14..bc23e990f9d 100644 --- a/packages/@ourworldindata/types/src/domainTypes/Various.ts +++ b/packages/@ourworldindata/types/src/domainTypes/Various.ts @@ -65,3 +65,8 @@ export class JsonError extends Error { export interface QueryParams { [key: string]: string | undefined } + +export enum R2GrapherConfigDirectory { + byUUID = "config/by-uuid", + publishedGrapherBySlug = "grapher/by-slug", +} diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 3c8e8af77d0..019d446f9ec 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -18,6 +18,7 @@ export { type RawPageview, type UserCountryInformation, type QueryParams, + R2GrapherConfigDirectory, } from "./domainTypes/Various.js" export { type BreadcrumbItem, type KeyValueProps } from "./domainTypes/Site.js" export { diff --git a/wrangler.toml b/wrangler.toml index 4d88b657784..ab8b57941aa 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -13,11 +13,26 @@ MAILGUN_DOMAIN = "mg.ourworldindata.org" SLACK_ERROR_CHANNEL_ID = "C016H0BNNB1" ENV = "preview" +[[r2_buckets]] +binding = "r2ChartConfigs" +bucket_name = "owid-grapher-configs-staging" + # Overrides for CF production deployment [env.production] compatibility_date = "2024-04-29" +[[env.production.r2_buckets]] +binding = "r2ChartConfigs" +bucket_name = "owid-grapher-configs" + [env.production.vars] ENV = "production" MAILGUN_DOMAIN = "mg.ourworldindata.org" SLACK_ERROR_CHANNEL_ID = "C5JJW19PS" +GRAPHER_CONFIG_R2_BUCKET_PATH = "v1" + + +[[env.preview.r2_buckets]] +binding = "r2ChartConfigs" +bucket_name = "owid-grapher-configs-staging" +