fix(bake): lightning deploys populate gdoc metadata

adminSiteClient/GdocsDiff.tsx

@@ -7,6 +7,9 @@ import { omit, OwidGdocInterface } from "@ourworldindata/utils"
 // Errors are already shown in the settings drawer, so we don't show those either
 export const GDOC_DIFF_OMITTABLE_PROPERTIES = [
     "errors",
+    // imageMetadata *does* count as something that meaningfully contributes to whether a Gdoc is different,
+    // And it's checked in the checkIsLightningUpdate function,
+    // but for the preview, it's already captured by changes to the body text, so we don't need to show these objects in the diff also
     "imageMetadata",
     "linkedCharts",
     "linkedDocuments",

adminSiteClient/gdocsDeploy.ts

@@ -36,7 +36,6 @@ export const checkIsLightningUpdate = (
     > = {
         breadcrumbs: true,
         errors: true,
-        imageMetadata: true,
         linkedCharts: true,
         linkedDocuments: true,
         relatedCharts: true,
@@ -47,6 +46,7 @@ export const checkIsLightningUpdate = (
         updatedAt: true,
         createdAt: false,
         id: false, // weird case - can't be updated
+        imageMetadata: false, // could require baking new images
         publicationContext: false, // requires an update of the blog roll
         published: false, // requires an update of the blog roll
         publishedAt: false, // could require an update of the blog roll

adminSiteServer/apiRouter.ts

@@ -2502,6 +2502,10 @@ apiRouter.put("/gdocs/:id", async (req, res) => {
         .getRepository(Gdoc)
         .create(getOwidGdocFromJSON(nextGdocJSON))
 
+    // Need to load these to compare them for lightning update candidacy
+    await prevGdoc.loadImageMetadata()
+    await nextGdoc.loadImageMetadata()
+
     // Deleting and recreating these is simpler than tracking orphans over the next code block
     await GdocXImage.delete({ gdocId: id })
     const filenames = nextGdoc.filenames

baker/DeployUtils.ts

@@ -7,8 +7,7 @@ import {
     BAKED_BASE_URL,
     BUILDKITE_API_ACCESS_TOKEN,
 } from "../settings/serverSettings.js"
-import { DeployChange, OwidGdocPublished } from "@ourworldindata/utils"
-import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
+import { DeployChange } from "@ourworldindata/utils"
 import { SiteBaker } from "../baker/SiteBaker.js"
 
 const deployQueueServer = new DeployQueueServer()
@@ -56,13 +55,10 @@ const triggerBakeAndDeploy = async (
         // otherwise, bake locally. This is used for local development or staging servers
         const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL)
         if (lightningQueue?.length) {
-            for (const change of lightningQueue) {
-                const gdoc = (await Gdoc.findOneByOrFail({
-                    published: true,
-                    slug: change.slug,
-                })) as OwidGdocPublished
-                await baker.bakeGDocPost(gdoc)
-            }
+            if (!lightningQueue.every((change) => change.slug))
+                throw new Error("Lightning deploy is missing a slug")
+
+            await baker.bakeGDocPosts(lightningQueue.map((c) => c.slug!))
         } else {
             await baker.bakeAll()
         }

baker/SiteBaker.tsx

@@ -208,7 +208,7 @@ export class SiteBaker {
     }
 
     // Bake an individual post/page
-    async bakeGDocPost(post: OwidGdocPublished) {
+    private async bakeGDocPost(post: OwidGdocPublished) {
         const html = renderGdoc(post)
         const outPath = path.join(this.bakedSiteDir, `${post.slug}.html`)
         await fs.mkdirp(path.dirname(outPath))
@@ -307,12 +307,27 @@ export class SiteBaker {
         this.progressBar.tick({ name: "✅ baked posts" })
     }
 
-    // Bake all GDoc posts
-    async bakeGDocPosts() {
+    // Bake all GDoc posts, or a subset of them if slugs are provided
+    async bakeGDocPosts(slugs?: string[]) {
         await db.getConnection()
         if (!this.bakeSteps.has("gdocPosts")) return
         const publishedGdocs = await Gdoc.getPublishedGdocs()
 
+        const gdocsToBake =
+            slugs !== undefined
+                ? publishedGdocs.filter((gdoc) => slugs.includes(gdoc.slug))
+                : publishedGdocs
+
+        // Ensure we have a published gdoc for each slug given
+        if (slugs !== undefined && slugs.length !== gdocsToBake.length) {
+            const slugsNotFound = slugs.filter(
+                (slug) => !gdocsToBake.find((gdoc) => gdoc.slug === slug)
+            )
+            throw new Error(
+                `Some of the gdoc slugs were not found or are not published: ${slugsNotFound}`
+            )
+        }
+
         // Prefetch publishedGdocs, imageMetadata, and linkedCharts instead of each instance fetching
         const publishedGdocsDictionary = keyBy(publishedGdocs.map(clone), "id")
         const imageMetadataDictionary = await Image.find().then((images) =>
@@ -349,7 +364,7 @@ export class SiteBaker {
                 )
         )
 
-        for (const publishedGdoc of publishedGdocs) {
+        for (const publishedGdoc of gdocsToBake) {
             // Pick the necessary metadata from the dictionaries we prefetched
             publishedGdoc.linkedDocuments = pick(
                 publishedGdocsDictionary,

baker/bakeGdocPost.ts

@@ -4,8 +4,6 @@ import yargs from "yargs"
 import { hideBin } from "yargs/helpers"
 import { SiteBaker } from "./SiteBaker.js"
 import { BAKED_SITE_DIR, BAKED_BASE_URL } from "../settings/serverSettings.js"
-import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
-import { OwidGdocPublished } from "@ourworldindata/utils"
 import * as db from "../db/db.js"
 
 yargs(hideBin(process.argv))
@@ -22,11 +20,7 @@ yargs(hideBin(process.argv))
             const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL)
 
             await db.getConnection()
-            const gdoc = (await Gdoc.findOneByOrFail({
-                published: true,
-                slug: slug,
-            })) as OwidGdocPublished
-            await baker.bakeGDocPost(gdoc)
+            await baker.bakeGDocPosts([slug])
             process.exit(0)
         }
     )

baker/bakeGdocPosts.ts

@@ -4,8 +4,6 @@ import yargs from "yargs"
 import { hideBin } from "yargs/helpers"
 import { SiteBaker } from "./SiteBaker.js"
 import { BAKED_SITE_DIR, BAKED_BASE_URL } from "../settings/serverSettings.js"
-import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
-import { OwidGdocPublished } from "@ourworldindata/utils"
 import * as db from "../db/db.js"
 
 yargs(hideBin(process.argv))
@@ -27,15 +25,7 @@ yargs(hideBin(process.argv))
             const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL)
 
             await db.getConnection()
-            await Promise.all(
-                slugs.map(async (slug) => {
-                    const gdoc = (await Gdoc.findOneByOrFail({
-                        published: true,
-                        slug: slug,
-                    })) as OwidGdocPublished
-                    await baker.bakeGDocPost(gdoc)
-                })
-            )
+            await baker.bakeGDocPosts(slugs)
             process.exit(0)
         }
     )

ops/buildkite/deploy-content

@@ -27,20 +27,16 @@ deploy_content () {
         exit 1
     fi
 
-    # Lightning deploys ARE CURRENTLY DISABLED because they're screwing up the site
     # NOTE: in theory we could run lightning bake in parallel to regular bake and only lock `deploy_to_cloudflare`
     # right now lightning bake has to wait if there's a regular bake
-    # if [ -n "${LIGHTNING_GDOC_SLUGS:-}" ]; then
-    #     # TODO: do we need to trigger `yarn buildLocalBake --steps gdriveImages` to get up to date images for the lightning post?
-    #     bake_gdoc_posts "$LIGHTNING_GDOC_SLUGS"
-    # else
-
-    update_owid_content_repo
-    sync_wordpress_uploads
-    bake_site
-    sync_baked_data_to_r2
-
-    # fi
+    if [ -n "${LIGHTNING_GDOC_SLUGS:-}" ]; then
+        bake_gdoc_posts "$LIGHTNING_GDOC_SLUGS"
+    else
+        update_owid_content_repo
+        sync_wordpress_uploads
+        bake_site
+        sync_baked_data_to_r2
+    fi
 
     create_dist
     deploy_to_cloudflare