Merge pull request #3482 from owid/baking-perf-enhancements

Grapher baking performance enhancements

baker/GrapherBaker.tsx

@@ -26,6 +26,7 @@ import { glob } from "glob"
 import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js"
 import {
     getPostEnrichedBySlug,
+    getPostIdFromSlug,
     getPostRelatedCharts,
     getRelatedArticles,
     getRelatedResearchAndWritingForVariable,
@@ -331,12 +332,13 @@ const renderGrapherPage = async (
     grapher: GrapherInterface,
     knex: db.KnexReadonlyTransaction
 ) => {
-    const postSlug = urlToSlug(grapher.originUrl || "")
-    const post = postSlug
-        ? await getPostEnrichedBySlug(knex, postSlug)
+    const postSlug = urlToSlug(grapher.originUrl || "") as string | undefined
+    // TODO: update this to use gdocs posts
+    const postId = postSlug
+        ? await getPostIdFromSlug(knex, postSlug)
         : undefined
-    const relatedCharts = post
-        ? await getPostRelatedCharts(knex, post.id)
+    const relatedCharts = postId
+        ? await getPostRelatedCharts(knex, postId)
         : undefined
     const relatedArticles = grapher.id
         ? await getRelatedArticles(knex, grapher.id)
@@ -345,7 +347,6 @@ const renderGrapherPage = async (
     return renderToHtmlPage(
         <GrapherPage
             grapher={grapher}
-            post={post}
             relatedCharts={relatedCharts}
             relatedArticles={relatedArticles}
             baseUrl={BAKED_BASE_URL}
@@ -484,7 +485,7 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
         const chartsToBake: { id: number; config: string; slug: string }[] =
             await knexRaw(
                 knex,
-                `
+                `-- sql
                 SELECT
                     id, config, config->>'$.slug' as slug
                 FROM charts WHERE JSON_EXTRACT(config, "$.isPublished")=true

db/db.ts

@@ -243,7 +243,7 @@ export const getPublishedDataInsights = (
             slug,
             ROW_NUMBER() OVER (ORDER BY publishedAt DESC) - 1 AS \`index\`
         FROM posts_gdocs
-        WHERE content->>'$.type' = '${OwidGdocType.DataInsight}'
+        WHERE type = '${OwidGdocType.DataInsight}'
             AND published = TRUE
             AND publishedAt < NOW()
         ORDER BY publishedAt DESC
@@ -266,7 +266,7 @@ export const getPublishedDataInsightCount = (
         `
         SELECT COUNT(*) AS count
         FROM posts_gdocs
-        WHERE content->>'$.type' = '${OwidGdocType.DataInsight}'
+        WHERE type = '${OwidGdocType.DataInsight}'
             AND published = TRUE
             AND publishedAt < NOW()`
     ).then((res) => res?.count ?? 0)
@@ -313,7 +313,7 @@ export const getHomepageId = (
         FROM
             posts_gdocs
         WHERE
-            content->>'$.type' = '${OwidGdocType.Homepage}'
+            type = '${OwidGdocType.Homepage}'
             AND published = TRUE`
     ).then((result) => result?.id)
 }
@@ -364,7 +364,7 @@ export const getPublishedGdocPosts = async (
         posts_gdocs g
     WHERE
         g.published = 1
-        AND g.content ->> '$.type' IN (:types)
+        AND g.type IN (:types)
         AND g.publishedAt <= NOW()
     ORDER BY g.publishedAt DESC`,
         {
@@ -410,7 +410,7 @@ export const getPublishedGdocPostsWithTags = async (
         gxt.tagId = t.id
     WHERE
         g.published = 1
-        AND g.content ->> '$.type' IN (:types)
+        AND g.type IN (:types)
         AND g.publishedAt <= NOW()
     GROUP BY g.id
     ORDER BY g.publishedAt DESC`,

db/migration/1712842552502-AddPostsSlugIndex.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class AddPostsSlugIndex1712842552502 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            ALTER TABLE posts ADD INDEX idx_posts_slug (slug(100));
+        `)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+            DROP INDEX idx_posts_slug ON posts;
+        `)
+    }
+}

db/migration/1712843782834-AddVirtualGdocsTypeColumn.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from "typeorm"
+
+export class AddVirtualGdocsTypeColumn1712843782834
+    implements MigrationInterface
+{
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+        ALTER TABLE posts_gdocs
+        ADD COLUMN type VARCHAR(255) GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(content, '$.type'))) VIRTUAL AFTER slug;`)
+
+        await queryRunner.query(`-- sql
+        CREATE INDEX idx_posts_gdocs_type ON posts_gdocs (type);`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`-- sql
+        DROP INDEX idx_posts_gdocs_type ON posts_gdocs;`)
+
+        await queryRunner.query(`-- sql
+        ALTER TABLE posts_gdocs
+        DROP COLUMN type;`)
+    }
+}

db/model/Gdoc/GdocBase.ts

@@ -834,7 +834,7 @@ export async function getMinimalGdocPostsByIds(
                 published,
                 content ->> '$.subtitle' as subtitle,
                 content ->> '$.excerpt' as excerpt,
-                content ->> '$.type' as type,
+                type,
                 content ->> '$."featured-image"' as "featured-image"
             FROM posts_gdocs
             WHERE id in (:ids)`,

db/model/Gdoc/GdocFactory.ts

@@ -199,7 +199,7 @@ export async function getAllMinimalGdocBaseObjects(
                 published,
                 content ->> '$.subtitle' as subtitle,
                 content ->> '$.excerpt' as excerpt,
-                content ->> '$.type' as type,
+                type,
                 content ->> '$."featured-image"' as "featured-image"
             FROM posts_gdocs
             WHERE published = 1
@@ -363,7 +363,7 @@ export async function getAndLoadPublishedDataInsights(
             SELECT *
             FROM posts_gdocs
             WHERE published = 1
-            AND content ->> '$.type' = ?
+            AND type = ?
             AND publishedAt <= NOW()
             ORDER BY publishedAt DESC
             ${limitOffsetClause}`,
@@ -413,7 +413,7 @@ export async function loadPublishedGdocAuthors(
             SELECT *
             FROM posts_gdocs
             WHERE published = 1
-            AND content ->> '$.type' IN (:types)`,
+            AND type IN (:types)`,
         {
             types: [OwidGdocType.Author],
         }

db/model/Gdoc/GdocHomepage.ts

@@ -45,7 +45,7 @@ export class GdocHomepage
             SELECT
                 id
             FROM posts_gdocs
-            WHERE content->>"$.type" = "${OwidGdocType.Homepage}"
+            WHERE type = "${OwidGdocType.Homepage}"
             AND published = TRUE
             AND id != ?`,
             [this.id]

db/model/Post.ts

@@ -100,6 +100,22 @@ export const setTagsForPost = async (
         )
 }
 
+export const getPostIdFromSlug = (
+    knex: db.KnexReadonlyTransaction,
+    slug: string
+): Promise<number | undefined> => {
+    return db
+        .knexRawFirst<{ id: number }>(
+            knex,
+            `-- sql
+        SELECT id
+        FROM posts
+        WHERE slug = ?`,
+            [slug]
+        )
+        .then((result) => result?.id)
+}
+
 export const getPostRawBySlug = async (
     trx: db.KnexReadonlyTransaction,
     slug: string
@@ -363,7 +379,7 @@ export const getWordpressPostReferencesByChartId = async (
                         slug from posts_gdocs pg
                     WHERE
                         pg.slug = p.slug
-                        AND pg.content ->> '$.type' <> 'fragment'
+                        AND pg.type != 'fragment'
                         AND pg.published = 1
                 )
             ORDER BY
@@ -401,7 +417,7 @@ export const getGdocsPostReferencesByChartId = async (
                 )
             WHERE
                 c.id = ?
-                AND pg.content ->> '$.type' NOT IN (
+                AND pg.type NOT IN (
                     '${OwidGdocType.Fragment}',
                     '${OwidGdocType.AboutPage}',
                     '${OwidGdocType.DataInsight}'
@@ -547,7 +563,7 @@ export const getRelatedResearchAndWritingForVariable = async (
             AND cd.variableId = ?
             AND cd.property IN ('x', 'y') -- ignore cases where the indicator is size, color etc
             AND p.published = 1
-            AND p.content ->> '$.type' != 'fragment'`,
+            AND p.type != 'fragment'`,
         [variableId]
     )
 
@@ -596,7 +612,7 @@ export const getLatestWorkByAuthor = async (
         WHERE
             pg.content ->> '$.authors' LIKE ?
             AND pg.published = TRUE
-            AND pg.content->>'$.type' = "${OwidGdocType.Article}"
+            AND pg.type = "${OwidGdocType.Article}"
         `,
         [`%${author}%`]
     )

site/GrapherPage.jsdom.test.tsx

@@ -4,7 +4,6 @@ import { GrapherInterface } from "@ourworldindata/types"
 import {
     DimensionProperty,
     KeyChartLevel,
-    DbEnrichedPost,
     RelatedChart,
 } from "@ourworldindata/utils"
 import React from "react"
@@ -30,21 +29,10 @@ const mockGrapher: GrapherInterface = {
 }
 
 let grapher: GrapherInterface
-let post: DbEnrichedPost
 let relatedCharts: RelatedChart[]
 
 beforeAll(() => {
     grapher = mockGrapher
-    post = {
-        id: 2681,
-        title: "Hunger and Undernourishment",
-        slug: "hunger-and-undernourishment",
-        published_at: "Tue Oct 08 2013 17:22:54 GMT+0000 (GMT)",
-        status: "publish",
-        type: "page",
-        updated_at: "Wed Mar 25 2020 14:11:30 GMT+0000 (GMT)",
-        content: "",
-    } as any
     relatedCharts = [
         {
             title: "Chart 1",
@@ -66,7 +54,6 @@ describe("when the page is rendered", () => {
         () =>
             (view = Enzyme.shallow(
                 <GrapherPage
-                    post={post}
                     grapher={grapher}
                     relatedCharts={relatedCharts}
                     baseGrapherUrl={"/grapher/"}

site/GrapherPage.tsx

@@ -8,7 +8,6 @@ import {
 import {
     flatten,
     PostReference,
-    DbEnrichedPost,
     RelatedChart,
     serializeJSONForHTML,
     GrapherInterface,
@@ -34,7 +33,6 @@ import { SiteHeader } from "./SiteHeader.js"
 
 export const GrapherPage = (props: {
     grapher: GrapherInterface
-    post?: DbEnrichedPost
     relatedCharts?: RelatedChart[]
     relatedArticles?: PostReference[]
     baseUrl: string