From 3300abf00323a6e020f5af44070a54384267748e Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Wed, 31 Jan 2024 17:32:57 +0000 Subject: [PATCH 01/25] feat: use snapshot version of getPostBySlug --- baker/pageOverrides.tsx | 4 ++-- baker/siteRenderers.tsx | 13 +++++++------ db/syncPostsToGrapher.ts | 2 +- db/wpdb.ts | 21 ++++++++++++++++++--- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/baker/pageOverrides.tsx b/baker/pageOverrides.tsx index 17b895fdfde..cc2f67abe96 100644 --- a/baker/pageOverrides.tsx +++ b/baker/pageOverrides.tsx @@ -2,14 +2,14 @@ import { PageOverrides } from "../site/LongFormPage.js" import { BAKED_BASE_URL } from "../settings/serverSettings.js" import { urlToSlug, FullPost, JsonError } from "@ourworldindata/utils" import { FormattingOptions } from "@ourworldindata/types" -import { getPostBySlug, isPostCitable } from "../db/wpdb.js" +import { getPostBySlugFromSnapshot, isPostCitable } from "../db/wpdb.js" import { getTopSubnavigationParentItem } from "../site/SiteSubnavigation.js" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" export const getPostBySlugLogToSlackNoThrow = async (slug: string) => { let post try { - post = await getPostBySlug(slug) + post = await getPostBySlugFromSnapshot(slug) } catch (err) { logErrorAndMaybeSendToBugsnag(err) } finally { diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index a43dabcb1da..cf8b757a2b1 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -63,9 +63,9 @@ import { formatPost } from "./formatWordpressPost.js" import { getBlogIndex, getLatestPostRevision, - getPostBySlug, isPostCitable, getBlockContent, + getPostBySlugFromSnapshot, } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" @@ -190,7 +190,7 @@ export const renderGdoc = (gdoc: OwidGdoc, isPreviewing: boolean = false) => { } export const renderPageBySlug = async (slug: string) => { - const post = await getPostBySlug(slug) + const post = await getPostBySlugFromSnapshot(slug) return renderPost(post) } @@ -480,7 +480,7 @@ const getCountryProfilePost = memoize( grapherExports?: GrapherExports ): Promise<[FormattedPost, FormattingOptions]> => { // Get formatted content from generic covid country profile page. - const genericCountryProfilePost = await getPostBySlug( + const genericCountryProfilePost = await getPostBySlugFromSnapshot( profileSpec.genericProfileSlug ) @@ -500,7 +500,7 @@ const getCountryProfilePost = memoize( // todo: we used to flush cache of this thing. const getCountryProfileLandingPost = memoize( async (profileSpec: CountryProfileSpec) => { - return getPostBySlug(profileSpec.landingPageSlug) + return getPostBySlugFromSnapshot(profileSpec.landingPageSlug) } ) @@ -559,7 +559,7 @@ const renderPostThumbnailBySlug = async ( let post try { - post = await getPostBySlug(slug) + post = await getPostBySlugFromSnapshot(slug) } catch (err) { // if no post is found, then we return early instead of throwing } @@ -599,7 +599,8 @@ export const renderProminentLinks = async ( ? (await Chart.getBySlug(resolvedUrl.slug))?.config ?.title // optim? : resolvedUrl.slug && - (await getPostBySlug(resolvedUrl.slug)).title) + (await getPostBySlugFromSnapshot(resolvedUrl.slug)) + .title) } finally { if (!title) { logErrorAndMaybeSendToBugsnag( diff --git a/db/syncPostsToGrapher.ts b/db/syncPostsToGrapher.ts index b66a7a34f4a..dd1130c2dff 100644 --- a/db/syncPostsToGrapher.ts +++ b/db/syncPostsToGrapher.ts @@ -351,7 +351,7 @@ const syncPostsToGrapher = async (): Promise => { wpApiSnapshot: post.post_type === "wp_block" ? await wpdb.getBlockApi(post.ID) - : await wpdb.getPostApiBySlug(post.post_name), + : await wpdb.getPostApiBySlugFromApi(post.post_name), featured_image: post.featured_image || "", published_at: post.post_date_gmt === zeroDateString diff --git a/db/wpdb.ts b/db/wpdb.ts index 41ecf2483b4..5df2f18e5da 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -48,6 +48,7 @@ import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" import { SiteNavigationStatic } from "../site/SiteNavigation.js" +import { getPostEnrichedBySlug } from "./model/Post.js" let _knexInstance: Knex @@ -446,7 +447,9 @@ export const getPostIdAndTypeBySlug = async ( return { id: rows[0].ID, type: rows[0].post_type } } -export const getPostApiBySlug = async (slug: string): Promise => { +export const getPostApiBySlugFromApi = async ( + slug: string +): Promise => { if (!isWordpressAPIEnabled) { throw new JsonError(`Need wordpress API to match slug ${slug}`, 404) } @@ -462,16 +465,28 @@ export const getPostApiBySlug = async (slug: string): Promise => { // We might want to cache this as the network of prominent links densifies and // multiple requests to the same posts are happening. -export const getPostBySlug = async (slug: string): Promise => { +export const DEPRECATEDgetPostBySlugFromApi = async ( + slug: string +): Promise => { if (!isWordpressAPIEnabled) { throw new JsonError(`Need wordpress API to match slug ${slug}`, 404) } - const postApi = await getPostApiBySlug(slug) + const postApi = await getPostApiBySlugFromApi(slug) return getFullPost(postApi) } +export const getPostBySlugFromSnapshot = async ( + slug: string +): Promise => { + const postEnriched = await getPostEnrichedBySlug(slug) + if (!postEnriched?.wpApiSnapshot) + throw new JsonError(`No page snapshot found by slug ${slug}`, 404) + + return getFullPost(postEnriched.wpApiSnapshot) +} + // the /revisions endpoint does not send back all the metadata required for // the proper rendering of the post (e.g. authors), hence the double request. export const getLatestPostRevision = async (id: number): Promise => { From 5c3fef8480ceb88077fcce67459e1d2639b4bb6a Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 14:07:52 +0000 Subject: [PATCH 02/25] refactor(wp): replace getPosts with snapshot version --- db/wpdb.ts | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/db/wpdb.ts b/db/wpdb.ts index 5df2f18e5da..29b5a70afd9 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -39,7 +39,12 @@ import { Tag, OwidGdocPostInterface, } from "@ourworldindata/utils" -import { OwidGdocLinkType, Topic } from "@ourworldindata/types" +import { + DbRawPost, + OwidGdocLinkType, + Topic, + parsePostRow, +} from "@ourworldindata/types" import { getContentGraph, WPPostTypeToGraphDocumentType, @@ -48,7 +53,7 @@ import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" import { SiteNavigationStatic } from "../site/SiteNavigation.js" -import { getPostEnrichedBySlug } from "./model/Post.js" +import { getPostEnrichedBySlug, postsTable } from "./model/Post.js" let _knexInstance: Knex @@ -360,7 +365,7 @@ export const selectHomepagePosts: FilterFnPostRestApi = (post) => // When passing multiple post types, the limit is applied to the resulting array // of sequentially sorted posts (all blog posts, then all pages, ...), so there // will be a predominance of a certain post type. -export const getPosts = async ( +export const DEPRECATEDgetPosts = async ( postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], filterFunc?: FilterFnPostRestApi, limit?: number @@ -410,6 +415,39 @@ export const getPosts = async ( return limit ? filteredPosts.slice(0, limit) : filteredPosts } +export const getPosts = async ( + postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], + filterFunc?: FilterFnPostRestApi +): Promise => { + const rawPosts: DbRawPost[] = ( + await db.knexInstance().raw( + ` + SELECT * FROM ${postsTable} + WHERE status = "publish" + AND type IN (?) + ORDER BY JSON_UNQUOTE(JSON_EXTRACT(wpApiSnapshot, '$.date')) DESC; + `, + [postTypes].join(",") + ) + )[0] + + const posts = rawPosts + .map(parsePostRow) + .map((p) => p.wpApiSnapshot) + .filter((p) => p !== null) as PostRestApi[] + + // Published pages excluded from public views + const excludedSlugs = [BLOG_SLUG] + + const filterConditions: Array = [ + (post): boolean => !excludedSlugs.includes(post.slug), + (post): boolean => !post.slug.endsWith("-country-profile"), + ] + if (filterFunc) filterConditions.push(filterFunc) + + return posts.filter((post) => filterConditions.every((c) => c(post))) +} + // todo / refactor : narrow down scope to getPostTypeById? export const getPostType = async (search: number | string): Promise => { const paramName = typeof search === "number" ? "id" : "slug" From ee00120b5617872a3e432b8d100b4f99e92b31dd Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 14:24:03 +0000 Subject: [PATCH 03/25] refactor(wp): move deprecated code to own file for clarity --- db/DEPRECATEDwpdb.ts | 93 ++++++++++++++++++++++++++++++++++++++++++++ db/wpdb.ts | 78 +++---------------------------------- 2 files changed, 98 insertions(+), 73 deletions(-) create mode 100644 db/DEPRECATEDwpdb.ts diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts new file mode 100644 index 00000000000..f5da87f76e2 --- /dev/null +++ b/db/DEPRECATEDwpdb.ts @@ -0,0 +1,93 @@ +/** + * + * DO NOT USE - DEPRECATED - FOR DOCUMENTATION PURPOSES ONLY + * + * Note: This file contains now deprecated functions that were querying the Wordpress + * tables or APIs. It is kept around for easier reference during the transition period + * but should not be used for new code. + */ + +import { + WP_PostType, + FilterFnPostRestApi, + PostRestApi, + FullPost, + JsonError, +} from "@ourworldindata/types" +import { BLOG_SLUG } from "../settings/serverSettings.js" +import { + WP_API_ENDPOINT, + apiQuery, + getEndpointSlugFromType, + getFullPost, + getPostApiBySlugFromApi, + isWordpressAPIEnabled, +} from "./wpdb.js" + +// Limit not supported with multiple post types: When passing multiple post +// types, the limit is applied to the resulting array of sequentially sorted +// posts (all blog posts, then all pages, ...), so there will be a predominance +// of a certain post type. +export const DEPRECATEDgetPosts = async ( + postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], + filterFunc?: FilterFnPostRestApi, + limit?: number +): Promise => { + if (!isWordpressAPIEnabled) return [] + + const perPage = 20 + const posts: PostRestApi[] = [] + + for (const postType of postTypes) { + const endpoint = `${WP_API_ENDPOINT}/${getEndpointSlugFromType( + postType + )}` + + // Get number of items to retrieve + const headers = await apiQuery(endpoint, { + searchParams: [["per_page", 1]], + returnResponseHeadersOnly: true, + }) + const maxAvailable = headers.get("X-WP-TotalPages") + const count = limit && limit < maxAvailable ? limit : maxAvailable + + for (let page = 1; page <= Math.ceil(count / perPage); page++) { + const postsCurrentPage = await apiQuery(endpoint, { + searchParams: [ + ["per_page", perPage], + ["page", page], + ], + }) + posts.push(...postsCurrentPage) + } + } + + // Published pages excluded from public views + const excludedSlugs = [BLOG_SLUG] + + const filterConditions: Array = [ + (post): boolean => !excludedSlugs.includes(post.slug), + (post): boolean => !post.slug.endsWith("-country-profile"), + ] + if (filterFunc) filterConditions.push(filterFunc) + + const filteredPosts = posts.filter((post) => + filterConditions.every((c) => c(post)) + ) + + return limit ? filteredPosts.slice(0, limit) : filteredPosts +} + +// We might want to cache this as the network of prominent links densifies and +// multiple requests to the same posts are happening. +export const DEPRECATEDgetPostBySlugFromApi = async ( + slug: string +): Promise => { + if (!isWordpressAPIEnabled) { + throw new JsonError(`Need wordpress API to match slug ${slug}`, 404) + } + + const postApi = await getPostApiBySlugFromApi(slug) + + return getFullPost(postApi) +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 29b5a70afd9..c8b3fd26da3 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -123,9 +123,9 @@ class WPDB { export const singleton = new WPDB() -const WP_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/wp/v2` -const OWID_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/owid/v1` -const WP_GRAPHQL_ENDPOINT = `${WORDPRESS_URL}/wp/graphql` +export const WP_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/wp/v2` +export const OWID_API_ENDPOINT = `${WORDPRESS_URL}/wp-json/owid/v1` +export const WP_GRAPHQL_ENDPOINT = `${WORDPRESS_URL}/wp/graphql` export const ENTRIES_CATEGORY_ID = 44 @@ -161,7 +161,7 @@ const graphqlQuery = async ( * * Note: throws on response.status >= 200 && response.status < 300. */ -const apiQuery = async ( +export const apiQuery = async ( endpoint: string, params?: { returnResponseHeadersOnly?: boolean @@ -356,65 +356,11 @@ export const getFeaturedImages = async (): Promise> => { } // page => pages, post => posts -const getEndpointSlugFromType = (type: string): string => `${type}s` +export const getEndpointSlugFromType = (type: string): string => `${type}s` export const selectHomepagePosts: FilterFnPostRestApi = (post) => post.meta?.owid_publication_context_meta_field?.homepage === true -// Limit not supported with multiple post types: -// When passing multiple post types, the limit is applied to the resulting array -// of sequentially sorted posts (all blog posts, then all pages, ...), so there -// will be a predominance of a certain post type. -export const DEPRECATEDgetPosts = async ( - postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], - filterFunc?: FilterFnPostRestApi, - limit?: number -): Promise => { - if (!isWordpressAPIEnabled) return [] - - const perPage = 20 - const posts: PostRestApi[] = [] - - for (const postType of postTypes) { - const endpoint = `${WP_API_ENDPOINT}/${getEndpointSlugFromType( - postType - )}` - - // Get number of items to retrieve - const headers = await apiQuery(endpoint, { - searchParams: [["per_page", 1]], - returnResponseHeadersOnly: true, - }) - const maxAvailable = headers.get("X-WP-TotalPages") - const count = limit && limit < maxAvailable ? limit : maxAvailable - - for (let page = 1; page <= Math.ceil(count / perPage); page++) { - const postsCurrentPage = await apiQuery(endpoint, { - searchParams: [ - ["per_page", perPage], - ["page", page], - ], - }) - posts.push(...postsCurrentPage) - } - } - - // Published pages excluded from public views - const excludedSlugs = [BLOG_SLUG] - - const filterConditions: Array = [ - (post): boolean => !excludedSlugs.includes(post.slug), - (post): boolean => !post.slug.endsWith("-country-profile"), - ] - if (filterFunc) filterConditions.push(filterFunc) - - const filteredPosts = posts.filter((post) => - filterConditions.every((c) => c(post)) - ) - - return limit ? filteredPosts.slice(0, limit) : filteredPosts -} - export const getPosts = async ( postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], filterFunc?: FilterFnPostRestApi @@ -501,20 +447,6 @@ export const getPostApiBySlugFromApi = async ( return apiQuery(`${WP_API_ENDPOINT}/${getEndpointSlugFromType(type)}/${id}`) } -// We might want to cache this as the network of prominent links densifies and -// multiple requests to the same posts are happening. -export const DEPRECATEDgetPostBySlugFromApi = async ( - slug: string -): Promise => { - if (!isWordpressAPIEnabled) { - throw new JsonError(`Need wordpress API to match slug ${slug}`, 404) - } - - const postApi = await getPostApiBySlugFromApi(slug) - - return getFullPost(postApi) -} - export const getPostBySlugFromSnapshot = async ( slug: string ): Promise => { From e56f0f2ace67f2289f3f8327887e236957fbfe34 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 20:16:03 +0000 Subject: [PATCH 04/25] refactor(wp): replace getLatestPostRevision with snapshot version --- baker/siteRenderers.tsx | 4 ++-- db/DEPRECATEDwpdb.ts | 32 ++++++++++++++++++++++++++++++++ db/model/Post.ts | 15 ++++++++++++++- db/wpdb.ts | 39 ++++++++++++--------------------------- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index cf8b757a2b1..a87b32edb2b 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -62,10 +62,10 @@ import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" import { getBlogIndex, - getLatestPostRevision, isPostCitable, getBlockContent, getPostBySlugFromSnapshot, + getPostByIdFromSnapshot, } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" @@ -195,7 +195,7 @@ export const renderPageBySlug = async (slug: string) => { } export const renderPreview = async (postId: number): Promise => { - const postApi = await getLatestPostRevision(postId) + const postApi = await getPostByIdFromSnapshot(postId) return renderPost(postApi) } diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index f5da87f76e2..97e15d66f7a 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -21,6 +21,7 @@ import { getEndpointSlugFromType, getFullPost, getPostApiBySlugFromApi, + getPostType, isWordpressAPIEnabled, } from "./wpdb.js" @@ -91,3 +92,34 @@ export const DEPRECATEDgetPostBySlugFromApi = async ( return getFullPost(postApi) } + +// the /revisions endpoint does not send back all the metadata required for +// the proper rendering of the post (e.g. authors), hence the double request. +export const DEPRECATEDgetLatestPostRevision = async ( + id: number +): Promise => { + const type = await getPostType(id) + const endpointSlug = getEndpointSlugFromType(type) + + const postApi = await apiQuery(`${WP_API_ENDPOINT}/${endpointSlug}/${id}`) + + const revision = ( + await apiQuery( + `${WP_API_ENDPOINT}/${endpointSlug}/${id}/revisions?per_page=1` + ) + )[0] + + // Since WP does not store metadata for revisions, some elements of a + // previewed page will not reflect the latest edits: + // - published date (will show the correct one - that is the one in the + // sidebar - for unpublished posts though. For published posts, the + // current published date is displayed, regardless of what is shown + // and could have been modified in the sidebar.) + // - authors + // ... + return getFullPost({ + ...postApi, + content: revision.content, + title: revision.title, + }) +} diff --git a/db/model/Post.ts b/db/model/Post.ts index f9a4247f844..e25e6057954 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -63,7 +63,12 @@ export const setTagsForPost = async ( export const getPostRawBySlug = async ( slug: string ): Promise => - (await db.knexTable("posts").where({ slug: slug }))[0] + (await db.knexTable(postsTable).where({ slug }))[0] + +export const getPostRawById = async ( + id: number +): Promise => + (await db.knexTable(postsTable).where({ id }))[0] export const getPostEnrichedBySlug = async ( slug: string @@ -72,3 +77,11 @@ export const getPostEnrichedBySlug = async ( if (!post) return undefined return parsePostRow(post) } + +export const getPostEnrichedById = async ( + id: number +): Promise => { + const post = await getPostRawById(id) + if (!post) return undefined + return parsePostRow(post) +} diff --git a/db/wpdb.ts b/db/wpdb.ts index c8b3fd26da3..1977ba25d0a 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -53,7 +53,11 @@ import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" import { SiteNavigationStatic } from "../site/SiteNavigation.js" -import { getPostEnrichedBySlug, postsTable } from "./model/Post.js" +import { + getPostEnrichedById, + getPostEnrichedBySlug, + postsTable, +} from "./model/Post.js" let _knexInstance: Knex @@ -457,33 +461,14 @@ export const getPostBySlugFromSnapshot = async ( return getFullPost(postEnriched.wpApiSnapshot) } -// the /revisions endpoint does not send back all the metadata required for -// the proper rendering of the post (e.g. authors), hence the double request. -export const getLatestPostRevision = async (id: number): Promise => { - const type = await getPostType(id) - const endpointSlug = getEndpointSlugFromType(type) - - const postApi = await apiQuery(`${WP_API_ENDPOINT}/${endpointSlug}/${id}`) - - const revision = ( - await apiQuery( - `${WP_API_ENDPOINT}/${endpointSlug}/${id}/revisions?per_page=1` - ) - )[0] +export const getPostByIdFromSnapshot = async ( + id: number +): Promise => { + const postEnriched = await getPostEnrichedById(id) + if (!postEnriched?.wpApiSnapshot) + throw new JsonError(`No page snapshot found by id ${id}`, 404) - // Since WP does not store metadata for revisions, some elements of a - // previewed page will not reflect the latest edits: - // - published date (will show the correct one - that is the one in the - // sidebar - for unpublished posts though. For published posts, the - // current published date is displayed, regardless of what is shown - // and could have been modified in the sidebar.) - // - authors - // ... - return getFullPost({ - ...postApi, - content: revision.content, - title: revision.title, - }) + return getFullPost(postEnriched.wpApiSnapshot) } export const getRelatedCharts = async ( From 522e6de892455e0b32ec731ea79c726efa552a65 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 20:23:24 +0000 Subject: [PATCH 05/25] refactor(wp): move snapshot functions to Post model --- baker/pageOverrides.tsx | 5 +++-- baker/siteRenderers.tsx | 31 ++++++++++++++++--------------- db/model/Post.ts | 29 ++++++++++++++++++++++++++++- db/wpdb.ts | 26 +------------------------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/baker/pageOverrides.tsx b/baker/pageOverrides.tsx index cc2f67abe96..e638ac89c06 100644 --- a/baker/pageOverrides.tsx +++ b/baker/pageOverrides.tsx @@ -2,14 +2,15 @@ import { PageOverrides } from "../site/LongFormPage.js" import { BAKED_BASE_URL } from "../settings/serverSettings.js" import { urlToSlug, FullPost, JsonError } from "@ourworldindata/utils" import { FormattingOptions } from "@ourworldindata/types" -import { getPostBySlugFromSnapshot, isPostCitable } from "../db/wpdb.js" +import { isPostCitable } from "../db/wpdb.js" import { getTopSubnavigationParentItem } from "../site/SiteSubnavigation.js" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" +import { getFullPostBySlugFromSnapshot } from "../db/model/Post.js" export const getPostBySlugLogToSlackNoThrow = async (slug: string) => { let post try { - post = await getPostBySlugFromSnapshot(slug) + post = await getFullPostBySlugFromSnapshot(slug) } catch (err) { logErrorAndMaybeSendToBugsnag(err) } finally { diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index a87b32edb2b..2b5942f1da3 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -60,13 +60,7 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { - getBlogIndex, - isPostCitable, - getBlockContent, - getPostBySlugFromSnapshot, - getPostByIdFromSnapshot, -} from "../db/wpdb.js" +import { getBlogIndex, isPostCitable, getBlockContent } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" @@ -87,7 +81,11 @@ import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer. import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" import { ExplorerFullQueryParams } from "../explorer/ExplorerConstants.js" import { resolveInternalRedirect } from "./redirects.js" -import { postsTable } from "../db/model/Post.js" +import { + getFullPostByIdFromSnapshot, + getFullPostBySlugFromSnapshot, + postsTable, +} from "../db/model/Post.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" import { GdocFactory } from "../db/model/Gdoc/GdocFactory.js" @@ -190,12 +188,12 @@ export const renderGdoc = (gdoc: OwidGdoc, isPreviewing: boolean = false) => { } export const renderPageBySlug = async (slug: string) => { - const post = await getPostBySlugFromSnapshot(slug) + const post = await getFullPostBySlugFromSnapshot(slug) return renderPost(post) } export const renderPreview = async (postId: number): Promise => { - const postApi = await getPostByIdFromSnapshot(postId) + const postApi = await getFullPostByIdFromSnapshot(postId) return renderPost(postApi) } @@ -480,7 +478,7 @@ const getCountryProfilePost = memoize( grapherExports?: GrapherExports ): Promise<[FormattedPost, FormattingOptions]> => { // Get formatted content from generic covid country profile page. - const genericCountryProfilePost = await getPostBySlugFromSnapshot( + const genericCountryProfilePost = await getFullPostBySlugFromSnapshot( profileSpec.genericProfileSlug ) @@ -500,7 +498,7 @@ const getCountryProfilePost = memoize( // todo: we used to flush cache of this thing. const getCountryProfileLandingPost = memoize( async (profileSpec: CountryProfileSpec) => { - return getPostBySlugFromSnapshot(profileSpec.landingPageSlug) + return getFullPostBySlugFromSnapshot(profileSpec.landingPageSlug) } ) @@ -559,7 +557,7 @@ const renderPostThumbnailBySlug = async ( let post try { - post = await getPostBySlugFromSnapshot(slug) + post = await getFullPostBySlugFromSnapshot(slug) } catch (err) { // if no post is found, then we return early instead of throwing } @@ -599,8 +597,11 @@ export const renderProminentLinks = async ( ? (await Chart.getBySlug(resolvedUrl.slug))?.config ?.title // optim? : resolvedUrl.slug && - (await getPostBySlugFromSnapshot(resolvedUrl.slug)) - .title) + ( + await getFullPostBySlugFromSnapshot( + resolvedUrl.slug + ) + ).title) } finally { if (!title) { logErrorAndMaybeSendToBugsnag( diff --git a/db/model/Post.ts b/db/model/Post.ts index e25e6057954..97b2b4d85b2 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -1,6 +1,13 @@ import * as db from "../db.js" import { Knex } from "knex" -import { DbEnrichedPost, DbRawPost, parsePostRow } from "@ourworldindata/utils" +import { + DbEnrichedPost, + DbRawPost, + FullPost, + JsonError, + parsePostRow, +} from "@ourworldindata/utils" +import { getFullPost } from "../wpdb.js" export const postsTable = "posts" @@ -85,3 +92,23 @@ export const getPostEnrichedById = async ( if (!post) return undefined return parsePostRow(post) } + +export const getFullPostBySlugFromSnapshot = async ( + slug: string +): Promise => { + const postEnriched = await getPostEnrichedBySlug(slug) + if (!postEnriched?.wpApiSnapshot) + throw new JsonError(`No page snapshot found by slug ${slug}`, 404) + + return getFullPost(postEnriched.wpApiSnapshot) +} + +export const getFullPostByIdFromSnapshot = async ( + id: number +): Promise => { + const postEnriched = await getPostEnrichedById(id) + if (!postEnriched?.wpApiSnapshot) + throw new JsonError(`No page snapshot found by id ${id}`, 404) + + return getFullPost(postEnriched.wpApiSnapshot) +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 1977ba25d0a..baec80ba1f5 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -53,11 +53,7 @@ import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" import { SiteNavigationStatic } from "../site/SiteNavigation.js" -import { - getPostEnrichedById, - getPostEnrichedBySlug, - postsTable, -} from "./model/Post.js" +import { postsTable } from "./model/Post.js" let _knexInstance: Knex @@ -451,26 +447,6 @@ export const getPostApiBySlugFromApi = async ( return apiQuery(`${WP_API_ENDPOINT}/${getEndpointSlugFromType(type)}/${id}`) } -export const getPostBySlugFromSnapshot = async ( - slug: string -): Promise => { - const postEnriched = await getPostEnrichedBySlug(slug) - if (!postEnriched?.wpApiSnapshot) - throw new JsonError(`No page snapshot found by slug ${slug}`, 404) - - return getFullPost(postEnriched.wpApiSnapshot) -} - -export const getPostByIdFromSnapshot = async ( - id: number -): Promise => { - const postEnriched = await getPostEnrichedById(id) - if (!postEnriched?.wpApiSnapshot) - throw new JsonError(`No page snapshot found by id ${id}`, 404) - - return getFullPost(postEnriched.wpApiSnapshot) -} - export const getRelatedCharts = async ( postId: number ): Promise => From f843b199c60e96165bb1a366af0311f93041fb20 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 20:28:50 +0000 Subject: [PATCH 06/25] refactor(wp): move unused code to deprecated file --- db/DEPRECATEDwpdb.ts | 57 ++++++++++++++++++++++++++++++++++++++++++++ db/wpdb.ts | 53 ---------------------------------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index 97e15d66f7a..67def9f48f0 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -23,6 +23,7 @@ import { getPostApiBySlugFromApi, getPostType, isWordpressAPIEnabled, + singleton, } from "./wpdb.js" // Limit not supported with multiple post types: When passing multiple post @@ -123,3 +124,59 @@ export const DEPRECATEDgetLatestPostRevision = async ( title: revision.title, }) } + +export const DEPRECATEDgetTagsByPostId = async (): Promise< + Map +> => { + const tagsByPostId = new Map() + const rows = await singleton.query(` + SELECT p.id, t.name + FROM wp_posts p + JOIN wp_term_relationships tr + on (p.id=tr.object_id) + JOIN wp_term_taxonomy tt + on (tt.term_taxonomy_id=tr.term_taxonomy_id + and tt.taxonomy='post_tag') + JOIN wp_terms t + on (tt.term_id=t.term_id) + `) + + for (const row of rows) { + let cats = tagsByPostId.get(row.id) + if (!cats) { + cats = [] + tagsByPostId.set(row.id, cats) + } + cats.push(row.name) + } + + return tagsByPostId +} + +// Retrieve a map of post ids to authors +export let cachedAuthorship: Map | undefined +export const DEPRECATEDgetAuthorship = async (): Promise< + Map +> => { + if (cachedAuthorship) return cachedAuthorship + + const authorRows = await singleton.query(` + SELECT object_id, terms.description FROM wp_term_relationships AS rels + LEFT JOIN wp_term_taxonomy AS terms ON terms.term_taxonomy_id=rels.term_taxonomy_id + WHERE terms.taxonomy='author' + ORDER BY rels.term_order ASC + `) + + const authorship = new Map() + for (const row of authorRows) { + let authors = authorship.get(row.object_id) + if (!authors) { + authors = [] + authorship.set(row.object_id, authors) + } + authors.push(row.description.split(" ").slice(0, 2).join(" ")) + } + + cachedAuthorship = authorship + return authorship +} diff --git a/db/wpdb.ts b/db/wpdb.ts index baec80ba1f5..8c2faf1607f 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -194,58 +194,6 @@ export const apiQuery = async ( : response.json() } -// Retrieve a map of post ids to authors -let cachedAuthorship: Map | undefined -export const getAuthorship = async (): Promise> => { - if (cachedAuthorship) return cachedAuthorship - - const authorRows = await singleton.query(` - SELECT object_id, terms.description FROM wp_term_relationships AS rels - LEFT JOIN wp_term_taxonomy AS terms ON terms.term_taxonomy_id=rels.term_taxonomy_id - WHERE terms.taxonomy='author' - ORDER BY rels.term_order ASC - `) - - const authorship = new Map() - for (const row of authorRows) { - let authors = authorship.get(row.object_id) - if (!authors) { - authors = [] - authorship.set(row.object_id, authors) - } - authors.push(row.description.split(" ").slice(0, 2).join(" ")) - } - - cachedAuthorship = authorship - return authorship -} - -export const getTagsByPostId = async (): Promise> => { - const tagsByPostId = new Map() - const rows = await singleton.query(` - SELECT p.id, t.name - FROM wp_posts p - JOIN wp_term_relationships tr - on (p.id=tr.object_id) - JOIN wp_term_taxonomy tt - on (tt.term_taxonomy_id=tr.term_taxonomy_id - and tt.taxonomy='post_tag') - JOIN wp_terms t - on (tt.term_id=t.term_id) - `) - - for (const row of rows) { - let cats = tagsByPostId.get(row.id) - if (!cats) { - cats = [] - tagsByPostId.set(row.id, cats) - } - cats.push(row.name) - } - - return tagsByPostId -} - export const getDocumentsInfo = async ( type: WP_PostType, cursor: string = "", @@ -847,7 +795,6 @@ export const getTables = async (): Promise> => { } export const flushCache = (): void => { - cachedAuthorship = undefined cachedFeaturedImages = undefined getBlogIndex.cache.clear?.() cachedTables = undefined From d574ebd6c1e86f6e16cd2172c337beb31f9d0ad3 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 20:34:58 +0000 Subject: [PATCH 07/25] refactor(wp): isPostSlugCitable only need slug --- baker/pageOverrides.tsx | 8 +++++--- baker/siteRenderers.tsx | 6 ++++-- db/migrateWpPostsToArchieMl.ts | 8 +++----- db/model/Post.ts | 18 ++++++++++++++++++ db/wpdb.ts | 19 ------------------- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/baker/pageOverrides.tsx b/baker/pageOverrides.tsx index e638ac89c06..a21cc1e3b8b 100644 --- a/baker/pageOverrides.tsx +++ b/baker/pageOverrides.tsx @@ -2,10 +2,12 @@ import { PageOverrides } from "../site/LongFormPage.js" import { BAKED_BASE_URL } from "../settings/serverSettings.js" import { urlToSlug, FullPost, JsonError } from "@ourworldindata/utils" import { FormattingOptions } from "@ourworldindata/types" -import { isPostCitable } from "../db/wpdb.js" import { getTopSubnavigationParentItem } from "../site/SiteSubnavigation.js" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" -import { getFullPostBySlugFromSnapshot } from "../db/model/Post.js" +import { + getFullPostBySlugFromSnapshot, + isPostSlugCitable, +} from "../db/model/Post.js" export const getPostBySlugLogToSlackNoThrow = async (slug: string) => { let post @@ -62,7 +64,7 @@ export const getPageOverrides = async ( const landing = await getLandingOnlyIfParent(post, formattingOptions) if (!landing) return - const isParentLandingCitable = await isPostCitable(landing) + const isParentLandingCitable = await isPostSlugCitable(landing.slug) if (!isParentLandingCitable) return return { diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 2b5942f1da3..3e706f781f1 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -60,7 +60,7 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { getBlogIndex, isPostCitable, getBlockContent } from "../db/wpdb.js" +import { getBlogIndex, getBlockContent } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" @@ -84,6 +84,7 @@ import { resolveInternalRedirect } from "./redirects.js" import { getFullPostByIdFromSnapshot, getFullPostBySlugFromSnapshot, + isPostSlugCitable, postsTable, } from "../db/model/Post.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" @@ -227,7 +228,8 @@ export const renderPost = async ( const pageOverrides = await getPageOverrides(post, formattingOptions) const citationStatus = - (await isPostCitable(post)) || isPageOverridesCitable(pageOverrides) + (await isPostSlugCitable(post.slug)) || + isPageOverridesCitable(pageOverrides) return renderToHtmlPage( => { relatedCharts = await getRelatedCharts(post.id) } - const shouldIncludeMaxAsAuthor = await isPostCitable( - // Only needs post.slug - post as any - ) + const shouldIncludeMaxAsAuthor = await isPostSlugCitable(post.slug) if ( shouldIncludeMaxAsAuthor && post.authors && diff --git a/db/model/Post.ts b/db/model/Post.ts index 97b2b4d85b2..010e6cd6ea7 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -1,6 +1,7 @@ import * as db from "../db.js" import { Knex } from "knex" import { + CategoryWithEntries, DbEnrichedPost, DbRawPost, FullPost, @@ -8,6 +9,7 @@ import { parsePostRow, } from "@ourworldindata/utils" import { getFullPost } from "../wpdb.js" +import { SiteNavigationStatic } from "../../site/SiteNavigation.js" export const postsTable = "posts" @@ -112,3 +114,19 @@ export const getFullPostByIdFromSnapshot = async ( return getFullPost(postEnriched.wpApiSnapshot) } + +export const isPostSlugCitable = async (slug: string): Promise => { + const entries = SiteNavigationStatic.categories + return entries.some((category) => { + return ( + category.entries.some((entry) => entry.slug === slug) || + (category.subcategories ?? []).some( + (subcategory: CategoryWithEntries) => { + return subcategory.entries.some( + (subCategoryEntry) => subCategoryEntry.slug === slug + ) + } + ) + ) + }) +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 8c2faf1607f..838f781ad85 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -18,7 +18,6 @@ import { Base64 } from "js-base64" import { registerExitHandler } from "./cleanup.js" import { RelatedChart, - CategoryWithEntries, FullPost, WP_PostType, DocumentNode, @@ -52,7 +51,6 @@ import { import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" -import { SiteNavigationStatic } from "../site/SiteNavigation.js" import { postsTable } from "./model/Post.js" let _knexInstance: Knex @@ -260,23 +258,6 @@ export const getDocumentsInfo = async ( } } -export const isPostCitable = async (post: FullPost): Promise => { - const entries = SiteNavigationStatic.categories - return entries.some((category) => { - return ( - category.entries.some((entry) => entry.slug === post.slug) || - (category.subcategories ?? []).some( - (subcategory: CategoryWithEntries) => { - return subcategory.entries.some( - (subCategoryEntry) => - subCategoryEntry.slug === post.slug - ) - } - ) - ) - }) -} - export const getPermalinks = async (): Promise<{ // Strip trailing slashes, and convert __ into / to allow custom subdirs like /about/media-coverage get: (ID: number, postName: string) => string From 4a75d0a0fabbf53a49edb9a5765336506a4011a1 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 20:43:37 +0000 Subject: [PATCH 08/25] refactor(wp): move more unused code to deprecated file --- db/DEPRECATEDwpdb.ts | 35 +++++++++++++++++++++++++++++++---- db/wpdb.ts | 26 -------------------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index 67def9f48f0..2841fe36673 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -18,12 +18,12 @@ import { BLOG_SLUG } from "../settings/serverSettings.js" import { WP_API_ENDPOINT, apiQuery, - getEndpointSlugFromType, getFullPost, getPostApiBySlugFromApi, - getPostType, isWordpressAPIEnabled, singleton, + OWID_API_ENDPOINT, + getEndpointSlugFromType, } from "./wpdb.js" // Limit not supported with multiple post types: When passing multiple post @@ -99,7 +99,7 @@ export const DEPRECATEDgetPostBySlugFromApi = async ( export const DEPRECATEDgetLatestPostRevision = async ( id: number ): Promise => { - const type = await getPostType(id) + const type = await DEPRECATEDgetPostType(id) const endpointSlug = getEndpointSlugFromType(type) const postApi = await apiQuery(`${WP_API_ENDPOINT}/${endpointSlug}/${id}`) @@ -154,7 +154,7 @@ export const DEPRECATEDgetTagsByPostId = async (): Promise< } // Retrieve a map of post ids to authors -export let cachedAuthorship: Map | undefined +let cachedAuthorship: Map | undefined export const DEPRECATEDgetAuthorship = async (): Promise< Map > => { @@ -180,3 +180,30 @@ export const DEPRECATEDgetAuthorship = async (): Promise< cachedAuthorship = authorship return authorship } + +// todo / refactor : narrow down scope to getPostTypeById? +export const DEPRECATEDgetPostType = async ( + search: number | string +): Promise => { + const paramName = typeof search === "number" ? "id" : "slug" + return apiQuery(`${OWID_API_ENDPOINT}/type`, { + searchParams: [[paramName, search]], + }) +} + +let cachedFeaturedImages: Map | undefined +export const getFeaturedImages = async (): Promise> => { + if (cachedFeaturedImages) return cachedFeaturedImages + + const rows = await singleton.query( + `SELECT wp_postmeta.post_id, wp_posts.guid FROM wp_postmeta INNER JOIN wp_posts ON wp_posts.ID=wp_postmeta.meta_value WHERE wp_postmeta.meta_key='_thumbnail_id'` + ) + + const featuredImages = new Map() + for (const row of rows) { + featuredImages.set(row.post_id, row.guid) + } + + cachedFeaturedImages = featuredImages + return featuredImages +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 838f781ad85..82cbc431546 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -267,23 +267,6 @@ export const getPermalinks = async (): Promise<{ postName.replace(/\/+$/g, "").replace(/--/g, "/").replace(/__/g, "/"), }) -let cachedFeaturedImages: Map | undefined -export const getFeaturedImages = async (): Promise> => { - if (cachedFeaturedImages) return cachedFeaturedImages - - const rows = await singleton.query( - `SELECT wp_postmeta.post_id, wp_posts.guid FROM wp_postmeta INNER JOIN wp_posts ON wp_posts.ID=wp_postmeta.meta_value WHERE wp_postmeta.meta_key='_thumbnail_id'` - ) - - const featuredImages = new Map() - for (const row of rows) { - featuredImages.set(row.post_id, row.guid) - } - - cachedFeaturedImages = featuredImages - return featuredImages -} - // page => pages, post => posts export const getEndpointSlugFromType = (type: string): string => `${type}s` @@ -323,14 +306,6 @@ export const getPosts = async ( return posts.filter((post) => filterConditions.every((c) => c(post))) } -// todo / refactor : narrow down scope to getPostTypeById? -export const getPostType = async (search: number | string): Promise => { - const paramName = typeof search === "number" ? "id" : "slug" - return apiQuery(`${OWID_API_ENDPOINT}/type`, { - searchParams: [[paramName, search]], - }) -} - // The API query in getPostType is cleaner but slower, which becomes more of an // issue with prominent links requesting posts by slugs (getPostBySlug) to // render (of which getPostType is a callee). @@ -776,7 +751,6 @@ export const getTables = async (): Promise> => { } export const flushCache = (): void => { - cachedFeaturedImages = undefined getBlogIndex.cache.clear?.() cachedTables = undefined } From 05f8ba08c25125aa2bce59d4c6aff8ef0f86fc80 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 20:52:58 +0000 Subject: [PATCH 09/25] refactor(wp): move no-op tables code to deprecated file --- db/DEPRECATEDwpdb.ts | 42 +++++++++++++++++++++++++++++++++++++++++- db/wpdb.ts | 39 --------------------------------------- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index 2841fe36673..360348027c0 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -192,7 +192,9 @@ export const DEPRECATEDgetPostType = async ( } let cachedFeaturedImages: Map | undefined -export const getFeaturedImages = async (): Promise> => { +export const DEPRECATEDgetFeaturedImages = async (): Promise< + Map +> => { if (cachedFeaturedImages) return cachedFeaturedImages const rows = await singleton.query( @@ -207,3 +209,41 @@ export const getFeaturedImages = async (): Promise> => { cachedFeaturedImages = featuredImages return featuredImages } +interface TablepressTable { + tableId: string + data: string[][] +} +export let cachedTables: Map | undefined +export const DEPRECATEDgetTables = async (): Promise< + Map +> => { + if (cachedTables) return cachedTables + + const optRows = await singleton.query(` + SELECT option_value AS json FROM wp_options WHERE option_name='tablepress_tables' + `) + + const tableToPostIds = JSON.parse(optRows[0].json).table_post + + const rows = await singleton.query(` + SELECT ID, post_content FROM wp_posts WHERE post_type='tablepress_table' + `) + + const tableContents = new Map() + for (const row of rows) { + tableContents.set(row.ID, row.post_content) + } + + cachedTables = new Map() + for (const tableId in tableToPostIds) { + const data = JSON.parse( + tableContents.get(tableToPostIds[tableId]) || "[]" + ) + cachedTables.set(tableId, { + tableId: tableId, + data: data, + }) + } + + return cachedTables +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 82cbc431546..b8f1298dcb1 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -712,45 +712,6 @@ export const getTopics = async (cursor: string = ""): Promise => { } } -export interface TablepressTable { - tableId: string - data: string[][] -} - -let cachedTables: Map | undefined -export const getTables = async (): Promise> => { - if (cachedTables) return cachedTables - - const optRows = await singleton.query(` - SELECT option_value AS json FROM wp_options WHERE option_name='tablepress_tables' - `) - - const tableToPostIds = JSON.parse(optRows[0].json).table_post - - const rows = await singleton.query(` - SELECT ID, post_content FROM wp_posts WHERE post_type='tablepress_table' - `) - - const tableContents = new Map() - for (const row of rows) { - tableContents.set(row.ID, row.post_content) - } - - cachedTables = new Map() - for (const tableId in tableToPostIds) { - const data = JSON.parse( - tableContents.get(tableToPostIds[tableId]) || "[]" - ) - cachedTables.set(tableId, { - tableId: tableId, - data: data, - }) - } - - return cachedTables -} - export const flushCache = (): void => { getBlogIndex.cache.clear?.() - cachedTables = undefined } From 60d82982c750d75d35bcf64d3b0c8dc19fc45129 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Mon, 5 Feb 2024 17:08:31 +0000 Subject: [PATCH 10/25] temp revert table deprecation --- db/DEPRECATEDwpdb.ts | 38 -------------------------------------- db/wpdb.ts | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index 360348027c0..b3ab02a6533 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -209,41 +209,3 @@ export const DEPRECATEDgetFeaturedImages = async (): Promise< cachedFeaturedImages = featuredImages return featuredImages } -interface TablepressTable { - tableId: string - data: string[][] -} -export let cachedTables: Map | undefined -export const DEPRECATEDgetTables = async (): Promise< - Map -> => { - if (cachedTables) return cachedTables - - const optRows = await singleton.query(` - SELECT option_value AS json FROM wp_options WHERE option_name='tablepress_tables' - `) - - const tableToPostIds = JSON.parse(optRows[0].json).table_post - - const rows = await singleton.query(` - SELECT ID, post_content FROM wp_posts WHERE post_type='tablepress_table' - `) - - const tableContents = new Map() - for (const row of rows) { - tableContents.set(row.ID, row.post_content) - } - - cachedTables = new Map() - for (const tableId in tableToPostIds) { - const data = JSON.parse( - tableContents.get(tableToPostIds[tableId]) || "[]" - ) - cachedTables.set(tableId, { - tableId: tableId, - data: data, - }) - } - - return cachedTables -} diff --git a/db/wpdb.ts b/db/wpdb.ts index b8f1298dcb1..501b2ab2fff 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -712,6 +712,44 @@ export const getTopics = async (cursor: string = ""): Promise => { } } +export interface TablepressTable { + tableId: string + data: string[][] +} +export let cachedTables: Map | undefined +export const getTables = async (): Promise> => { + if (cachedTables) return cachedTables + + const optRows = await singleton.query(` + SELECT option_value AS json FROM wp_options WHERE option_name='tablepress_tables' + `) + + const tableToPostIds = JSON.parse(optRows[0].json).table_post + + const rows = await singleton.query(` + SELECT ID, post_content FROM wp_posts WHERE post_type='tablepress_table' + `) + + const tableContents = new Map() + for (const row of rows) { + tableContents.set(row.ID, row.post_content) + } + + cachedTables = new Map() + for (const tableId in tableToPostIds) { + const data = JSON.parse( + tableContents.get(tableToPostIds[tableId]) || "[]" + ) + cachedTables.set(tableId, { + tableId: tableId, + data: data, + }) + } + + return cachedTables +} + export const flushCache = (): void => { getBlogIndex.cache.clear?.() + cachedTables = undefined } From 4a8aafcc27987477e82e851a001601aaa8bb4bfa Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Mon, 5 Feb 2024 17:12:16 +0000 Subject: [PATCH 11/25] refactor: remove caching of tables --- db/wpdb.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/db/wpdb.ts b/db/wpdb.ts index 501b2ab2fff..17994fa0a2d 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -716,10 +716,7 @@ export interface TablepressTable { tableId: string data: string[][] } -export let cachedTables: Map | undefined export const getTables = async (): Promise> => { - if (cachedTables) return cachedTables - const optRows = await singleton.query(` SELECT option_value AS json FROM wp_options WHERE option_name='tablepress_tables' `) @@ -735,21 +732,20 @@ export const getTables = async (): Promise> => { tableContents.set(row.ID, row.post_content) } - cachedTables = new Map() + const tables = new Map() for (const tableId in tableToPostIds) { const data = JSON.parse( tableContents.get(tableToPostIds[tableId]) || "[]" ) - cachedTables.set(tableId, { + tables.set(tableId, { tableId: tableId, data: data, }) } - return cachedTables + return tables } export const flushCache = (): void => { getBlogIndex.cache.clear?.() - cachedTables = undefined } From dc2280f8c7cb46fbaf8788efd82f82c249104a1b Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 21:00:20 +0000 Subject: [PATCH 12/25] refactor(wp): move getPosts to Post model --- baker/SiteBaker.tsx | 6 ++-- baker/algolia/indexToAlgolia.tsx | 3 +- baker/sitemap.ts | 4 +-- db/model/Post.ts | 37 +++++++++++++++++++++++ db/wpdb.ts | 50 ++++---------------------------- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index caf2d333483..7a4a02b3acf 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -75,7 +75,7 @@ import { bakeAllPublishedExplorers, } from "./ExplorerBaker.js" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" -import { postsTable } from "../db/model/Post.js" +import { getPostsFromSnapshots, postsTable } from "../db/model/Post.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" import { Image } from "../db/model/Image.js" import { generateEmbedSnippet } from "../site/viteUtils.js" @@ -424,7 +424,7 @@ export class SiteBaker { private async removeDeletedPosts() { if (!this.bakeSteps.has("removeDeletedPosts")) return - const postsApi = await wpdb.getPosts() + const postsApi = await getPostsFromSnapshots() const postSlugs = [] for (const postApi of postsApi) { @@ -454,7 +454,7 @@ export class SiteBaker { const alreadyPublishedViaGdocsSlugsSet = await db.getSlugsWithPublishedGdocsSuccessors(db.knexInstance()) - const postsApi = await wpdb.getPosts( + const postsApi = await getPostsFromSnapshots( undefined, (postrow) => !alreadyPublishedViaGdocsSlugsSet.has(postrow.slug) ) diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index 402490c36e0..12cf3641260 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -26,6 +26,7 @@ import { Pageview } from "../../db/model/Pageview.js" import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" import { ArticleBlocks } from "../../site/gdocs/components/ArticleBlocks.js" import React from "react" +import { getPostsFromSnapshots } from "../../db/model/Post.js" interface TypeAndImportance { type: PageType @@ -193,7 +194,7 @@ const getPagesRecords = async () => { // TODO: the knex instance should be handed down as a parameter const slugsWithPublishedGdocsSuccessors = await db.getSlugsWithPublishedGdocsSuccessors(db.knexInstance()) - const postsApi = await wpdb.getPosts(undefined, (post) => { + const postsApi = await getPostsFromSnapshots(undefined, (post) => { // Two things can happen here: // 1. There's a published Gdoc with the same slug // 2. This post has a Gdoc successor (which might have a different slug) diff --git a/baker/sitemap.ts b/baker/sitemap.ts index 5a563c3d792..fcc00552431 100644 --- a/baker/sitemap.ts +++ b/baker/sitemap.ts @@ -6,13 +6,13 @@ import { } from "../settings/serverSettings.js" import { dayjs, countries, queryParamsToStr } from "@ourworldindata/utils" import * as db from "../db/db.js" -import * as wpdb from "../db/wpdb.js" import urljoin from "url-join" import { countryProfileSpecs } from "../site/countryProfileProjects.js" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" import { EXPLORERS_ROUTE_FOLDER } from "../explorer/ExplorerConstants.js" import { ExplorerProgram } from "../explorer/ExplorerProgram.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" +import { getPostsFromSnapshots } from "../db/model/Post.js" interface SitemapUrl { loc: string @@ -62,7 +62,7 @@ export const makeSitemap = async (explorerAdminServer: ExplorerAdminServer) => { const knex = db.knexInstance() const alreadyPublishedViaGdocsSlugsSet = await db.getSlugsWithPublishedGdocsSuccessors(knex) - const postsApi = await wpdb.getPosts( + const postsApi = await getPostsFromSnapshots( undefined, (postrow) => !alreadyPublishedViaGdocsSlugsSet.has(postrow.slug) ) diff --git a/db/model/Post.ts b/db/model/Post.ts index 010e6cd6ea7..861f788dbc8 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -4,12 +4,16 @@ import { CategoryWithEntries, DbEnrichedPost, DbRawPost, + FilterFnPostRestApi, FullPost, JsonError, + PostRestApi, + WP_PostType, parsePostRow, } from "@ourworldindata/utils" import { getFullPost } from "../wpdb.js" import { SiteNavigationStatic } from "../../site/SiteNavigation.js" +import { BLOG_SLUG } from "../../settings/serverSettings.js" export const postsTable = "posts" @@ -130,3 +134,36 @@ export const isPostSlugCitable = async (slug: string): Promise => { ) }) } + +export const getPostsFromSnapshots = async ( + postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], + filterFunc?: FilterFnPostRestApi +): Promise => { + const rawPosts: DbRawPost[] = ( + await db.knexInstance().raw( + ` + SELECT * FROM ${postsTable} + WHERE status = "publish" + AND type IN (?) + ORDER BY JSON_UNQUOTE(JSON_EXTRACT(wpApiSnapshot, '$.date')) DESC; + `, + [postTypes].join(",") + ) + )[0] + + const posts = rawPosts + .map(parsePostRow) + .map((p) => p.wpApiSnapshot) + .filter((p) => p !== null) as PostRestApi[] + + // Published pages excluded from public views + const excludedSlugs = [BLOG_SLUG] + + const filterConditions: Array = [ + (post): boolean => !excludedSlugs.includes(post.slug), + (post): boolean => !post.slug.endsWith("-country-profile"), + ] + if (filterFunc) filterConditions.push(filterFunc) + + return posts.filter((post) => filterConditions.every((c) => c(post))) +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 17994fa0a2d..746008f4608 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -10,7 +10,6 @@ import { WORDPRESS_API_USER, WORDPRESS_URL, BAKED_BASE_URL, - BLOG_SLUG, } from "../settings/serverSettings.js" import * as db from "./db.js" import { Knex, knex } from "knex" @@ -38,12 +37,7 @@ import { Tag, OwidGdocPostInterface, } from "@ourworldindata/utils" -import { - DbRawPost, - OwidGdocLinkType, - Topic, - parsePostRow, -} from "@ourworldindata/types" +import { OwidGdocLinkType, Topic } from "@ourworldindata/types" import { getContentGraph, WPPostTypeToGraphDocumentType, @@ -51,7 +45,7 @@ import { import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" -import { postsTable } from "./model/Post.js" +import { getPostsFromSnapshots } from "./model/Post.js" let _knexInstance: Knex @@ -273,39 +267,6 @@ export const getEndpointSlugFromType = (type: string): string => `${type}s` export const selectHomepagePosts: FilterFnPostRestApi = (post) => post.meta?.owid_publication_context_meta_field?.homepage === true -export const getPosts = async ( - postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], - filterFunc?: FilterFnPostRestApi -): Promise => { - const rawPosts: DbRawPost[] = ( - await db.knexInstance().raw( - ` - SELECT * FROM ${postsTable} - WHERE status = "publish" - AND type IN (?) - ORDER BY JSON_UNQUOTE(JSON_EXTRACT(wpApiSnapshot, '$.date')) DESC; - `, - [postTypes].join(",") - ) - )[0] - - const posts = rawPosts - .map(parsePostRow) - .map((p) => p.wpApiSnapshot) - .filter((p) => p !== null) as PostRestApi[] - - // Published pages excluded from public views - const excludedSlugs = [BLOG_SLUG] - - const filterConditions: Array = [ - (post): boolean => !excludedSlugs.includes(post.slug), - (post): boolean => !post.slug.endsWith("-country-profile"), - ] - if (filterFunc) filterConditions.push(filterFunc) - - return posts.filter((post) => filterConditions.every((c) => c(post))) -} - // The API query in getPostType is cleaner but slower, which becomes more of an // issue with prominent links requesting posts by slugs (getPostBySlug) to // render (of which getPostType is a callee). @@ -649,9 +610,10 @@ export const getBlogIndex = memoize(async (): Promise => { await db.getConnection() // side effect: ensure connection is established const gdocPosts = await GdocPost.getListedGdocs() const wpPosts = await Promise.all( - await getPosts([WP_PostType.Post], selectHomepagePosts).then((posts) => - posts.map((post) => getFullPost(post, true)) - ) + await getPostsFromSnapshots( + [WP_PostType.Post], + selectHomepagePosts + ).then((posts) => posts.map((post) => getFullPost(post, true))) ) const gdocSlugs = new Set(gdocPosts.map(({ slug }) => slug)) From 56d02053994ab6f461af4875883a86211808f197 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 21:21:29 +0000 Subject: [PATCH 13/25] refactor(wp): move more non wp db functions into Post --- baker/GrapherBaker.tsx | 8 ++- baker/SiteBaker.tsx | 13 ++-- baker/algolia/indexToAlgolia.tsx | 4 +- baker/siteRenderers.tsx | 3 +- db/DEPRECATEDwpdb.ts | 2 +- db/migrateWpPostsToArchieMl.ts | 5 +- db/model/Post.ts | 113 ++++++++++++++++++++++++++++--- db/wpdb.ts | 96 +------------------------- 8 files changed, 125 insertions(+), 119 deletions(-) diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index 5251c9a9b0e..d2592c903b8 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -16,7 +16,6 @@ import { } from "@ourworldindata/utils" import { getRelatedArticles, - getRelatedCharts, getRelatedChartsForVariable, getRelatedResearchAndWritingForVariable, isWordpressAPIEnabled, @@ -33,7 +32,10 @@ import { import * as db from "../db/db.js" import { glob } from "glob" import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js" -import { getPostEnrichedBySlug } from "../db/model/Post.js" +import { + getPostEnrichedBySlug, + getPostRelatedCharts, +} from "../db/model/Post.js" import { JsonError, GrapherInterface, @@ -319,7 +321,7 @@ const renderGrapherPage = async (grapher: GrapherInterface) => { const post = postSlug ? await getPostEnrichedBySlug(postSlug) : undefined const relatedCharts = post && isWordpressDBEnabled - ? await getRelatedCharts(post.id) + ? await getPostRelatedCharts(post.id) : undefined const relatedArticles = grapher.id && isWordpressAPIEnabled diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 7a4a02b3acf..360782a32e3 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -75,7 +75,12 @@ import { bakeAllPublishedExplorers, } from "./ExplorerBaker.js" import { ExplorerAdminServer } from "../explorerAdminServer/ExplorerAdminServer.js" -import { getPostsFromSnapshots, postsTable } from "../db/model/Post.js" +import { + getBlogIndex, + getFullPost, + getPostsFromSnapshots, + postsTable, +} from "../db/model/Post.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" import { Image } from "../db/model/Image.js" import { generateEmbedSnippet } from "../site/viteUtils.js" @@ -428,7 +433,7 @@ export class SiteBaker { const postSlugs = [] for (const postApi of postsApi) { - const post = await wpdb.getFullPost(postApi) + const post = await getFullPost(postApi) postSlugs.push(post.slug) } @@ -462,7 +467,7 @@ export class SiteBaker { await pMap( postsApi, async (postApi) => - wpdb.getFullPost(postApi).then((post) => this.bakePost(post)), + getFullPost(postApi).then((post) => this.bakePost(post)), { concurrency: 10 } ) @@ -783,7 +788,7 @@ export class SiteBaker { // Bake the blog index private async bakeBlogIndex() { if (!this.bakeSteps.has("blogIndex")) return - const allPosts = await wpdb.getBlogIndex() + const allPosts = await getBlogIndex() const numPages = Math.ceil(allPosts.length / BLOG_POSTS_PER_PAGE) for (let i = 1; i <= numPages; i++) { diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index 12cf3641260..c1f51ec998e 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -26,7 +26,7 @@ import { Pageview } from "../../db/model/Pageview.js" import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" import { ArticleBlocks } from "../../site/gdocs/components/ArticleBlocks.js" import React from "react" -import { getPostsFromSnapshots } from "../../db/model/Post.js" +import { getFullPost, getPostsFromSnapshots } from "../../db/model/Post.js" interface TypeAndImportance { type: PageType @@ -93,7 +93,7 @@ async function generateWordpressRecords( const records: PageRecord[] = [] for (const postApi of postsApi) { - const rawPost = await wpdb.getFullPost(postApi) + const rawPost = await getFullPost(postApi) if (isEmpty(rawPost.content)) { // we have some posts that are only placeholders (e.g. for a redirect); don't index these console.log( diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 3e706f781f1..7aa6a00530c 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -60,7 +60,7 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { getBlogIndex, getBlockContent } from "../db/wpdb.js" +import { getBlockContent } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" @@ -82,6 +82,7 @@ import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" import { ExplorerFullQueryParams } from "../explorer/ExplorerConstants.js" import { resolveInternalRedirect } from "./redirects.js" import { + getBlogIndex, getFullPostByIdFromSnapshot, getFullPostBySlugFromSnapshot, isPostSlugCitable, diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index b3ab02a6533..30e087818ba 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -18,13 +18,13 @@ import { BLOG_SLUG } from "../settings/serverSettings.js" import { WP_API_ENDPOINT, apiQuery, - getFullPost, getPostApiBySlugFromApi, isWordpressAPIEnabled, singleton, OWID_API_ENDPOINT, getEndpointSlugFromType, } from "./wpdb.js" +import { getFullPost } from "./model/Post.js" // Limit not supported with multiple post types: When passing multiple post // types, the limit is applied to the resulting array of sequentially sorted diff --git a/db/migrateWpPostsToArchieMl.ts b/db/migrateWpPostsToArchieMl.ts index 4163bfb2677..5820c72594b 100644 --- a/db/migrateWpPostsToArchieMl.ts +++ b/db/migrateWpPostsToArchieMl.ts @@ -19,8 +19,7 @@ import { adjustHeadingLevels, findMinimumHeadingLevel, } from "./model/Gdoc/htmlToEnriched.js" -import { getRelatedCharts } from "./wpdb.js" -import { isPostSlugCitable } from "./model/Post.js" +import { getPostRelatedCharts, isPostSlugCitable } from "./model/Post.js" import { enrichedBlocksToMarkdown } from "./model/Gdoc/enrichedToMarkdown.js" // slugs from all the linear entries we want to migrate from @edomt @@ -109,7 +108,7 @@ const migrate = async (): Promise => { const text = post.content let relatedCharts: RelatedChart[] = [] if (isEntry) { - relatedCharts = await getRelatedCharts(post.id) + relatedCharts = await getPostRelatedCharts(post.id) } const shouldIncludeMaxAsAuthor = await isPostSlugCitable(post.slug) diff --git a/db/model/Post.ts b/db/model/Post.ts index 861f788dbc8..2131ac3b62a 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -1,19 +1,27 @@ -import * as db from "../db.js" -import { Knex } from "knex" +import * as db from "../db" import { - CategoryWithEntries, - DbEnrichedPost, DbRawPost, - FilterFnPostRestApi, + DbEnrichedPost, + parsePostRow, FullPost, JsonError, - PostRestApi, + CategoryWithEntries, WP_PostType, - parsePostRow, -} from "@ourworldindata/utils" -import { getFullPost } from "../wpdb.js" -import { SiteNavigationStatic } from "../../site/SiteNavigation.js" + FilterFnPostRestApi, + PostRestApi, + RelatedChart, + IndexPost, + OwidGdocPostInterface, + IMAGES_DIRECTORY, +} from "@ourworldindata/types" +import { Knex } from "knex" +import { memoize, orderBy } from "lodash" +import { BAKED_BASE_URL } from "../../settings/clientSettings.js" import { BLOG_SLUG } from "../../settings/serverSettings.js" +import { selectHomepagePosts } from "../wpdb.js" +import { GdocPost } from "./Gdoc/GdocPost.js" +import { SiteNavigationStatic } from "../../site/SiteNavigation.js" +import { decodeHTML } from "entities" export const postsTable = "posts" @@ -167,3 +175,88 @@ export const getPostsFromSnapshots = async ( return posts.filter((post) => filterConditions.every((c) => c(post))) } + +export const getPostRelatedCharts = async ( + postId: number +): Promise => + db.queryMysql(` + SELECT DISTINCT + charts.config->>"$.slug" AS slug, + charts.config->>"$.title" AS title, + charts.config->>"$.variantName" AS variantName, + chart_tags.keyChartLevel + FROM charts + INNER JOIN chart_tags ON charts.id=chart_tags.chartId + INNER JOIN post_tags ON chart_tags.tagId=post_tags.tag_id + WHERE post_tags.post_id=${postId} + AND charts.config->>"$.isPublished" = "true" + ORDER BY title ASC + `) + +export const getFullPost = async ( + postApi: PostRestApi, + excludeContent?: boolean +): Promise => ({ + id: postApi.id, + type: postApi.type, + slug: postApi.slug, + path: postApi.slug, // kept for transitioning between legacy BPES (blog post as entry section) and future hierarchical paths + title: decodeHTML(postApi.title.rendered), + date: new Date(postApi.date_gmt), + modifiedDate: new Date(postApi.modified_gmt), + authors: postApi.authors_name || [], + content: excludeContent ? "" : postApi.content.rendered, + excerpt: decodeHTML(postApi.excerpt.rendered), + imageUrl: `${BAKED_BASE_URL}${ + postApi.featured_media_paths.medium_large ?? "/default-thumbnail.jpg" + }`, + thumbnailUrl: `${BAKED_BASE_URL}${ + postApi.featured_media_paths?.thumbnail ?? "/default-thumbnail.jpg" + }`, + imageId: postApi.featured_media, + relatedCharts: + postApi.type === "page" + ? await getPostRelatedCharts(postApi.id) + : undefined, +}) + +export const getBlogIndex = memoize(async (): Promise => { + await db.getConnection() // side effect: ensure connection is established + const gdocPosts = await GdocPost.getListedGdocs() + const wpPosts = await Promise.all( + await getPostsFromSnapshots( + [WP_PostType.Post], + selectHomepagePosts + ).then((posts) => posts.map((post) => getFullPost(post, true))) + ) + + const gdocSlugs = new Set(gdocPosts.map(({ slug }) => slug)) + const posts = [...mapGdocsToWordpressPosts(gdocPosts)] + + // Only adding each wpPost if there isn't already a gdoc with the same slug, + // to make sure we use the most up-to-date metadata + for (const wpPost of wpPosts) { + if (!gdocSlugs.has(wpPost.slug)) { + posts.push(wpPost) + } + } + + return orderBy(posts, (post) => post.date.getTime(), ["desc"]) +}) + +export const mapGdocsToWordpressPosts = ( + gdocs: OwidGdocPostInterface[] +): IndexPost[] => { + return gdocs.map((gdoc) => ({ + title: gdoc.content["atom-title"] || gdoc.content.title || "Untitled", + slug: gdoc.slug, + type: gdoc.content.type, + date: gdoc.publishedAt as Date, + modifiedDate: gdoc.updatedAt as Date, + authors: gdoc.content.authors, + excerpt: gdoc.content["atom-excerpt"] || gdoc.content.excerpt, + imageUrl: gdoc.content["featured-image"] + ? `${BAKED_BASE_URL}${IMAGES_DIRECTORY}${gdoc.content["featured-image"]}` + : `${BAKED_BASE_URL}/default-thumbnail.jpg`, + })) +} diff --git a/db/wpdb.ts b/db/wpdb.ts index 746008f4608..daab8f4d218 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -1,4 +1,3 @@ -import { decodeHTML } from "entities" import { DatabaseConnection } from "./DatabaseConnection.js" import { WORDPRESS_DB_NAME, @@ -9,7 +8,6 @@ import { WORDPRESS_API_PASS, WORDPRESS_API_USER, WORDPRESS_URL, - BAKED_BASE_URL, } from "../settings/serverSettings.js" import * as db from "./db.js" import { Knex, knex } from "knex" @@ -17,7 +15,6 @@ import { Base64 } from "js-base64" import { registerExitHandler } from "./cleanup.js" import { RelatedChart, - FullPost, WP_PostType, DocumentNode, PostReference, @@ -26,16 +23,11 @@ import { PostRestApi, TopicId, GraphType, - memoize, - IndexPost, - orderBy, - IMAGES_DIRECTORY, uniqBy, sortBy, DataPageRelatedResearch, OwidGdocType, Tag, - OwidGdocPostInterface, } from "@ourworldindata/utils" import { OwidGdocLinkType, Topic } from "@ourworldindata/types" import { @@ -43,9 +35,8 @@ import { WPPostTypeToGraphDocumentType, } from "./contentGraph.js" import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" -import { GdocPost } from "./model/Gdoc/GdocPost.js" import { Link } from "./model/Link.js" -import { getPostsFromSnapshots } from "./model/Post.js" +import { getBlogIndex } from "./model/Post.js" let _knexInstance: Knex @@ -312,23 +303,6 @@ export const getPostApiBySlugFromApi = async ( return apiQuery(`${WP_API_ENDPOINT}/${getEndpointSlugFromType(type)}/${id}`) } -export const getRelatedCharts = async ( - postId: number -): Promise => - db.queryMysql(` - SELECT DISTINCT - charts.config->>"$.slug" AS slug, - charts.config->>"$.title" AS title, - charts.config->>"$.variantName" AS variantName, - chart_tags.keyChartLevel - FROM charts - INNER JOIN chart_tags ON charts.id=chart_tags.chartId - INNER JOIN post_tags ON chart_tags.tagId=post_tags.tag_id - WHERE post_tags.post_id=${postId} - AND charts.config->>"$.isPublished" = "true" - ORDER BY title ASC - `) - export const getPostTags = async ( postId: number ): Promise[]> => { @@ -579,74 +553,6 @@ export const getBlockContent = async ( return post.data?.wpBlock?.content ?? undefined } -export const getFullPost = async ( - postApi: PostRestApi, - excludeContent?: boolean -): Promise => ({ - id: postApi.id, - type: postApi.type, - slug: postApi.slug, - path: postApi.slug, // kept for transitioning between legacy BPES (blog post as entry section) and future hierarchical paths - title: decodeHTML(postApi.title.rendered), - date: new Date(postApi.date_gmt), - modifiedDate: new Date(postApi.modified_gmt), - authors: postApi.authors_name || [], - content: excludeContent ? "" : postApi.content.rendered, - excerpt: decodeHTML(postApi.excerpt.rendered), - imageUrl: `${BAKED_BASE_URL}${ - postApi.featured_media_paths.medium_large ?? "/default-thumbnail.jpg" - }`, - thumbnailUrl: `${BAKED_BASE_URL}${ - postApi.featured_media_paths?.thumbnail ?? "/default-thumbnail.jpg" - }`, - imageId: postApi.featured_media, - relatedCharts: - postApi.type === "page" - ? await getRelatedCharts(postApi.id) - : undefined, -}) - -export const getBlogIndex = memoize(async (): Promise => { - await db.getConnection() // side effect: ensure connection is established - const gdocPosts = await GdocPost.getListedGdocs() - const wpPosts = await Promise.all( - await getPostsFromSnapshots( - [WP_PostType.Post], - selectHomepagePosts - ).then((posts) => posts.map((post) => getFullPost(post, true))) - ) - - const gdocSlugs = new Set(gdocPosts.map(({ slug }) => slug)) - const posts = [...mapGdocsToWordpressPosts(gdocPosts)] - - // Only adding each wpPost if there isn't already a gdoc with the same slug, - // to make sure we use the most up-to-date metadata - for (const wpPost of wpPosts) { - if (!gdocSlugs.has(wpPost.slug)) { - posts.push(wpPost) - } - } - - return orderBy(posts, (post) => post.date.getTime(), ["desc"]) -}) - -export const mapGdocsToWordpressPosts = ( - gdocs: OwidGdocPostInterface[] -): IndexPost[] => { - return gdocs.map((gdoc) => ({ - title: gdoc.content["atom-title"] || gdoc.content.title || "Untitled", - slug: gdoc.slug, - type: gdoc.content.type, - date: gdoc.publishedAt as Date, - modifiedDate: gdoc.updatedAt as Date, - authors: gdoc.content.authors, - excerpt: gdoc.content["atom-excerpt"] || gdoc.content.excerpt, - imageUrl: gdoc.content["featured-image"] - ? `${BAKED_BASE_URL}${IMAGES_DIRECTORY}${gdoc.content["featured-image"]}` - : `${BAKED_BASE_URL}/default-thumbnail.jpg`, - })) -} - export const getTopics = async (cursor: string = ""): Promise => { if (!isWordpressAPIEnabled) return [] const query = `query { From 510f7a2271df3af2b538dc1d30bcda3228349d55 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 21:34:39 +0000 Subject: [PATCH 14/25] refactor(wp): move flush cache to Post --- baker/SiteBaker.tsx | 3 ++- db/model/Post.ts | 4 ++++ db/wpdb.ts | 5 ----- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 360782a32e3..267084c2440 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -79,6 +79,7 @@ import { getBlogIndex, getFullPost, getPostsFromSnapshots, + postsFlushCache, postsTable, } from "../db/model/Post.js" import { GdocPost } from "../db/model/Gdoc/GdocPost.js" @@ -967,7 +968,7 @@ export class SiteBaker { private flushCache() { // Clear caches to allow garbage collection while waiting for next run - wpdb.flushCache() + postsFlushCache() siteBakingFlushCache() redirectsFlushCache() this.progressBar.tick({ name: "✅ cache flushed" }) diff --git a/db/model/Post.ts b/db/model/Post.ts index 2131ac3b62a..6d44f7cb810 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -260,3 +260,7 @@ export const mapGdocsToWordpressPosts = ( : `${BAKED_BASE_URL}/default-thumbnail.jpg`, })) } + +export const postsFlushCache = (): void => { + getBlogIndex.cache.clear?.() +} diff --git a/db/wpdb.ts b/db/wpdb.ts index daab8f4d218..b8329d5d9ef 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -36,7 +36,6 @@ import { } from "./contentGraph.js" import { TOPICS_CONTENT_GRAPH } from "../settings/clientSettings.js" import { Link } from "./model/Link.js" -import { getBlogIndex } from "./model/Post.js" let _knexInstance: Knex @@ -613,7 +612,3 @@ export const getTables = async (): Promise> => { return tables } - -export const flushCache = (): void => { - getBlogIndex.cache.clear?.() -} From 8fb4e18b928e94b1c723a218f4298ffeab023c1c Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Thu, 1 Feb 2024 21:38:24 +0000 Subject: [PATCH 15/25] refactor(wp): move listed filter fn to Post --- db/model/Post.ts | 4 +++- db/wpdb.ts | 4 ---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/db/model/Post.ts b/db/model/Post.ts index 6d44f7cb810..c5e7694d155 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -18,7 +18,6 @@ import { Knex } from "knex" import { memoize, orderBy } from "lodash" import { BAKED_BASE_URL } from "../../settings/clientSettings.js" import { BLOG_SLUG } from "../../settings/serverSettings.js" -import { selectHomepagePosts } from "../wpdb.js" import { GdocPost } from "./Gdoc/GdocPost.js" import { SiteNavigationStatic } from "../../site/SiteNavigation.js" import { decodeHTML } from "entities" @@ -220,6 +219,9 @@ export const getFullPost = async ( : undefined, }) +const selectHomepagePosts: FilterFnPostRestApi = (post) => + post.meta?.owid_publication_context_meta_field?.homepage === true + export const getBlogIndex = memoize(async (): Promise => { await db.getConnection() // side effect: ensure connection is established const gdocPosts = await GdocPost.getListedGdocs() diff --git a/db/wpdb.ts b/db/wpdb.ts index b8329d5d9ef..b9662f31771 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -19,7 +19,6 @@ import { DocumentNode, PostReference, JsonError, - FilterFnPostRestApi, PostRestApi, TopicId, GraphType, @@ -254,9 +253,6 @@ export const getPermalinks = async (): Promise<{ // page => pages, post => posts export const getEndpointSlugFromType = (type: string): string => `${type}s` -export const selectHomepagePosts: FilterFnPostRestApi = (post) => - post.meta?.owid_publication_context_meta_field?.homepage === true - // The API query in getPostType is cleaner but slower, which becomes more of an // issue with prominent links requesting posts by slugs (getPostBySlug) to // render (of which getPostType is a callee). From 4263084d5fe4be539dcb0908185541e16d7ef828 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Fri, 2 Feb 2024 08:01:49 +0000 Subject: [PATCH 16/25] refactor(wp): rename getBlockContent --- baker/siteRenderers.tsx | 4 ++-- db/syncPostsToGrapher.ts | 2 +- db/wpdb.ts | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 7aa6a00530c..f6c234dfa87 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -60,7 +60,7 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { getBlockContent } from "../db/wpdb.js" +import { getBlockContentFromApi } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" @@ -714,7 +714,7 @@ export const renderExplorerPage = async ( const wpContent = program.wpBlockId ? await renderReusableBlock( - await getBlockContent(program.wpBlockId), + await getBlockContentFromApi(program.wpBlockId), program.wpBlockId ) : undefined diff --git a/db/syncPostsToGrapher.ts b/db/syncPostsToGrapher.ts index dd1130c2dff..b4579ea167d 100644 --- a/db/syncPostsToGrapher.ts +++ b/db/syncPostsToGrapher.ts @@ -350,7 +350,7 @@ const syncPostsToGrapher = async (): Promise => { ), wpApiSnapshot: post.post_type === "wp_block" - ? await wpdb.getBlockApi(post.ID) + ? await wpdb.getBlockApiFromApi(post.ID) : await wpdb.getPostApiBySlugFromApi(post.post_name), featured_image: post.featured_image || "", published_at: diff --git a/db/wpdb.ts b/db/wpdb.ts index b9662f31771..bfa33a48b34 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -525,7 +525,7 @@ export const getRelatedArticles = async ( ) } -export const getBlockApi = async (id: number): Promise => { +export const getBlockApiFromApi = async (id: number): Promise => { if (!isWordpressAPIEnabled) return undefined const query = ` @@ -538,12 +538,12 @@ export const getBlockApi = async (id: number): Promise => { return graphqlQuery(query, { id }) } -export const getBlockContent = async ( +export const getBlockContentFromApi = async ( id: number ): Promise => { if (!isWordpressAPIEnabled) return undefined - const post = await getBlockApi(id) + const post = await getBlockApiFromApi(id) return post.data?.wpBlock?.content ?? undefined } From 734355150253ac513df10bcf16e724c9e348a937 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Fri, 2 Feb 2024 09:17:02 +0000 Subject: [PATCH 17/25] feat(wp): type check snapshot --- db/model/Post.ts | 5 +++-- packages/@ourworldindata/types/src/dbTypes/Posts.ts | 13 ++++++++++++- packages/@ourworldindata/types/src/index.ts | 1 + .../types/src/wordpressTypes/WordpressTypes.ts | 8 ++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/db/model/Post.ts b/db/model/Post.ts index c5e7694d155..e214b32421c 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -13,6 +13,7 @@ import { IndexPost, OwidGdocPostInterface, IMAGES_DIRECTORY, + snapshotIsPostRestApi, } from "@ourworldindata/types" import { Knex } from "knex" import { memoize, orderBy } from "lodash" @@ -110,7 +111,7 @@ export const getFullPostBySlugFromSnapshot = async ( slug: string ): Promise => { const postEnriched = await getPostEnrichedBySlug(slug) - if (!postEnriched?.wpApiSnapshot) + if (!snapshotIsPostRestApi(postEnriched?.wpApiSnapshot)) throw new JsonError(`No page snapshot found by slug ${slug}`, 404) return getFullPost(postEnriched.wpApiSnapshot) @@ -120,7 +121,7 @@ export const getFullPostByIdFromSnapshot = async ( id: number ): Promise => { const postEnriched = await getPostEnrichedById(id) - if (!postEnriched?.wpApiSnapshot) + if (!snapshotIsPostRestApi(postEnriched?.wpApiSnapshot)) throw new JsonError(`No page snapshot found by id ${id}`, 404) return getFullPost(postEnriched.wpApiSnapshot) diff --git a/packages/@ourworldindata/types/src/dbTypes/Posts.ts b/packages/@ourworldindata/types/src/dbTypes/Posts.ts index 51f3e436204..ef6e347fe1a 100644 --- a/packages/@ourworldindata/types/src/dbTypes/Posts.ts +++ b/packages/@ourworldindata/types/src/dbTypes/Posts.ts @@ -2,6 +2,7 @@ import { WP_PostType, FormattingOptions, PostRestApi, + BlockGraphQlApi, } from "../wordpressTypes/WordpressTypes.js" import { OwidArticleBackportingStatistics, @@ -43,7 +44,7 @@ export type DbEnrichedPost = Omit< formattingOptions: FormattingOptions | null archieml: OwidGdocPostInterface | null archieml_update_statistics: OwidArticleBackportingStatistics | null - wpApiSnapshot: PostRestApi | null + wpApiSnapshot: PostRestApi | BlockGraphQlApi | null } export interface DbRawPostWithGdocPublishStatus extends DbRawPost { isGdocPublished: boolean @@ -94,3 +95,13 @@ export function serializePostRow(postRow: DbEnrichedPost): DbRawPost { wpApiSnapshot: JSON.stringify(postRow.wpApiSnapshot), } } + +export const snapshotIsPostRestApi = ( + snapshot: PostRestApi | BlockGraphQlApi | undefined | null +): snapshot is PostRestApi => { + if (!snapshot) return false + + return [WP_PostType.Page, WP_PostType.Post].includes( + (snapshot as PostRestApi).type + ) +} diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 9dd060b9978..1a44b6adb75 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -501,6 +501,7 @@ export { parsePostRow, serializePostRow, parsePostArchieml, + snapshotIsPostRestApi, } from "./dbTypes/Posts.js" export { type DbInsertPostGdoc, diff --git a/packages/@ourworldindata/types/src/wordpressTypes/WordpressTypes.ts b/packages/@ourworldindata/types/src/wordpressTypes/WordpressTypes.ts index c605d099ae3..0d87c93630d 100644 --- a/packages/@ourworldindata/types/src/wordpressTypes/WordpressTypes.ts +++ b/packages/@ourworldindata/types/src/wordpressTypes/WordpressTypes.ts @@ -51,6 +51,14 @@ export interface PostRestApi { } } +export interface BlockGraphQlApi { + data?: { + wpBlock?: { + content: string + } + } +} + export type FilterFnPostRestApi = (post: PostRestApi) => boolean export enum WP_ColumnStyle { From 798ba2cf1f15c37821b19c3404125259ca7bd261 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Fri, 2 Feb 2024 09:54:16 +0000 Subject: [PATCH 18/25] feat(wp): get block content from snapshot --- baker/siteRenderers.tsx | 4 ++-- db/model/Post.ts | 24 +++++++++++++++++-- .../types/src/dbTypes/Posts.ts | 10 +++++--- packages/@ourworldindata/types/src/index.ts | 1 + 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index f6c234dfa87..f83152de2d6 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -60,7 +60,6 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { getBlockContentFromApi } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" @@ -82,6 +81,7 @@ import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" import { ExplorerFullQueryParams } from "../explorer/ExplorerConstants.js" import { resolveInternalRedirect } from "./redirects.js" import { + getBlockContentFromSnapshot, getBlogIndex, getFullPostByIdFromSnapshot, getFullPostBySlugFromSnapshot, @@ -714,7 +714,7 @@ export const renderExplorerPage = async ( const wpContent = program.wpBlockId ? await renderReusableBlock( - await getBlockContentFromApi(program.wpBlockId), + await getBlockContentFromSnapshot(program.wpBlockId), program.wpBlockId ) : undefined diff --git a/db/model/Post.ts b/db/model/Post.ts index e214b32421c..d48f0ac72b5 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -14,6 +14,7 @@ import { OwidGdocPostInterface, IMAGES_DIRECTORY, snapshotIsPostRestApi, + snapshotIsBlockGraphQlApi, } from "@ourworldindata/types" import { Knex } from "knex" import { memoize, orderBy } from "lodash" @@ -111,7 +112,10 @@ export const getFullPostBySlugFromSnapshot = async ( slug: string ): Promise => { const postEnriched = await getPostEnrichedBySlug(slug) - if (!snapshotIsPostRestApi(postEnriched?.wpApiSnapshot)) + if ( + !postEnriched?.wpApiSnapshot || + !snapshotIsPostRestApi(postEnriched.wpApiSnapshot) + ) throw new JsonError(`No page snapshot found by slug ${slug}`, 404) return getFullPost(postEnriched.wpApiSnapshot) @@ -121,7 +125,10 @@ export const getFullPostByIdFromSnapshot = async ( id: number ): Promise => { const postEnriched = await getPostEnrichedById(id) - if (!snapshotIsPostRestApi(postEnriched?.wpApiSnapshot)) + if ( + !postEnriched?.wpApiSnapshot || + !snapshotIsPostRestApi(postEnriched.wpApiSnapshot) + ) throw new JsonError(`No page snapshot found by id ${id}`, 404) return getFullPost(postEnriched.wpApiSnapshot) @@ -267,3 +274,16 @@ export const mapGdocsToWordpressPosts = ( export const postsFlushCache = (): void => { getBlogIndex.cache.clear?.() } + +export const getBlockContentFromSnapshot = async ( + id: number +): Promise => { + const enrichedBlock = await getPostEnrichedById(id) + if ( + !enrichedBlock?.wpApiSnapshot || + !snapshotIsBlockGraphQlApi(enrichedBlock.wpApiSnapshot) + ) + return + + return enrichedBlock?.wpApiSnapshot.data?.wpBlock?.content +} diff --git a/packages/@ourworldindata/types/src/dbTypes/Posts.ts b/packages/@ourworldindata/types/src/dbTypes/Posts.ts index ef6e347fe1a..e4f39b2c73b 100644 --- a/packages/@ourworldindata/types/src/dbTypes/Posts.ts +++ b/packages/@ourworldindata/types/src/dbTypes/Posts.ts @@ -97,11 +97,15 @@ export function serializePostRow(postRow: DbEnrichedPost): DbRawPost { } export const snapshotIsPostRestApi = ( - snapshot: PostRestApi | BlockGraphQlApi | undefined | null + snapshot: PostRestApi | BlockGraphQlApi ): snapshot is PostRestApi => { - if (!snapshot) return false - return [WP_PostType.Page, WP_PostType.Post].includes( (snapshot as PostRestApi).type ) } + +export const snapshotIsBlockGraphQlApi = ( + snapshot: PostRestApi | BlockGraphQlApi +): snapshot is BlockGraphQlApi => { + return (snapshot as BlockGraphQlApi).data?.wpBlock?.content !== undefined +} diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 1a44b6adb75..1a46fc37ac7 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -502,6 +502,7 @@ export { serializePostRow, parsePostArchieml, snapshotIsPostRestApi, + snapshotIsBlockGraphQlApi, } from "./dbTypes/Posts.js" export { type DbInsertPostGdoc, From 344e42ffa7497c32f3219475d9dd80dbc2534734 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Fri, 2 Feb 2024 09:55:45 +0000 Subject: [PATCH 19/25] refactor(wp): deprecate getBlockContentFromApi --- db/DEPRECATEDwpdb.ts | 11 +++++++++++ db/wpdb.ts | 10 ---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index 30e087818ba..c577667a839 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -23,6 +23,7 @@ import { singleton, OWID_API_ENDPOINT, getEndpointSlugFromType, + getBlockApiFromApi, } from "./wpdb.js" import { getFullPost } from "./model/Post.js" @@ -209,3 +210,13 @@ export const DEPRECATEDgetFeaturedImages = async (): Promise< cachedFeaturedImages = featuredImages return featuredImages } + +export const DEPRECATEDgetBlockContentFromApi = async ( + id: number +): Promise => { + if (!isWordpressAPIEnabled) return undefined + + const post = await getBlockApiFromApi(id) + + return post.data?.wpBlock?.content ?? undefined +} diff --git a/db/wpdb.ts b/db/wpdb.ts index bfa33a48b34..cb192299c0e 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -538,16 +538,6 @@ export const getBlockApiFromApi = async (id: number): Promise => { return graphqlQuery(query, { id }) } -export const getBlockContentFromApi = async ( - id: number -): Promise => { - if (!isWordpressAPIEnabled) return undefined - - const post = await getBlockApiFromApi(id) - - return post.data?.wpBlock?.content ?? undefined -} - export const getTopics = async (cursor: string = ""): Promise => { if (!isWordpressAPIEnabled) return [] const query = `query { From 6ccc9936cdbf754512a5b9a919e764bcfc9d65e3 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Fri, 26 Jan 2024 20:51:45 +0000 Subject: [PATCH 20/25] fix(baker): init connection --- baker/SiteBaker.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 267084c2440..ffe8a633074 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -430,6 +430,9 @@ export class SiteBaker { private async removeDeletedPosts() { if (!this.bakeSteps.has("removeDeletedPosts")) return + + await db.getConnection() + const postsApi = await getPostsFromSnapshots() const postSlugs = [] From 461907c81f7c54737cc22d3078207dd744d186c1 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Fri, 2 Feb 2024 17:29:02 +0000 Subject: [PATCH 21/25] refactor(wp): deprecate getTopics --- adminSiteServer/apiRouter.ts | 3 ++- db/DEPRECATEDwpdb.ts | 33 +++++++++++++++++++++++++++++++++ db/wpdb.ts | 31 ++----------------------------- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 92d637ff4a4..6f6f08de40b 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -6,6 +6,7 @@ import * as db from "../db/db.js" import { imageStore } from "../db/model/Image.js" import { GdocXImage } from "../db/model/GdocXImage.js" import * as wpdb from "../db/wpdb.js" +import { DEPRECATEDgetTopics } from "../db/DEPRECATEDwpdb.js" import { UNCATEGORIZED_TAG_ID, BAKE_ON_CHANGE, @@ -567,7 +568,7 @@ apiRouter.get( ) apiRouter.get("/topics.json", async (req: Request, res: Response) => ({ - topics: await wpdb.getTopics(), + topics: await DEPRECATEDgetTopics(), })) apiRouter.get( "/editorData/variables.json", diff --git a/db/DEPRECATEDwpdb.ts b/db/DEPRECATEDwpdb.ts index c577667a839..8de9003ddf5 100644 --- a/db/DEPRECATEDwpdb.ts +++ b/db/DEPRECATEDwpdb.ts @@ -13,6 +13,7 @@ import { PostRestApi, FullPost, JsonError, + Topic, } from "@ourworldindata/types" import { BLOG_SLUG } from "../settings/serverSettings.js" import { @@ -24,6 +25,8 @@ import { OWID_API_ENDPOINT, getEndpointSlugFromType, getBlockApiFromApi, + graphqlQuery, + ENTRIES_CATEGORY_ID, } from "./wpdb.js" import { getFullPost } from "./model/Post.js" @@ -220,3 +223,33 @@ export const DEPRECATEDgetBlockContentFromApi = async ( return post.data?.wpBlock?.content ?? undefined } + +export const DEPRECATEDgetTopics = async ( + cursor: string = "" +): Promise => { + if (!isWordpressAPIEnabled) return [] + + const query = `query { + pages (first: 100, after:"${cursor}", where: {categoryId:${ENTRIES_CATEGORY_ID}} ) { + pageInfo { + hasNextPage + endCursor + } + nodes { + id: databaseId + name: title + } + } + }` + + const documents = await graphqlQuery(query, { cursor }) + const pageInfo = documents.data.pages.pageInfo + const topics: Topic[] = documents.data.pages.nodes + if (topics.length === 0) return [] + + if (pageInfo.hasNextPage) { + return topics.concat(await DEPRECATEDgetTopics(pageInfo.endCursor)) + } else { + return topics + } +} diff --git a/db/wpdb.ts b/db/wpdb.ts index cb192299c0e..efc8c89acd1 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -28,7 +28,7 @@ import { OwidGdocType, Tag, } from "@ourworldindata/utils" -import { OwidGdocLinkType, Topic } from "@ourworldindata/types" +import { OwidGdocLinkType } from "@ourworldindata/types" import { getContentGraph, WPPostTypeToGraphDocumentType, @@ -117,7 +117,7 @@ export const ENTRIES_CATEGORY_ID = 44 * every query. So it is the caller's responsibility to throw (if necessary) on * "faux 404". */ -const graphqlQuery = async ( +export const graphqlQuery = async ( query: string, variables: any = {} ): Promise => { @@ -538,33 +538,6 @@ export const getBlockApiFromApi = async (id: number): Promise => { return graphqlQuery(query, { id }) } -export const getTopics = async (cursor: string = ""): Promise => { - if (!isWordpressAPIEnabled) return [] - const query = `query { - pages (first: 100, after:"${cursor}", where: {categoryId:${ENTRIES_CATEGORY_ID}} ) { - pageInfo { - hasNextPage - endCursor - } - nodes { - id: databaseId - name: title - } - } - }` - - const documents = await graphqlQuery(query, { cursor }) - const pageInfo = documents.data.pages.pageInfo - const topics: Topic[] = documents.data.pages.nodes - if (topics.length === 0) return [] - - if (pageInfo.hasNextPage) { - return topics.concat(await getTopics(pageInfo.endCursor)) - } else { - return topics - } -} - export interface TablepressTable { tableId: string data: string[][] From 3c12fa7645d798bc15f68834b4a93b9ce623d1db Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Mon, 5 Feb 2024 20:11:42 +0000 Subject: [PATCH 22/25] fix: getPostsFromSnapshots query parameter --- db/model/Post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/model/Post.ts b/db/model/Post.ts index d48f0ac72b5..5ef3de4b661 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -162,7 +162,7 @@ export const getPostsFromSnapshots = async ( AND type IN (?) ORDER BY JSON_UNQUOTE(JSON_EXTRACT(wpApiSnapshot, '$.date')) DESC; `, - [postTypes].join(",") + [postTypes] ) )[0] From ae56d1401208e4e3feead9565bbc350ccb96a552 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Tue, 13 Feb 2024 20:02:41 +0000 Subject: [PATCH 23/25] refactor: remove promise on isPostSlugCitable --- baker/pageOverrides.tsx | 2 +- baker/siteRenderers.tsx | 3 +-- db/migrateWpPostsToArchieMl.ts | 2 +- db/model/Post.ts | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/baker/pageOverrides.tsx b/baker/pageOverrides.tsx index a21cc1e3b8b..c46b69e40b3 100644 --- a/baker/pageOverrides.tsx +++ b/baker/pageOverrides.tsx @@ -64,7 +64,7 @@ export const getPageOverrides = async ( const landing = await getLandingOnlyIfParent(post, formattingOptions) if (!landing) return - const isParentLandingCitable = await isPostSlugCitable(landing.slug) + const isParentLandingCitable = isPostSlugCitable(landing.slug) if (!isParentLandingCitable) return return { diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index f83152de2d6..07b133e01b9 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -229,8 +229,7 @@ export const renderPost = async ( const pageOverrides = await getPageOverrides(post, formattingOptions) const citationStatus = - (await isPostSlugCitable(post.slug)) || - isPageOverridesCitable(pageOverrides) + isPostSlugCitable(post.slug) || isPageOverridesCitable(pageOverrides) return renderToHtmlPage( => { relatedCharts = await getPostRelatedCharts(post.id) } - const shouldIncludeMaxAsAuthor = await isPostSlugCitable(post.slug) + const shouldIncludeMaxAsAuthor = isPostSlugCitable(post.slug) if ( shouldIncludeMaxAsAuthor && post.authors && diff --git a/db/model/Post.ts b/db/model/Post.ts index 5ef3de4b661..737f4ba8ca2 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -134,7 +134,7 @@ export const getFullPostByIdFromSnapshot = async ( return getFullPost(postEnriched.wpApiSnapshot) } -export const isPostSlugCitable = async (slug: string): Promise => { +export const isPostSlugCitable = (slug: string): boolean => { const entries = SiteNavigationStatic.categories return entries.some((category) => { return ( From 35e868765aefca3976286101ccd9d295b3fff9d1 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Tue, 13 Feb 2024 20:08:14 +0000 Subject: [PATCH 24/25] refactor: use mysql json shorthand accessor --- db/model/Post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/model/Post.ts b/db/model/Post.ts index 737f4ba8ca2..6662b337e9d 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -160,7 +160,7 @@ export const getPostsFromSnapshots = async ( SELECT * FROM ${postsTable} WHERE status = "publish" AND type IN (?) - ORDER BY JSON_UNQUOTE(JSON_EXTRACT(wpApiSnapshot, '$.date')) DESC; + ORDER BY wpApiSnapshot->>'$.date' DESC; `, [postTypes] ) From e93d1bae64a486ba63fc27d43c781d44f8359a68 Mon Sep 17 00:00:00 2001 From: Matthieu Bergel Date: Mon, 19 Feb 2024 13:48:21 +0000 Subject: [PATCH 25/25] refactor(wp): add parsePostWpApiSnapshot function --- db/model/Post.ts | 12 +++++++----- packages/@ourworldindata/types/src/dbTypes/Posts.ts | 6 +++++- packages/@ourworldindata/types/src/index.ts | 2 ++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/db/model/Post.ts b/db/model/Post.ts index 6662b337e9d..1e968c9406a 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -3,6 +3,7 @@ import { DbRawPost, DbEnrichedPost, parsePostRow, + parsePostWpApiSnapshot, FullPost, JsonError, CategoryWithEntries, @@ -154,11 +155,12 @@ export const getPostsFromSnapshots = async ( postTypes: string[] = [WP_PostType.Post, WP_PostType.Page], filterFunc?: FilterFnPostRestApi ): Promise => { - const rawPosts: DbRawPost[] = ( + const rawPosts: Pick[] = ( await db.knexInstance().raw( ` - SELECT * FROM ${postsTable} - WHERE status = "publish" + SELECT wpApiSnapshot FROM ${postsTable} + WHERE wpApiSnapshot IS NOT NULL + AND status = "publish" AND type IN (?) ORDER BY wpApiSnapshot->>'$.date' DESC; `, @@ -167,9 +169,9 @@ export const getPostsFromSnapshots = async ( )[0] const posts = rawPosts - .map(parsePostRow) .map((p) => p.wpApiSnapshot) - .filter((p) => p !== null) as PostRestApi[] + .filter((snapshot) => snapshot !== null) + .map((snapshot) => parsePostWpApiSnapshot(snapshot!)) // Published pages excluded from public views const excludedSlugs = [BLOG_SLUG] diff --git a/packages/@ourworldindata/types/src/dbTypes/Posts.ts b/packages/@ourworldindata/types/src/dbTypes/Posts.ts index e4f39b2c73b..b58b67b8144 100644 --- a/packages/@ourworldindata/types/src/dbTypes/Posts.ts +++ b/packages/@ourworldindata/types/src/dbTypes/Posts.ts @@ -66,6 +66,10 @@ export function parsePostArchieml(archieml: string): any { return JSON.parse(archieml) } +export function parsePostWpApiSnapshot(wpApiSnapshot: string): PostRestApi { + return JSON.parse(wpApiSnapshot) +} + export function parsePostRow(postRow: DbRawPost): DbEnrichedPost { return { ...postRow, @@ -78,7 +82,7 @@ export function parsePostRow(postRow: DbRawPost): DbEnrichedPost { ? JSON.parse(postRow.archieml_update_statistics) : null, wpApiSnapshot: postRow.wpApiSnapshot - ? JSON.parse(postRow.wpApiSnapshot) + ? parsePostWpApiSnapshot(postRow.wpApiSnapshot) : null, } } diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 1a46fc37ac7..c9de653fa08 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -138,6 +138,7 @@ export { WP_ColumnStyle, WP_PostType, type PostRestApi, + type BlockGraphQlApi, type FilterFnPostRestApi, type FormattingOptions, SubNavId, @@ -499,6 +500,7 @@ export { parsePostFormattingOptions, parsePostAuthors, parsePostRow, + parsePostWpApiSnapshot, serializePostRow, parsePostArchieml, snapshotIsPostRestApi,