Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge content graph and chart references #3179

Merged
merged 6 commits into from
Feb 19, 2024
Merged

Merge content graph and chart references #3179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
eccbd6d
refactor(graph): get wordpress post references for charts from db
mlbrgl Feb 7, 2024
a822b35
refactor(graph): get gdocs post references for charts from raw query
mlbrgl Feb 7, 2024
083296a
refactor(graph): merge references fn into Post
mlbrgl Feb 7, 2024
0ff456d
refactor(graph): use references in getRelatedArticles
mlbrgl Feb 7, 2024
cb88ad2
refactor(graph): simplify getReferencesByChartId
mlbrgl Feb 7, 2024
8943401
refactor(related articles): use url instead of slug
mlbrgl Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions adminSiteClient/ChartEditor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
type DetailDictionary,
type RawPageview,
Topic,
PostReference,
} from "@ourworldindata/utils"
import { computed, observable, runInAction, when } from "mobx"
import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js"
Expand Down Expand Up @@ -54,13 +55,6 @@ export const getFullReferencesCount = (references: References): number => {
)
}

export interface PostReference {
id: string
title: string
slug: string
url: string
}

export interface ChartRedirect {
id: number
slug: string
Expand Down
116 changes: 11 additions & 105 deletions adminSiteServer/apiRouter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import { transaction } from "../db/db.js"
import * as db from "../db/db.js"
import { imageStore } from "../db/model/Image.js"
import { GdocXImage } from "../db/model/GdocXImage.js"
import * as wpdb from "../db/wpdb.js"
import { DEPRECATEDgetTopics } from "../db/DEPRECATEDwpdb.js"
import {
UNCATEGORIZED_TAG_ID,
Expand Down Expand Up @@ -43,7 +42,6 @@ import {
SuggestedChartRevisionStatus,
variableAnnotationAllowedColumnNamesAndTypes,
VariableAnnotationsResponseRow,
isUndefined,
OwidVariableWithSource,
OwidChartDimensionInterface,
DimensionProperty,
Expand All @@ -53,7 +51,6 @@ import {
} from "@ourworldindata/utils"
import {
GrapherInterface,
OwidGdocLinkType,
OwidGdocType,
grapherKeysToSerialize,
} from "@ourworldindata/types"
Expand Down Expand Up @@ -82,7 +79,8 @@ import {
postsTable,
setTagsForPost,
getTagsByPostId,
getPermalinks,
getWordpressPostReferencesByChartId,
getGdocsPostReferencesByChartId,
} from "../db/model/Post.js"
import {
checkFullDeployFallback,
Expand Down Expand Up @@ -149,114 +147,22 @@ async function getLogsByChartId(chartId: number): Promise<ChartRevision[]> {
return logs
}

const getPostsForSlugs = async (
slugs: string[]
): Promise<{ ID: number; post_title: string; post_name: string }[]> => {
if (!wpdb.singleton) return []
// Hacky approach to find all the references to a chart by searching for
// the chart URL through the Wordpress database.
// The Grapher should work without the Wordpress database, so we need to
// handle failures gracefully.
// NOTE: Sometimes slugs can be substrings of other slugs, e.g.
// `grapher/gdp` is a substring of `grapher/gdp-maddison`. We need to be
// careful not to erroneously match those, which is why we switched to a
// REGEXP.
try {
const posts = await wpdb.singleton.query(
`
SELECT ID, post_title, post_name
FROM wp_posts
WHERE
(post_type='page' OR post_type='post' OR post_type='wp_block')
AND post_status='publish'
AND (
${slugs
.map(
() =>
`post_content REGEXP CONCAT('grapher/', ?, '[^a-zA-Z_\-]')`
)
.join(" OR ")}
)
`,
slugs.map(lodash.escapeRegExp)
)
return posts
} catch (error) {
console.warn(`Error in getReferencesByChartId`)
console.error(error)
// We can ignore errors due to not being able to connect.
return []
}
}

const getReferencesByChartId = async (chartId: number): Promise<References> => {
const rows = await db.queryMysql(
`
SELECT config->"$.slug" AS slug
FROM charts
WHERE id = ?
UNION
SELECT slug AS slug
FROM chart_slug_redirects
WHERE chart_id = ?
`,
[chartId, chartId]
)

const slugs: string[] = rows
.map(
(row: { slug?: string }) =>
row.slug && row.slug.replace(/^"|"$/g, "")
)
.filter((slug: string | undefined) => !isUndefined(slug))

if (!slugs || slugs.length === 0)
return {
postsGdocs: [],
postsWordpress: [],
explorers: [],
}

const postsPromise = getPostsForSlugs(slugs)
const permalinksPromise = getPermalinks()
const publishedLinksToChartPromise = Link.getPublishedLinksTo(
slugs,
OwidGdocLinkType.Grapher
)
const postsWordpressPromise = getWordpressPostReferencesByChartId(chartId)
const postGdocsPromise = getGdocsPostReferencesByChartId(chartId)
const explorerSlugsPromise = db.queryMysql(
`select distinct explorerSlug from explorer_charts where chartId = ?`,
[chartId]
)
const [posts, permalinks, publishedLinksToChart, explorerSlugs] =
await Promise.all([
postsPromise,
permalinksPromise,
publishedLinksToChartPromise,
explorerSlugsPromise,
])

const publishedGdocPostsThatReferenceChart = publishedLinksToChart.map(
(link) => ({
id: link.source.id,
title: link.source.content.title ?? "",
slug: link.source.slug,
url: `${BAKED_BASE_URL}/${link.source.slug}`,
})
)

const publishedWPPostsThatReferenceChart = posts.map((post) => {
const slug = permalinks.get(post.ID, post.post_name)
return {
id: post.ID.toString(),
title: post.post_title,
slug: slug,
url: `${BAKED_BASE_URL}/${slug}`,
}
})
const [postsWordpress, postsGdocs, explorerSlugs] = await Promise.all([
postsWordpressPromise,
postGdocsPromise,
explorerSlugsPromise,
])

return {
postsGdocs: publishedGdocPostsThatReferenceChart,
postsWordpress: publishedWPPostsThatReferenceChart,
postsGdocs,
postsWordpress,
explorers: explorerSlugs.map(
(row: { explorerSlug: string }) => row.explorerSlug
),
Expand Down
151 changes: 88 additions & 63 deletions db/model/Post.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,81 +294,106 @@ export const getBlockContentFromSnapshot = async (
return enrichedBlock?.wpApiSnapshot.data?.wpBlock?.content
}

/*
* Get all the related research and writing for a chart
*/
export const getRelatedArticles = async (
export const getWordpressPostReferencesByChartId = async (
chartId: number
): Promise<PostReference[] | undefined> => {
const relatedPosts: PostReference[] = (
): Promise<PostReference[]> => {
const relatedWordpressPosts: PostReference[] = (
await db.knexInstance().raw(
`
SELECT
p.title,
p.slug,
p.id
FROM
posts_with_gdoc_publish_status p
JOIN posts_links pl ON p.id = pl.sourceId
JOIN charts c ON pl.target = c.slug
OR pl.target IN (
SELECT
cr.slug
FROM
chart_slug_redirects cr
WHERE
cr.chart_id = c.id
)
WHERE
c.id = ?
AND p.status = 'publish'
AND p.isGdocPublished = 0
AND p.type != 'wp_block'
-- note: we are not filtering by linkType to cast of wider net: if a post links to an
-- explorer having the same slug as the grapher chart, we want to surface it as
-- a "Related research" as it is most likely relevant.
UNION
SELECT
pg.content ->> '$.title' AS title,
pg.slug AS slug,
pg.id AS id
FROM
posts_gdocs pg
JOIN posts_gdocs_links pgl ON pg.id = pgl.sourceId
JOIN charts c ON pgl.target = c.slug
OR pgl.target IN (
SELECT
cr.slug
FROM
chart_slug_redirects cr
WHERE
cr.chart_id = c.id
)
WHERE
c.id = ?
AND pg.content ->> '$.type' <> 'fragment'
AND pg.published = 1
-- note: we are not filtering by linkType here either, for the same reason as above.
SELECT DISTINCT
p.title,
p.slug,
p.id,
CONCAT("${BAKED_BASE_URL}","/",p.slug) as url
FROM
posts p
JOIN posts_links pl ON p.id = pl.sourceId
JOIN charts c ON pl.target = c.slug
OR pl.target IN (
SELECT
cr.slug
FROM
chart_slug_redirects cr
WHERE
cr.chart_id = c.id
)
WHERE
c.id = ?
AND p.status = 'publish'
AND p.type != 'wp_block'
AND pl.linkType = 'grapher'
AND p.slug NOT IN (
-- We want to exclude the slugs of published gdocs, since they override the Wordpress posts
-- published under the same slugs.
SELECT
slug from posts_gdocs pg
WHERE
pg.slug = p.slug
AND pg.content ->> '$.type' <> 'fragment'
AND pg.published = 1
)
ORDER BY
p.title ASC
`,
[chartId, chartId]
[chartId]
)
)[0]

return uniqBy(relatedPosts, "slug").sort(
return relatedWordpressPosts
}

export const getGdocsPostReferencesByChartId = async (
chartId: number
): Promise<PostReference[]> => {
const relatedGdocsPosts: PostReference[] = (
await db.knexInstance().raw(
`
SELECT DISTINCT
pg.content ->> '$.title' AS title,
pg.slug AS slug,
pg.id AS id,
CONCAT("${BAKED_BASE_URL}","/",pg.slug) as url
FROM
posts_gdocs pg
JOIN posts_gdocs_links pgl ON pg.id = pgl.sourceId
JOIN charts c ON pgl.target = c.slug
OR pgl.target IN (
SELECT
cr.slug
FROM
chart_slug_redirects cr
WHERE
cr.chart_id = c.id
)
WHERE
c.id = ?
AND pg.content ->> '$.type' <> 'fragment'
AND pg.published = 1
ORDER BY
pg.content ->> '$.title' ASC
`,
[chartId]
)
)[0]

return relatedGdocsPosts
}

/*
* Get all the gdocs and Wordpress posts mentioning a chart
*/
export const getRelatedArticles = async (
chartId: number
): Promise<PostReference[] | undefined> => {
const wordpressPosts = await getWordpressPostReferencesByChartId(chartId)
const gdocsPosts = await getGdocsPostReferencesByChartId(chartId)

return [...wordpressPosts, ...gdocsPosts].sort(
// Alphabetise
(a, b) => (a.title.toLowerCase() > b.title.toLowerCase() ? 1 : -1)
)
}

export const getPermalinks = async (): Promise<{
// Strip trailing slashes, and convert __ into / to allow custom subdirs like /about/media-coverage
get: (ID: number, postName: string) => string
}> => ({
// Strip trailing slashes, and convert __ into / to allow custom subdirs like /about/media-coverage
get: (ID: number, postName: string): string =>
postName.replace(/\/+$/g, "").replace(/--/g, "/").replace(/__/g, "/"),
})

export const getPostTags = async (
postId: number
): Promise<Pick<Tag, "id" | "name">[]> => {
Expand Down
3 changes: 2 additions & 1 deletion packages/@ourworldindata/types/src/domainTypes/ContentGraph.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ export interface CategoryWithEntries {
}

export interface PostReference {
id: number | string
id: string
title: string
slug: string
url: string
}
5 changes: 1 addition & 4 deletions site/RelatedArticles/RelatedArticles.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import React from "react"
import { PostReference } from "@ourworldindata/utils"
import { BAKED_BASE_URL } from "../../settings/serverSettings.js"

export const RelatedArticles = ({
articles,
Expand All @@ -11,9 +10,7 @@ export const RelatedArticles = ({
<ul className="research">
{articles.map((article) => (
<li key={article.slug}>
<a href={`${BAKED_BASE_URL}/${article.slug}`}>
{article.title}
</a>
<a href={article.url}>{article.title}</a>
</li>
))}
</ul>
Expand Down
Loading