diff --git a/.bundlewatch.config.json b/.bundlewatch.config.json index 6bc3ab83d21..ff6b8646ca9 100644 --- a/.bundlewatch.config.json +++ b/.bundlewatch.config.json @@ -6,7 +6,7 @@ }, { "path": "./dist/assets/owid.mjs", - "maxSize": "260 KB" + "maxSize": "545 KB" } ], "defaultCompression": "none" diff --git a/adminSiteClient/ChartList.tsx b/adminSiteClient/ChartList.tsx index bf073e08ae2..d86c23223be 100644 --- a/adminSiteClient/ChartList.tsx +++ b/adminSiteClient/ChartList.tsx @@ -1,17 +1,13 @@ import React from "react" import { observer } from "mobx-react" -import { action, runInAction, observable } from "mobx" -import * as lodash from "lodash" - -import { Link } from "./Link.js" +import { runInAction, observable } from "mobx" import { Tag } from "./TagBadge.js" import { bind } from "decko" -import { EditableTags, Timeago } from "./Forms.js" import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" -import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js" import { ChartTypeName, GrapherInterface } from "@ourworldindata/grapher" import { startCase } from "@ourworldindata/utils" import { References, getFullReferencesCount } from "./ChartEditor.js" +import { ChartRow } from "./ChartRow.js" // These properties are coming from OldChart.ts export interface ChartListItem { @@ -36,129 +32,6 @@ export interface ChartListItem { tags: Tag[] } -function showChartType(chart: ChartListItem) { - const chartType = chart.type ?? ChartTypeName.LineChart - const displayType = ChartTypeName[chartType] - ? startCase(ChartTypeName[chartType]) - : "Unknown" - - if (chart.tab === "map") { - if (chart.hasChartTab) return `Map + ${displayType}` - else return "Map" - } else { - if (chart.hasMapTab) return `${displayType} + Map` - else return displayType - } -} - -@observer -class ChartRow extends React.Component<{ - chart: ChartListItem - searchHighlight?: (text: string) => string | JSX.Element - availableTags: Tag[] - onDelete: (chart: ChartListItem) => void -}> { - static contextType = AdminAppContext - context!: AdminAppContextType - - async saveTags(tags: Tag[]) { - const { chart } = this.props - const json = await this.context.admin.requestJSON( - `/api/charts/${chart.id}/setTags`, - { tags }, - "POST" - ) - if (json.success) { - runInAction(() => (chart.tags = tags)) - } - } - - @action.bound onSaveTags(tags: Tag[]) { - this.saveTags(tags) - } - - render() { - const { chart, searchHighlight, availableTags } = this.props - - const highlight = searchHighlight || lodash.identity - - return ( - - - {chart.isPublished && ( - - - - )} - - - {chart.isPublished ? ( - - {highlight(chart.title ?? "")} - - ) : ( - - Draft: {" "} - {highlight(chart.title ?? "")} - - )}{" "} - {chart.variantName ? ( - - ({highlight(chart.variantName)}) - - ) : undefined} - {chart.internalNotes && ( -
- {highlight(chart.internalNotes)} -
- )} - - {chart.id} - {showChartType(chart)} - - - - - - - - - - - - Edit - - - - - - - ) - } -} - @observer export class ChartList extends React.Component<{ charts: ChartListItem[] @@ -260,3 +133,18 @@ export class ChartList extends React.Component<{ ) } } + +export function showChartType(chart: ChartListItem) { + const chartType = chart.type ?? ChartTypeName.LineChart + const displayType = ChartTypeName[chartType] + ? startCase(ChartTypeName[chartType]) + : "Unknown" + + if (chart.tab === "map") { + if (chart.hasChartTab) return `Map + ${displayType}` + else return "Map" + } else { + if (chart.hasMapTab) return `${displayType} + Map` + else return displayType + } +} diff --git a/adminSiteClient/ChartRow.tsx b/adminSiteClient/ChartRow.tsx new file mode 100644 index 00000000000..b49e2886285 --- /dev/null +++ b/adminSiteClient/ChartRow.tsx @@ -0,0 +1,124 @@ +import React from "react" +import { observer } from "mobx-react" +import { action, runInAction } from "mobx" +import * as lodash from "lodash" +import { Link } from "./Link.js" +import { Tag } from "./TagBadge.js" +import { Timeago } from "./Forms.js" +import { EditableTags, TaggableType } from "./EditableTags.js" +import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" +import { + BAKED_GRAPHER_EXPORTS_BASE_URL, + BAKED_GRAPHER_URL, +} from "../settings/clientSettings.js" +import { ChartListItem, showChartType } from "./ChartList.js" + +@observer +export class ChartRow extends React.Component<{ + chart: ChartListItem + searchHighlight?: (text: string) => string | JSX.Element + availableTags: Tag[] + onDelete: (chart: ChartListItem) => void +}> { + static contextType = AdminAppContext + context!: AdminAppContextType + + async saveTags(tags: Tag[]) { + const { chart } = this.props + const json = await this.context.admin.requestJSON( + `/api/charts/${chart.id}/setTags`, + { tags }, + "POST" + ) + if (json.success) { + runInAction(() => (chart.tags = tags)) + } + } + + @action.bound onSaveTags(tags: Tag[]) { + this.saveTags(tags) + } + + render() { + const { chart, searchHighlight, availableTags } = this.props + + const highlight = searchHighlight || lodash.identity + + return ( + + + {chart.isPublished && ( + + + + )} + + + {chart.isPublished ? ( + + {highlight(chart.title ?? "")} + + ) : ( + + Draft: {" "} + {highlight(chart.title ?? "")} + + )}{" "} + {chart.variantName ? ( + + ({highlight(chart.variantName)}) + + ) : undefined} + {chart.internalNotes && ( +
+ {highlight(chart.internalNotes)} +
+ )} + + {chart.id} + {showChartType(chart)} + + + + + + + + + + + + Edit + + + + + + + ) + } +} diff --git a/adminSiteClient/DatasetEditPage.tsx b/adminSiteClient/DatasetEditPage.tsx index 63692bd0ed9..36ed71f37bd 100644 --- a/adminSiteClient/DatasetEditPage.tsx +++ b/adminSiteClient/DatasetEditPage.tsx @@ -9,13 +9,8 @@ import { OwidSource } from "@ourworldindata/utils" import { AdminLayout } from "./AdminLayout.js" import { Link } from "./Link.js" -import { - BindString, - Toggle, - FieldsRow, - EditableTags, - Timeago, -} from "./Forms.js" +import { BindString, Toggle, FieldsRow, Timeago } from "./Forms.js" +import { EditableTags } from "./EditableTags.js" import { ChartList, ChartListItem } from "./ChartList.js" import { Tag } from "./TagBadge.js" import { VariableList, VariableListItem } from "./VariableList.js" diff --git a/adminSiteClient/DatasetList.tsx b/adminSiteClient/DatasetList.tsx index 68b54b73196..ea712a199f7 100644 --- a/adminSiteClient/DatasetList.tsx +++ b/adminSiteClient/DatasetList.tsx @@ -7,7 +7,8 @@ import { bind } from "decko" import { Link } from "./Link.js" import { Tag } from "./TagBadge.js" import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" -import { EditableTags, Timeago } from "./Forms.js" +import { Timeago } from "./Forms.js" +import { EditableTags } from "./EditableTags.js" export interface DatasetListItem { id: number diff --git a/adminSiteClient/EditTags.tsx b/adminSiteClient/EditTags.tsx new file mode 100644 index 00000000000..74c6a7dd841 --- /dev/null +++ b/adminSiteClient/EditTags.tsx @@ -0,0 +1,73 @@ +import React from "react" +import { action } from "mobx" +import { observer } from "mobx-react" +import { Tag } from "./TagBadge.js" +import { + ReactTags, + ReactTagsAPI, + Tag as TagAutocomplete, +} from "react-tag-autocomplete" + +@observer +export class EditTags extends React.Component<{ + tags: Tag[] + suggestions: Tag[] + onDelete: (index: number) => void + onAdd: (tag: Tag) => void + onSave: () => void +}> { + dismissable: boolean = true + reactTagsApi = React.createRef() + + @action.bound onClickSomewhere() { + if (this.dismissable) this.props.onSave() + this.dismissable = true + } + + @action.bound onClick() { + this.dismissable = false + } + + @action.bound onKeyDown(e: KeyboardEvent) { + if (e.key === "Escape") { + this.props.onSave() + } + } + + onAdd = (tag: TagAutocomplete) => { + this.props.onAdd(convertAutocompleteTotag(tag)) + } + + componentDidMount() { + document.addEventListener("click", this.onClickSomewhere) + document.addEventListener("keydown", this.onKeyDown) + this.reactTagsApi.current?.input?.focus() + } + + componentWillUnmount() { + document.removeEventListener("click", this.onClickSomewhere) + document.removeEventListener("keydown", this.onKeyDown) + } + + render() { + const { tags, suggestions } = this.props + return ( +
+ +
+ ) + } +} + +const convertTagToAutocomplete = (t: Tag) => ({ value: t.id, label: t.name }) +const convertAutocompleteTotag = (t: TagAutocomplete) => ({ + id: t.value as number, + name: t.label, +}) diff --git a/adminSiteClient/EditableTags.tsx b/adminSiteClient/EditableTags.tsx new file mode 100644 index 00000000000..e27c45de487 --- /dev/null +++ b/adminSiteClient/EditableTags.tsx @@ -0,0 +1,206 @@ +import React from "react" +import * as lodash from "lodash" +import { observable, action } from "mobx" +import { observer } from "mobx-react" +import { KeyChartLevel, Tag } from "@ourworldindata/utils" +import { TagBadge } from "./TagBadge.js" +import { EditTags } from "./EditTags.js" +import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" +import { faEdit, faWandMagicSparkles } from "@fortawesome/free-solid-svg-icons" + +export enum TaggableType { + Charts = "charts", +} + +interface TaggableItem { + id?: number + type: TaggableType +} + +@observer +export class EditableTags extends React.Component<{ + tags: Tag[] + suggestions: Tag[] + onSave: (tags: Tag[]) => void + disabled?: boolean + hasKeyChartSupport?: boolean + hasSuggestionsSupport?: boolean + taggable?: TaggableItem +}> { + static contextType = AdminAppContext + context!: AdminAppContextType + + @observable isEditing: boolean = false + base: React.RefObject = React.createRef() + + @observable tags: Tag[] = lodash.clone(this.props.tags) + + @action.bound onAddTag(tag: Tag) { + this.tags.push(tag) + this.tags = lodash + // we only want to keep one occurrence of the same tag, whether + // entered manually or suggested through GPT. In case GPT suggests a + // tag that is already in the list, we want to keep the first one to + // preserve its status and key chart level + .uniqBy(this.tags, (t) => t.id) + .filter(filterUncategorizedTag) + + this.ensureUncategorized() + } + + @action.bound onRemoveTag(index: number) { + this.tags.splice(index, 1) + this.ensureUncategorized() + } + + @action.bound onToggleKey(index: number) { + const currentKeyChartLevel = + this.tags[index].keyChartLevel || KeyChartLevel.None + + // We cycle through 4 states of key chart levels for a given topic / chart combination + this.tags[index].keyChartLevel = + currentKeyChartLevel === KeyChartLevel.None + ? KeyChartLevel.Top + : currentKeyChartLevel - 1 + + this.props.onSave(this.tags.filter(filterUncategorizedTag)) + } + + @action.bound ensureUncategorized() { + if (this.tags.length === 0) { + const uncategorized = this.props.suggestions.find( + (t) => t.name === "Uncategorized" + ) + if (uncategorized) this.tags.push(uncategorized) + } + } + + @action.bound onToggleEdit() { + if (this.isEditing) { + this.props.onSave( + this.tags + .filter(filterUncategorizedTag) + .map(setDefaultKeyChartLevel) + .map(setTagStatusToApprovedIfUnset) + ) + } + this.isEditing = !this.isEditing + } + + @action.bound async onSuggest() { + const { taggable } = this.props + if (!taggable?.id) return + + const json: Record<"topics", Tag[]> = await this.context.admin.getJSON( + `/api/gpt/suggest-topics/${taggable.type}/${taggable.id}.json` + ) + + if (!json?.topics?.length) return + + json.topics + .map(setDefaultKeyChartLevel) + .map(setTagStatusToPending) + .forEach((tag) => { + this.onAddTag(tag) + }) + + this.props.onSave(this.tags.filter(filterUncategorizedTag)) + } + + @action.bound onApprove(index: number) { + this.tags[index].isApproved = true + this.props.onSave(this.tags.filter(filterUncategorizedTag)) + } + + componentDidMount() { + this.componentDidUpdate() + } + + componentDidUpdate() { + this.ensureUncategorized() + } + + render() { + const { disabled, hasKeyChartSupport, hasSuggestionsSupport } = + this.props + const { tags } = this + + return ( +
+ {this.isEditing ? ( + + ) : ( +
+ {tags.map((t, i) => ( + this.onToggleKey(i) + : undefined + } + onApprove={ + hasSuggestionsSupport && + filterUncategorizedTag(t) + ? () => this.onApprove(i) + : undefined + } + key={t.id} + tag={t} + /> + ))} + {!disabled && ( + <> + {hasSuggestionsSupport && ( + + )} + + + )} +
+ )} +
+ ) + } +} + +const filterUncategorizedTag = (t: Tag) => t.name !== "Uncategorized" + +const filterUnlistedTag = (t: Tag) => t.name !== "Unlisted" + +const setDefaultKeyChartLevel = (t: Tag) => { + if (t.keyChartLevel === undefined) t.keyChartLevel = KeyChartLevel.None + return t +} + +const setTagStatusToPending = (t: Tag) => { + t.isApproved = false + return t +} + +const setTagStatusToApprovedIfUnset = (t: Tag) => { + if (t.isApproved === undefined) t.isApproved = true + return t +} diff --git a/adminSiteClient/Forms.tsx b/adminSiteClient/Forms.tsx index a33cc4f7cb1..b1087093b12 100644 --- a/adminSiteClient/Forms.tsx +++ b/adminSiteClient/Forms.tsx @@ -5,19 +5,12 @@ */ import React from "react" -import * as lodash from "lodash" import { bind } from "decko" -import { observable, action } from "mobx" +import { action } from "mobx" import { observer } from "mobx-react" import cx from "classnames" -import { - pick, - capitalize, - dayjs, - Tippy, - KeyChartLevel, -} from "@ourworldindata/utils" +import { pick, capitalize, dayjs, Tippy } from "@ourworldindata/utils" import { Colorpicker } from "./Colorpicker.js" import { faCog, @@ -989,168 +982,6 @@ export class Timeago extends React.Component<{ } } -import { TagBadge, Tag } from "./TagBadge.js" - -import ReactTags from "react-tag-autocomplete" - -@observer -class EditTags extends React.Component<{ - tags: Tag[] - suggestions: Tag[] - onDelete: (index: number) => void - onAdd: (tag: Tag) => void - onSave: () => void -}> { - dismissable: boolean = true - - @action.bound onClickSomewhere() { - if (this.dismissable) this.props.onSave() - this.dismissable = true - } - - @action.bound onClick() { - this.dismissable = false - } - - componentDidMount() { - document.addEventListener("click", this.onClickSomewhere) - } - - componentWillUnmount() { - document.removeEventListener("click", this.onClickSomewhere) - } - - render() { - const { tags, suggestions } = this.props - return ( -
- -
- ) - } -} - -const filterUncategorizedTag = (t: Tag) => t.name !== "Uncategorized" - -@observer -export class EditableTags extends React.Component<{ - tags: Tag[] - suggestions: Tag[] - onSave: (tags: Tag[]) => void - disabled?: boolean - hasKeyChartSupport?: boolean -}> { - @observable isEditing: boolean = false - base: React.RefObject = React.createRef() - - @observable tags: Tag[] = lodash.clone(this.props.tags) - - @action.bound onAddTag(tag: Tag) { - this.tags.push(tag) - this.tags = lodash - .uniqBy(this.tags, (t) => t.id) - .filter(filterUncategorizedTag) - - this.ensureUncategorized() - } - - @action.bound onRemoveTag(index: number) { - this.tags.splice(index, 1) - this.ensureUncategorized() - } - - @action.bound onToggleKey(index: number) { - const currentKeyChartLevel = - this.tags[index].keyChartLevel || KeyChartLevel.None - - // We cycle through 4 states of key chart levels for a given topic / chart combination - this.tags[index].keyChartLevel = - currentKeyChartLevel === KeyChartLevel.None - ? KeyChartLevel.Top - : currentKeyChartLevel - 1 - - this.props.onSave(this.tags.filter(filterUncategorizedTag)) - } - - @action.bound ensureUncategorized() { - if (this.tags.length === 0) { - const uncategorized = this.props.suggestions.find( - (t) => t.name === "Uncategorized" - ) - if (uncategorized) this.tags.push(uncategorized) - } - } - - @action.bound onToggleEdit() { - if (this.isEditing) { - // Add a default key chart level to new tags - this.tags.forEach( - (tag) => - (tag.keyChartLevel = - tag.keyChartLevel ?? KeyChartLevel.None) - ) - this.props.onSave(this.tags.filter(filterUncategorizedTag)) - } - this.isEditing = !this.isEditing - } - - componentDidMount() { - this.componentDidUpdate() - } - - componentDidUpdate() { - this.ensureUncategorized() - } - - render() { - const { disabled, hasKeyChartSupport } = this.props - const { tags } = this - - return ( -
- {this.isEditing ? ( - - ) : ( -
- {tags.map((t, i) => ( - this.onToggleKey(i) - : undefined - } - key={t.id} - tag={t} - /> - ))} - {!disabled && ( - - )} -
- )} -
- ) - } -} - @observer export class Button extends React.Component<{ children: any diff --git a/adminSiteClient/GdocsEditLink.tsx b/adminSiteClient/GdocsEditLink.tsx index a1562625c4c..57c4d1baf39 100644 --- a/adminSiteClient/GdocsEditLink.tsx +++ b/adminSiteClient/GdocsEditLink.tsx @@ -16,7 +16,7 @@ export const GdocsEditLink = ({ className="gdoc-edit-link" rel="noopener noreferrer" > + Edit - ) diff --git a/adminSiteClient/GdocsIndexPage.tsx b/adminSiteClient/GdocsIndexPage.tsx index 8e6ac97880c..471b46dc9a0 100644 --- a/adminSiteClient/GdocsIndexPage.tsx +++ b/adminSiteClient/GdocsIndexPage.tsx @@ -1,7 +1,8 @@ import React from "react" import cx from "classnames" import { AdminLayout } from "./AdminLayout.js" -import { EditableTags, Modal, SearchField } from "./Forms.js" +import { Modal, SearchField } from "./Forms.js" +import { EditableTags } from "./EditableTags.js" import { faCirclePlus, faLightbulb, diff --git a/adminSiteClient/PostsIndexPage.tsx b/adminSiteClient/PostsIndexPage.tsx index 793fb804bf6..a42f3303c78 100644 --- a/adminSiteClient/PostsIndexPage.tsx +++ b/adminSiteClient/PostsIndexPage.tsx @@ -6,7 +6,8 @@ import * as lodash from "lodash" import { highlight as fuzzyHighlight } from "@ourworldindata/grapher" import { AdminLayout } from "./AdminLayout.js" -import { SearchField, FieldsRow, EditableTags, Timeago } from "./Forms.js" +import { SearchField, FieldsRow, Timeago } from "./Forms.js" +import { EditableTags } from "./EditableTags.js" import { AdminAppContext, AdminAppContextType } from "./AdminAppContext.js" import { WORDPRESS_URL } from "../settings/clientSettings.js" import { Tag } from "./TagBadge.js" diff --git a/adminSiteClient/TagBadge.tsx b/adminSiteClient/TagBadge.tsx index cbc082f99d0..02c18d2d2a4 100644 --- a/adminSiteClient/TagBadge.tsx +++ b/adminSiteClient/TagBadge.tsx @@ -5,6 +5,9 @@ import { KeyChartLevel, Tag } from "@ourworldindata/utils" import { Link } from "./Link.js" import Tippy from "@tippyjs/react" import { TagBucketSortingIcon } from "./TagBucketSortingIcon.js" +import cx from "classnames" +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" +import { faClock } from "@fortawesome/free-solid-svg-icons" export type { Tag } @@ -12,6 +15,7 @@ export type { Tag } export class TagBadge extends React.Component<{ tag: Tag onToggleKey?: () => void + onApprove?: () => void searchHighlight?: (text: string) => string | JSX.Element }> { levelToDesc(level?: KeyChartLevel) { @@ -28,20 +32,30 @@ export class TagBadge extends React.Component<{ } render() { - const { tag, searchHighlight, onToggleKey } = this.props + const { tag, searchHighlight, onToggleKey, onApprove } = this.props + const isPending = !tag.isApproved && onApprove const keyChartLevelDesc = tag.keyChartLevel === KeyChartLevel.None ? "Not a key chart, will be hidden in the all charts block of the topic page" : `Chart will show ${this.levelToDesc( tag.keyChartLevel )} of the all charts block on the topic page` - return ( - + {searchHighlight ? searchHighlight(tag.name) : tag.name} - {onToggleKey ? ( + {isPending ? ( + + + + + + ) : onToggleKey ? ( { const { newVariable, isV2MetadataVariable } = this const isDisabled = true + const pathFragments = variable.catalogPath + ? getETLPathComponents(variable.catalogPath) + : undefined + if (this.isDeleted) return @@ -116,22 +121,55 @@ class VariableEditor extends React.Component<{ variable: VariablePageData }> {

Indicator metadata

{isV2MetadataVariable && ( - - View data page - + <> + + View data page + +

+ )} +

Metadata is non-editable and can be only changed in ETL.

+

+ Open the metadata.yaml file: + + garden level + + ,{" "} + + grapher level + + .{" "} + + (opens on master branch - switch as + needed in the Github UI) + +

+ { } const tagsByPostId = await getTagsByPostId() const tags = - tagsByPostId.get(postId)?.map(({ id }) => Tag.create({ id })) || [] + tagsByPostId.get(postId)?.map(({ id }) => TagEntity.create({ id })) || + [] const archieMl = JSON.parse(post.archieml) as OwidGdocInterface const gdocId = await createGdocAndInsertOwidGdocContent( archieMl.content, @@ -2590,7 +2599,7 @@ apiRouter.post( const gdoc = await Gdoc.findOneBy({ id: gdocId }) if (!gdoc) return Error(`Unable to find Gdoc with ID: ${gdocId}`) const tags = await dataSource - .getRepository(Tag) + .getRepository(TagEntity) .findBy({ id: In(tagIds) }) gdoc.tags = tags await gdoc.save() @@ -2598,4 +2607,71 @@ apiRouter.post( } ) +apiRouter.get( + `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, + async (req: Request, res: Response): Promise> => { + const openai = new OpenAI() + const chart = await Chart.findOneBy({ + id: parseIntOrUndefined(req.params.chartId), + }) + if (!chart) + throw new JsonError( + `No chart found for id ${req.params.chartId}`, + 404 + ) + + const topics: Tag[] = await db.queryMysql(` + SELECT t.id, t.name + FROM tags t + WHERE t.isTopic IS TRUE + AND t.parentId IN (${PUBLIC_TAG_PARENT_IDS.join(",")}) + `) + + if (!topics.length) throw new JsonError("No topics found", 404) + + const prompt = ` + You will be provided with the chart metadata (delimited with XML tags), + as well as a list of possible topics (delimited with XML tags). + Classify the chart into two of the provided topics. + + ${chart.config.title} + ${chart.config.subtitle} + ${chart.config.originUrl} + + + ${topics.map( + (topic) => `${topic.name}\n` + )} + + + Respond with the two categories you think best describe the chart. + + Format your response as follows: + [ + { "id": 1, "name": "Topic 1" }, + { "id": 2, "name": "Topic 2" } + ]` + + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: prompt }], + model: "gpt-3.5-turbo", + }) + + const json = completion.choices[0]?.message?.content + if (!json) throw new JsonError("No response from GPT", 500) + + const selectedTopics: Tag[] = JSON.parse(json) + + // We only want to return topics that are in the list of possible + // topics, in case of hallucinations + const confirmedTopics = selectedTopics.filter((topic) => + topics.map((t) => t.id).includes(topic.id) + ) + + return { + topics: confirmedTopics, + } + } +) + export { apiRouter } diff --git a/baker/BuildkiteTrigger.ts b/baker/BuildkiteTrigger.ts index 6ba269f6805..24be01cdb9e 100644 --- a/baker/BuildkiteTrigger.ts +++ b/baker/BuildkiteTrigger.ts @@ -78,7 +78,9 @@ export class BuildkiteTrigger { if (status === "passed") { return } else { - throw new Error("Build failed! See Buildkite for details.") + throw new Error( + `Build failed with status "${status}". See Buildkite for details.` + ) } } diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 81c097f0485..068a7d6f546 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -10,6 +10,7 @@ import { ALGOLIA_SECRET_KEY, } from "../../settings/serverSettings.js" import { countries } from "@ourworldindata/utils" +import { SearchIndexName } from "../../site/search/searchTypes.js" export const CONTENT_GRAPH_ALGOLIA_INDEX = "graph" @@ -55,7 +56,7 @@ export const configureAlgolia = async () => { unretrievableAttributes: ["views_7d", "score"], } - const chartsIndex = client.initIndex("charts") + const chartsIndex = client.initIndex(SearchIndexName.Charts) await chartsIndex.setSettings({ ...baseSettings, @@ -86,7 +87,7 @@ export const configureAlgolia = async () => { optionalWords: ["vs"], }) - const pagesIndex = client.initIndex("pages") + const pagesIndex = client.initIndex(SearchIndexName.Pages) await pagesIndex.setSettings({ ...baseSettings, @@ -109,6 +110,22 @@ export const configureAlgolia = async () => { disableExactOnAttributes: ["tags"], }) + const explorersIndex = client.initIndex(SearchIndexName.Explorers) + + await explorersIndex.setSettings({ + ...baseSettings, + searchableAttributes: [ + "unordered(slug)", + "unordered(title)", + "unordered(subtitle)", + "unordered(text)", + ], + customRanking: ["desc(views_7d)"], + attributeForDistinct: "slug", + attributesForFaceting: [], + disableTypoToleranceOnAttributes: ["text"], + }) + const synonyms = [ ["kids", "children"], ["pork", "pigmeat"], @@ -248,6 +265,9 @@ export const configureAlgolia = async () => { await chartsIndex.saveSynonyms(algoliaSynonyms, { replaceExistingSynonyms: true, }) + await explorersIndex.saveSynonyms(algoliaSynonyms, { + replaceExistingSynonyms: true, + }) if (TOPICS_CONTENT_GRAPH) { const graphIndex = client.initIndex(CONTENT_GRAPH_ALGOLIA_INDEX) diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts index 07e4ef60e8a..ad3e4836290 100644 --- a/baker/algolia/indexChartsToAlgolia.ts +++ b/baker/algolia/indexChartsToAlgolia.ts @@ -3,7 +3,7 @@ import { getRelatedArticles } from "../../db/wpdb.js" import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" import { getAlgoliaClient } from "./configureAlgolia.js" import { isPathRedirectedToExplorer } from "../../explorerAdminServer/ExplorerRedirects.js" -import { ChartRecord } from "../../site/search/searchTypes.js" +import { ChartRecord, SearchIndexName } from "../../site/search/searchTypes.js" import { KeyChartLevel, MarkdownTextWrap } from "@ourworldindata/utils" import { Pageview } from "../../db/model/Pageview.js" import { Link } from "../../db/model/Link.js" @@ -105,7 +105,7 @@ const indexChartsToAlgolia = async () => { return } - const index = client.initIndex("charts") + const index = client.initIndex(SearchIndexName.Charts) await db.getConnection() const records = await getChartsRecords() diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts new file mode 100644 index 00000000000..160fafab58c --- /dev/null +++ b/baker/algolia/indexExplorersToAlgolia.ts @@ -0,0 +1,155 @@ +import cheerio from "cheerio" +import { isArray } from "lodash" +import { match } from "ts-pattern" +import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils" +import { getAlgoliaClient } from "./configureAlgolia.js" +import * as db from "../../db/db.js" +import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" +import { Pageview } from "../../db/model/Pageview.js" +import { chunkParagraphs } from "../chunk.js" +import { SearchIndexName } from "../../site/search/searchTypes.js" + +type ExplorerBlockColumns = { + type: "columns" + block: { name: string; additionalInfo?: string }[] +} + +type ExplorerBlockGraphers = { + type: "graphers" + block: { + title: string + subtitle?: string + }[] +} + +type ExplorerEntry = { + slug: string + title: string + subtitle: string + views_7d: number + blocks: string // (ExplorerBlockLineChart | ExplorerBlockColumns | ExplorerBlockGraphers)[] +} + +type ExplorerRecord = { + slug: string + title: string + subtitle: string + views_7d: number + text: string +} + +function extractTextFromExplorer(blocksString: string): string { + const blockText = new Set() + const blocks = JSON.parse(blocksString) + + if (isArray(blocks)) { + for (const block of blocks) { + if (checkIsPlainObjectWithGuard(block) && "type" in block) { + match(block) + .with( + { type: "columns" }, + (columns: ExplorerBlockColumns) => { + columns.block.forEach( + ({ name = "", additionalInfo = "" }) => { + blockText.add(name) + blockText.add(additionalInfo) + } + ) + } + ) + .with( + { type: "graphers" }, + (graphers: ExplorerBlockGraphers) => { + graphers.block.forEach( + ({ title = "", subtitle = "" }) => { + blockText.add(title) + blockText.add(subtitle) + } + ) + } + ) + .otherwise(() => { + // type: "tables" + // do nothing + }) + } + } + } + + return [...blockText].join(" ") +} + +function getNullishJSONValueAsPlaintext(value: string): string { + return value !== "null" ? cheerio.load(value)("body").text() : "" +} + +const getExplorerRecords = async (): Promise => { + const pageviews = await Pageview.getViewsByUrlObj() + + const explorerRecords = await db + .queryMysql( + ` + SELECT slug, + COALESCE(config->>"$.explorerSubtitle", "null") AS subtitle, + COALESCE(config->>"$.explorerTitle", "null") AS title, + COALESCE(config->>"$.blocks", "null") AS blocks + FROM explorers + WHERE isPublished = true + ` + ) + .then((results: ExplorerEntry[]) => + results.flatMap(({ slug, title, subtitle, blocks }) => { + const textFromExplorer = extractTextFromExplorer(blocks) + const uniqueTextTokens = new Set([ + ...textFromExplorer.split(" "), + ]) + const textChunks = chunkParagraphs( + [...uniqueTextTokens].join(" "), + 1000 + ) + const explorerRecords = [] + let i = 0 + for (const chunk of textChunks) { + explorerRecords.push({ + slug, + title: getNullishJSONValueAsPlaintext(title), + subtitle: getNullishJSONValueAsPlaintext(subtitle), + views_7d: + pageviews[`/explorers/${slug}`]?.views_7d || 0, + text: chunk, + objectID: `${slug}-${i}`, + }) + i++ + } + return explorerRecords + }) + ) + + return explorerRecords +} + +const indexExplorersToAlgolia = async () => { + if (!ALGOLIA_INDEXING) return + + const client = getAlgoliaClient() + if (!client) { + console.error( + `Failed indexing explorers (Algolia client not initialized)` + ) + return + } + + try { + const index = client.initIndex(SearchIndexName.Explorers) + + await db.getConnection() + const records = await getExplorerRecords() + await index.replaceAllObjects(records) + + await db.closeTypeOrmAndKnexConnections() + } catch (e) { + console.log("Error indexing explorers to Algolia: ", e) + } +} + +indexExplorersToAlgolia() diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index f7553d44570..79e5fea883c 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -15,7 +15,11 @@ import { formatPost } from "../formatWordpressPost.js" import ReactDOMServer from "react-dom/server.js" import { getAlgoliaClient } from "./configureAlgolia.js" import { htmlToText } from "html-to-text" -import { PageRecord, PageType } from "../../site/search/searchTypes.js" +import { + PageRecord, + PageType, + SearchIndexName, +} from "../../site/search/searchTypes.js" import { Pageview } from "../../db/model/Pageview.js" import { Gdoc } from "../../db/model/Gdoc/Gdoc.js" import { ArticleBlocks } from "../../site/gdocs/ArticleBlocks.js" @@ -216,7 +220,7 @@ const indexToAlgolia = async () => { console.error(`Failed indexing pages (Algolia client not initialized)`) return } - const index = client.initIndex("pages") + const index = client.initIndex(SearchIndexName.Pages) await db.getConnection() const records = await getPagesRecords() diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index d3a58080395..05da0c012c0 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -65,9 +65,6 @@ import { getPostBySlug, isPostCitable, getBlockContent, - getPosts, - mapGdocsToWordpressPosts, - getFullPost, } from "../db/wpdb.js" import { queryMysql, knexTable } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" @@ -226,11 +223,7 @@ export const renderPost = async ( } export const renderFrontPage = async () => { - const wpPosts = await Promise.all( - (await getPosts()).map((post) => getFullPost(post, true)) - ) - const gdocPosts = await Gdoc.getPublishedGdocs() - const posts = [...wpPosts, ...mapGdocsToWordpressPosts(gdocPosts)] + const posts = await getBlogIndex() const NUM_FEATURED_POSTS = 6 @@ -280,6 +273,10 @@ export const renderFrontPage = async () => { if (post) { return post } + console.log( + "Featured work error: could not find listed post with slug: ", + manuallySetPost.slug + ) } return filteredPosts[missingPosts++] }) diff --git a/datapage/Datapage.ts b/datapage/Datapage.ts index a6d62dc9039..eb964304e4b 100644 --- a/datapage/Datapage.ts +++ b/datapage/Datapage.ts @@ -19,23 +19,12 @@ import { dayjs, getAttributionFromVariable, gdocIdRegex, + getETLPathComponents, } from "@ourworldindata/utils" import { ExplorerProgram } from "../explorer/ExplorerProgram.js" import { Gdoc } from "../db/model/Gdoc/Gdoc.js" import { GrapherInterface } from "@ourworldindata/grapher" -interface ETLPathComponents { - channel: string - publisher: string - version: string - dataset: string -} - -const getETLPathComponents = (path: string): ETLPathComponents => { - const [channel, publisher, version, dataset] = path.split("/") - return { channel, publisher, version, dataset } -} - export const getDatapageDataV2 = async ( variableMetadata: OwidVariableWithSource, partialGrapherConfig: GrapherInterface @@ -63,7 +52,7 @@ export const getDatapageDataV2 = async ( variableMetadata.descriptionShort ?? partialGrapherConfig.subtitle, descriptionFromProducer: variableMetadata.descriptionFromProducer, - producerShort: variableMetadata.presentation?.producerShort, + attributionShort: variableMetadata.presentation?.attributionShort, titleVariant: variableMetadata.presentation?.titleVariant, topicTagsLinks: variableMetadata.presentation?.topicTagsLinks ?? [], attribution: getAttributionFromVariable(variableMetadata), diff --git a/db/migrateWpPostsToArchieMl.ts b/db/migrateWpPostsToArchieMl.ts index c25cc1fed7e..37f749b0238 100644 --- a/db/migrateWpPostsToArchieMl.ts +++ b/db/migrateWpPostsToArchieMl.ts @@ -17,6 +17,136 @@ import { adjustHeadingLevels, } from "./model/Gdoc/htmlToEnriched.js" +// Hard-coded slugs to avoid WP dependency +const entries = new Set([ + "population", + "population-change", + "age-structure", + "gender-ratio", + "life-and-death", + "life-expectancy", + "child-mortality", + "fertility-rate", + "distribution-of-the-world-population", + "urbanization", + "health", + "health-risks", + "air-pollution", + "outdoor-air-pollution", + "indoor-air-pollution", + "obesity", + "smoking", + "alcohol-consumption", + "infectious-diseases", + "monkeypox", + "coronavirus", + "hiv-aids", + "malaria", + "eradication-of-diseases", + "smallpox", + "polio", + "pneumonia", + "tetanus", + "health-institutions-and-interventions", + "financing-healthcare", + "vaccination", + "life-death-health", + "maternal-mortality", + "health-meta", + "causes-of-death", + "burden-of-disease", + "cancer", + "environment", + "nuclear-energy", + "energy-access", + "renewable-energy", + "fossil-fuels", + "waste", + "plastic-pollution", + "air-and-climate", + "co2-and-greenhouse-gas-emissions", + "climate-change", + "water", + "clean-water-sanitation", + "water-access", + "sanitation", + "water-use-stress", + "land-and-ecosystems", + "forests-and-deforestation", + "land-use", + "natural-disasters", + "food", + "nutrition", + "famines", + "food-supply", + "human-height", + "micronutrient-deficiency", + "diet-compositions", + "food-production", + "meat-production", + "agricultural-inputs", + "employment-in-agriculture", + "growth-inequality", + "public-sector", + "government-spending", + "taxation", + "military-personnel-spending", + "financing-education", + "poverty-and-prosperity", + "economic-inequality", + "poverty", + "economic-growth", + "economic-inequality-by-gender", + "labor", + "child-labor", + "working-hours", + "female-labor-supply", + "corruption", + "trade-migration", + "trade-and-globalization", + "tourism", + "education", + "educational-outcomes", + "global-education", + "literacy", + "pre-primary-education", + "primary-and-secondary-education", + "quality-of-education", + "tertiary-education", + "inputs-to-education", + "teachers-and-professors", + "media-education", + "technology", + "space-exploration-satellites", + "transport", + "work-life", + "culture", + "trust", + "housing", + "homelessness", + "time-use", + "relationships", + "marriages-and-divorces", + "social-connections-and-loneliness", + "happiness-wellbeing", + "happiness-and-life-satisfaction", + "human-development-index", + "politics", + "human-rights", + "lgbt-rights", + "women-rights", + "democracy", + "violence-rights", + "war-peace", + "biological-and-chemical-weapons", + "war-and-peace", + "terrorism", + "nuclear-weapons", + "violence", + "violence-against-rights-for-children", + "homicides", +]) + const migrate = async (): Promise => { const writeToFile = false const errors = [] @@ -33,7 +163,7 @@ const migrate = async (): Promise => { "excerpt", "created_at_in_wordpress", "updated_at" - ).from(db.knexTable(Post.postsTable)) //.where("id", "=", "22821")) + ).from(db.knexTable(Post.postsTable)) //.where("id", "=", "38189") for (const post of posts) { try { @@ -83,6 +213,7 @@ const migrate = async (): Promise => { slug: post.slug, content: { body: archieMlBodyElements, + toc: [], title: post.title, subtitle: post.excerpt, excerpt: post.excerpt, @@ -92,7 +223,9 @@ const migrate = async (): Promise => { dateline: dateline, // TODO: this discards block level elements - those might be needed? refs: undefined, - type: OwidGdocType.Article, + type: entries.has(post.slug) + ? OwidGdocType.TopicPage + : OwidGdocType.Article, }, published: false, createdAt: diff --git a/db/migration/1694348415243-AddIsIsApprovedToChartTags.ts b/db/migration/1694348415243-AddIsIsApprovedToChartTags.ts new file mode 100644 index 00000000000..e59e62bee67 --- /dev/null +++ b/db/migration/1694348415243-AddIsIsApprovedToChartTags.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class AddIsIsApprovedToChartTags1694348415243 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE chart_tags + ADD COLUMN isApproved TINYINT(1) NOT NULL DEFAULT 0; + `) + await queryRunner.query(` + UPDATE chart_tags SET isApproved = 1; + `) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE chart_tags + DROP COLUMN isApproved; + `) + } +} diff --git a/db/migration/1694509192714-RenameVariableMetadataV2.ts b/db/migration/1694509192714-RenameVariableMetadataV2.ts new file mode 100644 index 00000000000..dbe079f1002 --- /dev/null +++ b/db/migration/1694509192714-RenameVariableMetadataV2.ts @@ -0,0 +1,48 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class RenameVariableMetadataV21694509192714 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE variables + RENAME COLUMN producerShort TO attributionShort; + `) + + await queryRunner.query(` + ALTER TABLE origins + RENAME COLUMN datasetUrlMain TO urlMain, + RENAME COLUMN datasetUrlDownload TO urlDownload, + RENAME COLUMN datasetTitleProducer TO title, + RENAME COLUMN datasetDescriptionProducer TO description, + RENAME COLUMN datasetTitleOwid TO titleSnapshot, + RENAME COLUMN datasetDescriptionOwid TO descriptionSnapshot, + RENAME COLUMN citationProducer TO citationFull; + `) + + await queryRunner.query(` + DROP INDEX idx_datasetTitleOwid ON origins; + `) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE variables + RENAME COLUMN attributionShort TO producerShort; + `) + + await queryRunner.query(` + ALTER TABLE origins + RENAME COLUMN urlMain TO datasetUrlMain, + RENAME COLUMN urlDownload TO datasetUrlDownload, + RENAME COLUMN title TO datasetTitleProducer, + RENAME COLUMN description TO datasetDescriptionProducer, + RENAME COLUMN titleSnapshot TO datasetTitleOwid, + RENAME COLUMN descriptionSnapshot TO datasetDescriptionOwid, + RENAME COLUMN citationFull TO citationProducer; + `) + + await queryRunner.query(`-- sql + CREATE INDEX idx_datasetTitleOwid ON origins(datasetTitleOwid);`) + } +} diff --git a/db/migration/1694549232436-IterableResearchAndWriting.ts b/db/migration/1694549232436-IterableResearchAndWriting.ts new file mode 100644 index 00000000000..c0c41cf6d61 --- /dev/null +++ b/db/migration/1694549232436-IterableResearchAndWriting.ts @@ -0,0 +1,55 @@ +import { OwidGdocContent } from "@ourworldindata/utils" +import { MigrationInterface, QueryRunner } from "typeorm" + +export class IterableResearchAndWriting1694549232436 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await migrateResearchAndWritingBlocks(queryRunner, (node) => { + const primary = node.primary as any + const secondary = node.secondary as any + node.primary = [primary] + node.secondary = [secondary] + }) + } + + public async down(queryRunner: QueryRunner): Promise { + await migrateResearchAndWritingBlocks(queryRunner, (node) => { + const primary = node.primary as any + const secondary = node.secondary as any + node.primary = primary[0] + node.secondary = secondary[0] + }) + } +} + +async function migrateResearchAndWritingBlocks( + queryRunner: QueryRunner, + callback: (node: any) => void +): Promise { + const allGdocs = await queryRunner.query( + "SELECT id, slug, content FROM posts_gdocs" + ) + for (const gdoc of allGdocs) { + const content = JSON.parse(gdoc.content) as OwidGdocContent + if (!content.body) continue + + let hasResearchAndWriting = false + + // Not recursively traversing because none of our topic pages have research-and-writing blocks nested inside containers + content.body.forEach((node) => { + if (node.type === "research-and-writing") { + hasResearchAndWriting = true + callback(node) + } + return node + }) + + if (hasResearchAndWriting) { + await queryRunner.query( + "UPDATE posts_gdocs SET content = ? WHERE id = ?", + [JSON.stringify(content), gdoc.id] + ) + } + } +} diff --git a/db/model/Chart.ts b/db/model/Chart.ts index f8821284b50..f05f651ddc4 100644 --- a/db/model/Chart.ts +++ b/db/model/Chart.ts @@ -13,13 +13,14 @@ import { getDataForMultipleVariables } from "./Variable.js" import { User } from "./User.js" import { ChartRevision } from "./ChartRevision.js" import { + KeyChartLevel, MultipleOwidVariableDataDimensionsMap, Tag, } from "@ourworldindata/utils" import type { GrapherInterface } from "@ourworldindata/grapher" // XXX hardcoded filtering to public parent tags -const PUBLIC_TAG_PARENT_IDS = [ +export const PUBLIC_TAG_PARENT_IDS = [ 1515, 1507, 1513, 1504, 1502, 1509, 1506, 1501, 1514, 1511, 1500, 1503, 1505, 1508, 1512, 1510, ] @@ -116,12 +117,13 @@ WHERE c.config -> "$.isPublished" = true const tagRows = tags.map((tag) => [ tag.id, chartId, - tag.keyChartLevel, + tag.keyChartLevel ?? KeyChartLevel.None, + tag.isApproved ? 1 : 0, ]) await t.execute(`DELETE FROM chart_tags WHERE chartId=?`, [chartId]) if (tagRows.length) await t.execute( - `INSERT INTO chart_tags (tagId, chartId, keyChartLevel) VALUES ?`, + `INSERT INTO chart_tags (tagId, chartId, keyChartLevel, isApproved) VALUES ?`, [tagRows] ) @@ -130,10 +132,14 @@ WHERE c.config -> "$.isPublished" = true tags.map((t) => t.id), ])) as { parentId: number }[]) : [] - const isIndexable = parentIds.some((t) => - PUBLIC_TAG_PARENT_IDS.includes(t.parentId) - ) + // A chart is indexable if it is not tagged "Unlisted" and has at + // least one public parent tag + const isIndexable = tags.some((t) => t.name === "Unlisted") + ? false + : parentIds.some((t) => + PUBLIC_TAG_PARENT_IDS.includes(t.parentId) + ) await t.execute("update charts set is_indexable = ? where id = ?", [ isIndexable, chartId, @@ -145,7 +151,7 @@ WHERE c.config -> "$.isPublished" = true charts: { id: number; tags: any[] }[] ): Promise { const chartTags = await db.queryMysql(` - SELECT ct.chartId, ct.tagId, ct.keyChartLevel, t.name as tagName FROM chart_tags ct + SELECT ct.chartId, ct.tagId, ct.keyChartLevel, ct.isApproved, t.name as tagName FROM chart_tags ct JOIN charts c ON c.id=ct.chartId JOIN tags t ON t.id=ct.tagId `) @@ -163,6 +169,7 @@ WHERE c.config -> "$.isPublished" = true id: ct.tagId, name: ct.tagName, keyChartLevel: ct.keyChartLevel, + isApproved: !!ct.isApproved, }) } } diff --git a/db/model/Gdoc/Gdoc.ts b/db/model/Gdoc/Gdoc.ts index 78af8035191..92c075c1ae0 100644 --- a/db/model/Gdoc/Gdoc.ts +++ b/db/model/Gdoc/Gdoc.ts @@ -605,6 +605,7 @@ export class Gdoc extends BaseEntity implements OwidGdocInterface { "aside", "callout", "expandable-paragraph", + "entry-summary", "gray-section", "heading", "horizontal-rule", diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index a80df713323..5874eee862c 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -32,6 +32,7 @@ import { EnrichedBlockResearchAndWritingLink, RawBlockResearchAndWritingLink, RawBlockAlign, + RawBlockEntrySummary, } from "@ourworldindata/utils" import { spanToHtmlString } from "./gdocUtils.js" import { match, P } from "ts-pattern" @@ -347,14 +348,20 @@ export function enrichedBlockToRawBlock( return { type: b.type, value: { - primary: enrichedLinkToRawLink(b.primary), - secondary: enrichedLinkToRawLink(b.secondary), - more: { - heading: b.more.heading, - articles: b.more.articles.map( - enrichedLinkToRawLink - ), - }, + primary: b.primary.map((enriched) => + enrichedLinkToRawLink(enriched) + ), + secondary: b.secondary.map((enriched) => + enrichedLinkToRawLink(enriched) + ), + more: b.more + ? { + heading: b.more.heading, + articles: b.more.articles.map( + enrichedLinkToRawLink + ), + } + : undefined, rows: b.rows.map(({ heading, articles }) => ({ heading: heading, articles: articles.map(enrichedLinkToRawLink), @@ -372,5 +379,13 @@ export function enrichedBlockToRawBlock( }, } }) + .with({ type: "entry-summary" }, (b): RawBlockEntrySummary => { + return { + type: b.type, + value: { + items: b.items, + }, + } + }) .exhaustive() } diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index d852460e6c3..78c58d70e1c 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -355,16 +355,25 @@ export const enrichedBlockExamples: Record< "research-and-writing": { type: "research-and-writing", parseErrors: [], - primary: { - value: { - url: "https://docs.google.com/document/d/abcd", + primary: [ + { + value: { + url: "https://docs.google.com/document/d/abcd", + }, }, - }, - secondary: { - value: { - url: "https://docs.google.com/document/d/abcd", + ], + secondary: [ + { + value: { + url: "https://docs.google.com/document/d/1234", + }, }, - }, + { + value: { + url: "https://docs.google.com/document/d/5678", + }, + }, + ], more: { heading: "More Key Articles on Poverty", articles: [ @@ -418,4 +427,9 @@ export const enrichedBlockExamples: Record< content: [enrichedBlockText], parseErrors: [], }, + "entry-summary": { + type: "entry-summary", + items: [{ text: "Hello", slug: "#link-to-something" }], + parseErrors: [], + }, } diff --git a/db/model/Gdoc/gdocUtils.ts b/db/model/Gdoc/gdocUtils.ts index a74dc8a4b5f..d076ffd1e81 100644 --- a/db/model/Gdoc/gdocUtils.ts +++ b/db/model/Gdoc/gdocUtils.ts @@ -139,12 +139,10 @@ export const getAllLinksFromResearchAndWritingBlock = ( ): EnrichedBlockResearchAndWritingLink[] => { const { primary, secondary, more, rows } = block const rowArticles = rows.flatMap((row) => row.articles) - const allLinks = excludeNullish([ - primary, - secondary, - ...more.articles, - ...rowArticles, - ]) + const allLinks = excludeNullish([...primary, ...secondary, ...rowArticles]) + if (more) { + allLinks.push(...more.articles) + } return allLinks } diff --git a/db/model/Gdoc/htmlToEnriched.ts b/db/model/Gdoc/htmlToEnriched.ts index 6670031cc5d..3d5960ec890 100644 --- a/db/model/Gdoc/htmlToEnriched.ts +++ b/db/model/Gdoc/htmlToEnriched.ts @@ -26,9 +26,22 @@ import { EnrichedBlockProminentLink, BlockImageSize, detailOnDemandRegex, + EnrichedBlockEntrySummary, + EnrichedBlockEntrySummaryItem, + spansToUnformattedPlainText, + checkNodeIsSpanLink, + Url, + EnrichedBlockCallout, } from "@ourworldindata/utils" import { match, P } from "ts-pattern" -import { compact, flatten, isPlainObject, partition } from "lodash" +import { + compact, + flatten, + get, + isArray, + isPlainObject, + partition, +} from "lodash" import cheerio from "cheerio" import { spansToSimpleString } from "./gdocUtils.js" @@ -223,6 +236,10 @@ type ErrorNames = | "unhandled html tag found" | "prominent link missing title" | "prominent link missing url" + | "summary item isn't text" + | "summary item doesn't have link" + | "summary item has DataValue" + | "unknown content type inside summary block" interface BlockParseError { name: ErrorNames @@ -337,11 +354,12 @@ function isArchieMlComponent( export function convertAllWpComponentsToArchieMLBlocks( blocksOrComponents: ArchieBlockOrWpComponent[] ): OwidEnrichedGdocBlock[] { - return blocksOrComponents.flatMap((blockOrComponent) => { - if (isArchieMlComponent(blockOrComponent)) return [blockOrComponent] + return blocksOrComponents.flatMap((blockOrComponentOrToc) => { + if (isArchieMlComponent(blockOrComponentOrToc)) + return [blockOrComponentOrToc] else { return convertAllWpComponentsToArchieMLBlocks( - blockOrComponent.childrenResults + blockOrComponentOrToc.childrenResults ) } }) @@ -596,6 +614,92 @@ function finishWpComponent( } } else return { ...content, errors } }) + .with("owid/summary", () => { + // Summaries can either be lists of anchor links, or paragraphs of text + // If it's a paragraph of text, we want to turn it into a callout block + // If it's a list of anchor links, we want to turn it into a toc block + const contentIsAllText = + content.content.find( + (block) => "type" in block && block.type !== "text" + ) === undefined + + if (contentIsAllText) { + const callout: EnrichedBlockCallout = { + type: "callout", + title: "Summary", + text: content.content as EnrichedBlockText[], + parseErrors: [], + } + return { errors: [], content: [callout] } + } + + const contentIsList = + content.content.length === 1 && + "type" in content.content[0] && + content.content[0].type === "list" + if (contentIsList) { + const listItems = get(content, ["content", 0, "items"]) + const items: EnrichedBlockEntrySummaryItem[] = [] + const errors = content.errors + if (isArray(listItems)) { + listItems.forEach((item) => { + if (item.type === "text") { + const value = item.value[0] + if (checkNodeIsSpanLink(value)) { + const { hash } = Url.fromURL(value.url) + const text = spansToUnformattedPlainText( + value.children + ) + if (text.includes("DataValue")) { + errors.push({ + name: "summary item has DataValue", + details: text, + }) + } + items.push({ + // Remove "#" from the beginning of the slug + slug: hash.slice(1), + text: text, + }) + } else { + errors.push({ + name: "summary item doesn't have link", + details: value + ? `spanType is ${value.spanType}` + : "No item", + }) + } + } else { + errors.push({ + name: "summary item isn't text", + details: `item is type: ${item.type}`, + }) + } + }) + } + const toc: EnrichedBlockEntrySummary = { + type: "entry-summary", + items, + parseErrors: [], + } + return { errors: [], content: [toc] } + } + + const error: BlockParseError = { + name: "unknown content type inside summary block", + details: + "Unknown summary content: " + + content.content + .map((block) => + "type" in block ? block.type : block.tagName + ) + .join(", "), + } + return { + errors: [error], + content: [], + } + }) .otherwise(() => { return { errors: [ diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index 8be3b5aac28..59a943423dc 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -30,6 +30,8 @@ import { RawBlockTopicPageIntro, RawBlockExpandableParagraph, RawBlockAlign, + RawBlockEntrySummary, + isArray, } from "@ourworldindata/utils" import { match } from "ts-pattern" @@ -470,14 +472,26 @@ function* rawResearchAndWritingToArchieMLString( } yield "{.research-and-writing}" if (primary) { - yield "{.primary}" - yield* rawLinkToArchie(primary) - yield "{}" + yield "[.primary]" + if (isArray(primary)) { + for (const link of primary) { + yield* rawLinkToArchie(link) + } + } else { + yield* rawLinkToArchie(primary) + } + yield "[]" } if (secondary) { - yield "{.secondary}" - yield* rawLinkToArchie(secondary) - yield "{}" + yield "[.secondary]" + if (isArray(secondary)) { + for (const link of secondary) { + yield* rawLinkToArchie(link) + } + } else { + yield* rawLinkToArchie(secondary) + } + yield "[]" } if (more) { yield "{.more}" @@ -522,6 +536,21 @@ function* rawBlockAlignToArchieMLString( yield "{}" } +function* rawBlockEntrySummaryToArchieMLString( + block: RawBlockEntrySummary +): Generator { + yield "{.entry-summary}" + yield "[.items]" + if (block.value.items) { + for (const item of block.value.items) { + yield* propertyToArchieMLString("text", item) + yield* propertyToArchieMLString("slug", item) + } + } + yield "[]" + yield "{}" +} + export function* OwidRawGdocBlockToArchieMLStringGenerator( block: OwidRawGdocBlock ): Generator { @@ -581,6 +610,7 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( rawResearchAndWritingToArchieMLString ) .with({ type: "align" }, rawBlockAlignToArchieMLString) + .with({ type: "entry-summary" }, rawBlockEntrySummaryToArchieMLString) .exhaustive() yield* content } diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index 41f57bf11e0..74912f0c06d 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -95,6 +95,9 @@ import { RawBlockAlign, FaqDictionary, EnrichedFaq, + RawBlockEntrySummary, + EnrichedBlockEntrySummary, + EnrichedBlockEntrySummaryItem, } from "@ourworldindata/utils" import { extractUrl, @@ -173,6 +176,7 @@ export function parseRawBlocksToEnrichedBlocks( ) .with({ type: "expandable-paragraph" }, parseExpandableParagraph) .with({ type: "align" }, parseAlign) + .with({ type: "entry-summary" }, parseEntrySummary) .exhaustive() } @@ -1346,12 +1350,16 @@ function parseResearchAndWritingBlock( ): EnrichedBlockResearchAndWriting { const createError = ( error: ParseError, - primary = { - value: { url: "" }, - }, - secondary = { - value: { url: "" }, - }, + primary = [ + { + value: { url: "" }, + }, + ], + secondary = [ + { + value: { url: "" }, + }, + ], more: EnrichedBlockResearchAndWritingRow = { heading: "", articles: [], @@ -1417,14 +1425,19 @@ function parseResearchAndWritingBlock( if (!raw.value.primary) return createError({ message: "Missing primary link" }) - const primary = enrichLink(raw.value.primary) - - if (!raw.value.secondary) - return createError({ message: "Missing secondary link" }) - const secondary = enrichLink(raw.value.secondary) + const primary: EnrichedBlockResearchAndWritingLink[] = [] + if (isArray(raw.value.primary)) { + primary.push(...raw.value.primary.map((link) => enrichLink(link))) + } else { + primary.push(enrichLink(raw.value.primary)) + } - if (!raw.value.more) - return createError({ message: "No 'more' section defined" }) + const secondary: EnrichedBlockResearchAndWritingLink[] = [] + if (isArray(raw.value.secondary)) { + secondary.push(...raw.value.secondary.map((link) => enrichLink(link))) + } else if (raw.value.secondary) { + secondary.push(enrichLink(raw.value.secondary)) + } function parseRow( rawRow: RawBlockResearchAndWritingRow, @@ -1450,7 +1463,7 @@ function parseResearchAndWritingBlock( return { heading: "", articles: [] } } - const more = parseRow(raw.value.more, true) + const more = raw.value.more ? parseRow(raw.value.more, true) : undefined const rows = raw.value.rows?.map((row) => parseRow(row)) || [] return { @@ -1496,6 +1509,34 @@ function parseAlign(b: RawBlockAlign): EnrichedBlockAlign { } } +function parseEntrySummary( + raw: RawBlockEntrySummary +): EnrichedBlockEntrySummary { + const parseErrors: ParseError[] = [] + const items: EnrichedBlockEntrySummaryItem[] = [] + + if (raw.value.items) { + raw.value.items.forEach((item, i) => { + if (!item.text || !item.slug) { + parseErrors.push({ + message: `entry-summary item ${i} is not valid. It must have a text and a slug property`, + }) + } else { + items.push({ + text: item.text, + slug: item.slug, + }) + } + }) + } + + return { + type: "entry-summary", + items, + parseErrors, + } +} + export function parseRefs({ refs, refsByFirstAppearance, diff --git a/db/model/Origin.ts b/db/model/Origin.ts index 905c3ece2ce..448ec86cf74 100644 --- a/db/model/Origin.ts +++ b/db/model/Origin.ts @@ -3,16 +3,10 @@ import { Entity, PrimaryGeneratedColumn, Column, BaseEntity } from "typeorm" @Entity("origins") export class Origin extends BaseEntity { @PrimaryGeneratedColumn() id!: number - @Column({ type: "varchar", nullable: true }) datasetTitleProducer!: - | string - | null - @Column({ type: "varchar", nullable: true }) datasetTitleOwid!: - | string - | null - @Column({ type: "text", nullable: true }) datasetDescriptionOwid!: - | string - | null - @Column({ type: "text", nullable: true }) datasetDescriptionProducer!: + @Column({ type: "varchar", nullable: true }) title!: string | null + @Column({ type: "varchar", nullable: true }) titleSnapshot!: string | null + @Column({ type: "text", nullable: true }) description!: string | null + @Column({ type: "text", nullable: true }) descriptionSnapshot!: | string | null @Column({ type: "varchar", nullable: true }) producer!: string | null @@ -20,9 +14,9 @@ export class Origin extends BaseEntity { @Column({ type: "varchar", nullable: true }) attributionShort!: | string | null - @Column({ type: "text", nullable: true }) citationProducer!: string | null - @Column({ type: "text", nullable: true }) datasetUrlMain!: string | null - @Column({ type: "text", nullable: true }) datasetUrlDownload!: string | null + @Column({ type: "text", nullable: true }) citationFull!: string | null + @Column({ type: "text", nullable: true }) urlMain!: string | null + @Column({ type: "text", nullable: true }) urlDownload!: string | null @Column({ type: "date", nullable: true }) dateAccessed!: Date | null @Column({ type: "varchar", nullable: true }) datePublished!: string | null @Column({ type: "varchar", nullable: true }) versionProducer!: string | null diff --git a/db/model/Variable.ts b/db/model/Variable.ts index 37e6a66b77d..f1c7e2653b6 100644 --- a/db/model/Variable.ts +++ b/db/model/Variable.ts @@ -44,7 +44,7 @@ export interface VariableRow { processingLevel?: "minor" | "major" titlePublic?: string titleVariant?: string - producerShort?: string + attributionShort?: string citationInline?: string descriptionShort?: string descriptionFromProducer?: string diff --git a/db/wpdb.ts b/db/wpdb.ts index cd721aea00b..8e6ad4957a9 100644 --- a/db/wpdb.ts +++ b/db/wpdb.ts @@ -751,24 +751,26 @@ export const getFullPost = async ( }) export const getBlogIndex = memoize(async (): Promise => { - // TODO: do not get post content in the first place - const wordpressPosts = await getPosts( - [WP_PostType.Post], - selectHomepagePosts + await db.getConnection() // side effect: ensure connection is established + const gdocPosts = await Gdoc.getListedGdocs() + const wpPosts = await Promise.all( + await getPosts([WP_PostType.Post], selectHomepagePosts).then((posts) => + posts.map((post) => getFullPost(post, true)) + ) ) - const wordpressPostsCards = await Promise.all( - wordpressPosts.map((post) => getFullPost(post, true)) - ) + const gdocSlugs = new Set(gdocPosts.map(({ slug }) => slug)) + const posts = [...mapGdocsToWordpressPosts(gdocPosts)] - await db.getConnection() // side effect: ensure connection is established - const listedGdocs = await Gdoc.getListedGdocs() + // Only adding each wpPost if there isn't already a gdoc with the same slug, + // to make sure we use the most up-to-date metadata + for (const wpPost of wpPosts) { + if (!gdocSlugs.has(wpPost.slug)) { + posts.push(wpPost) + } + } - return orderBy( - [...wordpressPostsCards, ...mapGdocsToWordpressPosts(listedGdocs)], - (post) => post.date.getTime(), - ["desc"] - ) + return orderBy(posts, (post) => post.date.getTime(), ["desc"]) }) export const mapGdocsToWordpressPosts = ( diff --git a/explorer/Explorer.tsx b/explorer/Explorer.tsx index 1f54f279abc..82cd2e6cf0a 100644 --- a/explorer/Explorer.tsx +++ b/explorer/Explorer.tsx @@ -38,6 +38,7 @@ import { isInIFrame, keyBy, keyMap, + MarkdownTextWrap, omitUndefinedValues, PromiseCache, PromiseSwitcher, @@ -578,6 +579,7 @@ export class Explorer grapher.reset() this.updateGrapherFromExplorerCommon() grapher.updateFromObject(config) + grapher.forceDisableIntroAnimation = true await grapher.downloadLegacyDataFromOwidVariableIds() let grapherTable = grapher.inputTable @@ -750,12 +752,12 @@ export class Explorer
{this.explorerProgram.explorerTitle}
-
+
+ +
) } diff --git a/package.json b/package.json index 8056390af7a..78d32254b32 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,9 @@ "testJest": "lerna run buildTests && jest" }, "dependencies": { + "@algolia/autocomplete-js": "^1.10.0", + "@algolia/autocomplete-plugin-recent-searches": "^1.11.0", + "@algolia/autocomplete-theme-classic": "^1.11.0", "@aws-sdk/client-s3": "^3.352.0", "@bugsnag/core": "^7.19.0", "@bugsnag/js": "^7.20.0", @@ -105,7 +108,7 @@ "@types/url-parse": "^1.4.8", "@types/webfontloader": "^1.6.34", "@types/workerpool": "^6.1.0", - "algoliasearch": "^4.17.0", + "algoliasearch": "^4.19.1", "antd": "^4.23.1", "archieml": "^0.5.0", "bcrypt": "^5.1.0", @@ -132,7 +135,6 @@ "express": "^4.18.1", "express-async-errors": "^3.1.1", "express-rate-limit": "^5.1.3", - "fast-cartesian": "^5.1.0", "filenamify": "^4.1.0", "fortune": "^5.5.17", "fs-extra": "^11.1.1", @@ -142,6 +144,7 @@ "googleapis": "^108.0.0", "handsontable": "^12.3.3", "html-to-text": "^8.2.0", + "instantsearch.js": "^4.56.9", "js-base64": "^3.7.2", "js-cookie": "^3.0.1", "js-yaml": "^4.1.0", @@ -164,6 +167,7 @@ "nodejs-polars": "^0.7.2", "normalize.css": "^8.0.1", "nx": "15.3.0", + "openai": "^4.6.0", "p-map": "^4.0.0", "papaparse": "^5.3.1", "parsimmon": "^1.18.1", @@ -182,12 +186,13 @@ "react-error-boundary": "^4.0.10", "react-flip-toolkit": "^7.0.9", "react-horizontal-scrolling-menu": "^4.0.3", + "react-instantsearch-hooks-web": "^6.47.2", "react-intersection-observer": "^9.4.0", "react-move": "^6.5.0", "react-recaptcha": "^2.3.10", "react-router-dom": "^5.3.1", "react-select": "^5.7.3", - "react-tag-autocomplete": "^6.3.0", + "react-tag-autocomplete": "^7.1.0", "react-zoom-pan-pinch": "^2.1.3", "reflect-metadata": "^0.1.13", "rxjs": "6", diff --git a/packages/@ourworldindata/core-table/package.json b/packages/@ourworldindata/core-table/package.json index dda2f8bc896..5e726f88b96 100644 --- a/packages/@ourworldindata/core-table/package.json +++ b/packages/@ourworldindata/core-table/package.json @@ -16,8 +16,7 @@ "dependencies": { "@ourworldindata/utils": "workspace:^", "d3": "^6.1.1", - "dayjs": "^1.11.5", - "fast-cartesian": "^5.1.0" + "dayjs": "^1.11.5" }, "devDependencies": { "@types/d3": "^6", diff --git a/packages/@ourworldindata/core-table/src/CoreColumnDef.ts b/packages/@ourworldindata/core-table/src/CoreColumnDef.ts index efdac3ee311..3f48540f74a 100644 --- a/packages/@ourworldindata/core-table/src/CoreColumnDef.ts +++ b/packages/@ourworldindata/core-table/src/CoreColumnDef.ts @@ -64,7 +64,7 @@ export interface CoreColumnDef extends ColumnColorScale { name?: string // The display name for the column titlePublic?: string // The Metadata V2 display title for the variable titleVariant?: string // The Metadata V2 title disambiguation fragment for the variant (e.g. "projected") - producerShort?: string // The Metadata V2 title disambiguation fragment for the producer + attributionShort?: string // The Metadata V2 title disambiguation fragment for the producer description?: string descriptionShort?: string descriptionFromProducer?: string diff --git a/packages/@ourworldindata/core-table/src/CoreTable.test.ts b/packages/@ourworldindata/core-table/src/CoreTable.test.ts index e8093705912..89c099b01e1 100755 --- a/packages/@ourworldindata/core-table/src/CoreTable.test.ts +++ b/packages/@ourworldindata/core-table/src/CoreTable.test.ts @@ -329,7 +329,37 @@ uk,2001` const table = new CoreTable(csv) expect(table.numRows).toEqual(3) const completed = table.complete(["country", "year"]) + expect(completed.numRows).toEqual(6) + expect(completed.rows).toEqual( + expect.arrayContaining([ + // compare in any order + { country: "usa", year: 2000 }, + { country: "usa", year: 2001 }, + { country: "usa", year: 2002 }, + { country: "uk", year: 2000 }, + { country: "uk", year: 2001 }, + { country: "uk", year: 2002 }, + ]) + ) +}) + +it("can sort a table", () => { + const table = new CoreTable(`country,year,population +uk,1800,100 +iceland,1700,200 +iceland,1800,300 +uk,1700,400 +germany,1400,500`) + + const sorted = table.sortBy(["country", "year"]) + expect(sorted.rows).toEqual([ + { country: "germany", year: 1400, population: 500 }, + { country: "iceland", year: 1700, population: 200 }, + { country: "iceland", year: 1800, population: 300 }, + { country: "uk", year: 1700, population: 400 }, + { country: "uk", year: 1800, population: 100 }, + ]) }) describe("adding rows", () => { diff --git a/packages/@ourworldindata/core-table/src/CoreTable.ts b/packages/@ourworldindata/core-table/src/CoreTable.ts index f26fcffc2ed..4646ccb0b61 100644 --- a/packages/@ourworldindata/core-table/src/CoreTable.ts +++ b/packages/@ourworldindata/core-table/src/CoreTable.ts @@ -57,7 +57,6 @@ import { getDropIndexes, parseDelimited, rowsFromMatrix, - cartesianProduct, sortColumnStore, emptyColumnsInFirstRowInDelimited, truncate, @@ -950,7 +949,11 @@ export class CoreTable< appendRows(rows: ROW_TYPE[], opDescription: string): this { return this.concat( - [new (this.constructor as any)(rows, this.defs) as CoreTable], + [ + new (this.constructor as typeof CoreTable)(rows, this.defs, { + parent: this, + }), + ], opDescription ) } @@ -1441,13 +1444,65 @@ export class CoreTable< * ``` * */ - complete(columnSlugs: ColumnSlug[]): this { - const index = this.rowIndex(columnSlugs) - const cols = this.getColumns(columnSlugs) - const product = cartesianProduct(...cols.map((col) => col.uniqValues)) - const toAdd = product.filter((row) => !index.has(row.join(" "))) - return this.appendRows( - rowsFromMatrix([columnSlugs, ...toAdd]), + complete(columnSlugs: [ColumnSlug, ColumnSlug]): this { + if (columnSlugs.length !== 2) + throw new Error("Can only run complete() for exactly 2 columns") + + const [slug1, slug2] = columnSlugs + const col1 = this.get(slug1) + const col2 = this.get(slug2) + + // The output table will have exactly this many rows, since we assume that [col1, col2] are primary keys + // (i.e. there are no two rows with the same key), and every combination that doesn't exist yet we will add. + const cartesianProductSize = col1.numUniqs * col2.numUniqs + if (this.numRows >= cartesianProductSize) { + if (this.numRows > cartesianProductSize) + throw new Error("Table has more rows than expected") + + // Table is already complete + return this + } + + // Map that points from a value in col1 to a set of values in col2. + // It's filled with all the values that already exist in the table, so we + // can later take the difference. + const existingRowValues = new Map>() + for (const index of this.indices) { + const val1 = col1.values[index] + const val2 = col2.values[index] + if (!existingRowValues.has(val1)) + existingRowValues.set(val1, new Set()) + existingRowValues.get(val1)!.add(val2) + } + + // The below code should be as performant as possible, since it's often iterating over hundreds of thousands of rows. + // The below implementation has been benchmarked against a few alternatives (using flatMap, map, and Array.from), and + // is the fastest. + // See https://jsperf.app/zudoye. + const rowsToAddCol1 = [] + const rowsToAddCol2 = [] + // Add rows for all combinations of values that are not contained in `existingRowValues`. + for (const val1 of col1.uniqValuesAsSet) { + const existingVals2 = existingRowValues.get(val1) + for (const val2 of col2.uniqValuesAsSet) { + if (!existingVals2?.has(val2)) { + rowsToAddCol1.push(val1) + rowsToAddCol2.push(val2) + } + } + } + const appendColumnStore: CoreColumnStore = { + [slug1]: rowsToAddCol1, + [slug2]: rowsToAddCol2, + } + const appendTable = new (this.constructor as typeof CoreTable)( + appendColumnStore, + this.defs, + { parent: this } + ) + + return this.concat( + [appendTable], `Append missing combos of ${columnSlugs}` ) } @@ -1562,10 +1617,20 @@ class FilterMask { apply(columnStore: CoreColumnStore): CoreColumnStore { const columnsObject: CoreColumnStore = {} + const keepIndexes: number[] = [] + for (let i = 0; i < this.numRows; i++) { + if (this.mask[i]) keepIndexes.push(i) + } + if (keepIndexes.length === this.numRows) return columnStore + Object.keys(columnStore).forEach((slug) => { - columnsObject[slug] = columnStore[slug].filter( - (slug, index) => this.mask[index] - ) + const originalColumn = columnStore[slug] + const newColumn: CoreValueType[] = new Array(keepIndexes.length) + for (let i = 0; i < keepIndexes.length; i++) { + newColumn[i] = originalColumn[keepIndexes[i]] + } + + columnsObject[slug] = newColumn }) return columnsObject } diff --git a/packages/@ourworldindata/core-table/src/CoreTableColumns.ts b/packages/@ourworldindata/core-table/src/CoreTableColumns.ts index 9d627a66eb8..74d2f38bd61 100644 --- a/packages/@ourworldindata/core-table/src/CoreTableColumns.ts +++ b/packages/@ourworldindata/core-table/src/CoreTableColumns.ts @@ -323,11 +323,17 @@ export abstract class AbstractCoreColumn { } @imemo get uniqValues(): JS_TYPE[] { - return uniq(this.values) + const set = this.uniqValuesAsSet + + // Turn into array, faster than spread operator + const arr = new Array(set.size) + let i = 0 + set.forEach((val) => (arr[i++] = val)) + return arr } @imemo get uniqValuesAsSet(): Set { - return new Set(this.uniqValues) + return new Set(this.values) } /** @@ -394,7 +400,7 @@ export abstract class AbstractCoreColumn { } @imemo get numUniqs(): number { - return this.uniqValues.length + return this.uniqValuesAsSet.size } @imemo get valuesAscending(): JS_TYPE[] { diff --git a/packages/@ourworldindata/core-table/src/CoreTableUtils.test.ts b/packages/@ourworldindata/core-table/src/CoreTableUtils.test.ts index d5639e51e27..0b56666307a 100755 --- a/packages/@ourworldindata/core-table/src/CoreTableUtils.test.ts +++ b/packages/@ourworldindata/core-table/src/CoreTableUtils.test.ts @@ -18,7 +18,6 @@ import { concatColumnStores, guessColumnDefFromSlugAndRow, standardizeSlugs, - cartesianProduct, } from "./CoreTableUtils.js" import { ErrorValueTypes } from "./ErrorValues.js" import { imemo } from "@ourworldindata/utils" @@ -504,21 +503,3 @@ describe(concatColumnStores, () => { }) }) }) - -describe(cartesianProduct, () => { - it("correctly calculates a cartesian product", () => { - const a = [1, 2, 3] - const b = ["a", "b"] - - const product = cartesianProduct(a, b) - - expect(product).toEqual([ - [1, "a"], - [1, "b"], - [2, "a"], - [2, "b"], - [3, "a"], - [3, "b"], - ]) - }) -}) diff --git a/packages/@ourworldindata/core-table/src/CoreTableUtils.ts b/packages/@ourworldindata/core-table/src/CoreTableUtils.ts index c86f3f9ee3b..9bfe779abb1 100644 --- a/packages/@ourworldindata/core-table/src/CoreTableUtils.ts +++ b/packages/@ourworldindata/core-table/src/CoreTableUtils.ts @@ -1,9 +1,7 @@ import { dsvFormat, DSVParsedArray } from "d3-dsv" -import fastCartesian from "fast-cartesian" import { findIndexFast, first, - flatten, max, range, sampleFrom, @@ -371,20 +369,6 @@ export const makeKeyFn = .map((slug) => toString(columnStore[slug][rowIndex])) .join(" ") -export const appendRowsToColumnStore = ( - columnStore: CoreColumnStore, - rows: CoreRow[] -): CoreColumnStore => { - const slugs = Object.keys(columnStore) - const newColumnStore = columnStore - slugs.forEach((slug) => { - newColumnStore[slug] = columnStore[slug].concat( - rows.map((row) => row[slug]) - ) - }) - return newColumnStore -} - const getColumnStoreLength = (store: CoreColumnStore): number => { return max(Object.values(store).map((v) => v.length)) ?? 0 } @@ -399,22 +383,26 @@ export const concatColumnStores = ( const slugs = slugsToKeep ?? Object.keys(first(stores)!) const newColumnStore: CoreColumnStore = {} + + // The below code is performance-critical. + // That's why it's written using for loops and mutable arrays rather than using map or flatMap: + // To this day, that's still faster in JS. slugs.forEach((slug) => { - newColumnStore[slug] = flatten( - stores.map((store, i) => { - const values = store[slug] ?? [] - const toFill = Math.max(0, lengths[i] - values.length) - if (toFill === 0) { - return values - } else { - return values.concat( - new Array(lengths[i] - values.length).fill( - ErrorValueTypes.MissingValuePlaceholder - ) + let newColumnValues: CoreValueType[] = [] + for (const [i, store] of stores.entries()) { + const values = store[slug] ?? [] + const toFill = Math.max(0, lengths[i] - values.length) + + newColumnValues = newColumnValues.concat(values) + if (toFill > 0) { + newColumnValues = newColumnValues.concat( + new Array(toFill).fill( + ErrorValueTypes.MissingValuePlaceholder ) - } - }) - ) + ) + } + } + newColumnStore[slug] = newColumnValues }) return newColumnStore } @@ -625,10 +613,6 @@ export const trimArray = (arr: any[]): any[] => { return arr.slice(0, rightIndex + 1) } -export function cartesianProduct(...allEntries: T[][]): T[][] { - return fastCartesian(allEntries) -} - const applyNewSortOrder = (arr: any[], newOrder: number[]): any[] => newOrder.map((index) => arr[index]) @@ -640,9 +624,22 @@ export const sortColumnStore = ( if (!firstCol) return {} const len = firstCol.length const newOrder = range(0, len).sort(makeSortByFn(columnStore, slugs)) + + // Check if column store is already sorted (which is the case if newOrder is equal to range(0, startLen)). + // If it's not sorted, we will detect that within the first few iterations usually. + let isSorted = true + for (let i = 0; i <= len; i++) { + if (newOrder[i] !== i) { + isSorted = false + break + } + } + // Column store is already sorted; return existing store unchanged + if (isSorted) return columnStore + const newStore: CoreColumnStore = {} - Object.keys(columnStore).forEach((slug) => { - newStore[slug] = applyNewSortOrder(columnStore[slug], newOrder) + Object.entries(columnStore).forEach(([slug, colValues]) => { + newStore[slug] = applyNewSortOrder(colValues, newOrder) }) return newStore } diff --git a/packages/@ourworldindata/core-table/src/OwidTable.ts b/packages/@ourworldindata/core-table/src/OwidTable.ts index c1a0e026a35..86b9c66d1bc 100644 --- a/packages/@ourworldindata/core-table/src/OwidTable.ts +++ b/packages/@ourworldindata/core-table/src/OwidTable.ts @@ -65,7 +65,7 @@ export class OwidTable extends CoreTable { } @imemo get availableEntityNameSet(): Set { - return new Set(this.entityNameColumn.uniqValues) + return this.entityNameColumn.uniqValuesAsSet } // todo: can we remove at some point? diff --git a/packages/@ourworldindata/core-table/src/index.ts b/packages/@ourworldindata/core-table/src/index.ts index 2fc1dbf549b..7d64ade241c 100644 --- a/packages/@ourworldindata/core-table/src/index.ts +++ b/packages/@ourworldindata/core-table/src/index.ts @@ -88,7 +88,6 @@ export { toleranceInterpolation, interpolateRowValuesWithTolerance, makeKeyFn, - appendRowsToColumnStore, concatColumnStores, rowsToColumnStore, autodetectColumnDefs, @@ -107,7 +106,6 @@ export { isCellEmpty, trimEmptyRows, trimArray, - cartesianProduct, sortColumnStore, emptyColumnsInFirstRowInDelimited, } from "./CoreTableUtils.js" diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index bf25ab345d6..acf7f3c99f7 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -396,6 +396,12 @@ export class Grapher @observable sortBy?: SortBy @observable sortOrder?: SortOrder @observable sortColumnSlug?: string + // TODO: this is a crude fix that is used to turn off the intro + // animation in maps (fading colors in from gray) because + // they end up with the wrong target colors (i.e. the colors + // are initially correct but then the animation screws them up). + // This flag can be removed once the animation bug is properly fixed. + @observable forceDisableIntroAnimation: boolean = false @observable.ref _isInFullScreenMode = false @@ -886,7 +892,7 @@ export class Grapher @observable private _baseFontSize = BASE_FONT_SIZE @computed get baseFontSize(): number { - if (this.isExportingtoSvgOrPng) return 18 + if (this.isExportingtoSvgOrPng) return Math.max(this._baseFontSize, 18) return this._baseFontSize } @@ -1460,7 +1466,7 @@ export class Grapher if (uniqueAttributions.length > 3) return "Multiple sources" - return uniqueAttributions.join(", ") + return uniqueAttributions.join("; ") } @computed private get axisDimensions(): ChartDimension[] { @@ -1578,18 +1584,22 @@ export class Grapher return this.dimensions.some((d) => d.property === DimensionProperty.y) } - get staticSVG(): string { + generateStaticSvg(bounds: Bounds = this.idealBounds): string { const _isExportingtoSvgOrPng = this.isExportingtoSvgOrPng this.isExportingtoSvgOrPng = true const staticSvg = ReactDOMServer.renderToStaticMarkup( - + ) this.isExportingtoSvgOrPng = _isExportingtoSvgOrPng return staticSvg } + get staticSVG(): string { + return this.generateStaticSvg() + } + @computed get disableIntroAnimation(): boolean { - return this.isExportingtoSvgOrPng + return this.isExportingtoSvgOrPng || this.forceDisableIntroAnimation } @computed get mapConfig(): MapConfig { diff --git a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts index ec05ab0861a..4848f321ff5 100644 --- a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts +++ b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts @@ -563,7 +563,7 @@ const columnDefFromOwidVariable = ( nonRedistributable, } = variable - const { attribution, titlePublic, titleVariant, producerShort } = + const { attribution, titlePublic, titleVariant, attributionShort } = variable.presentation || {} // Without this the much used var 123 appears as "Countries Continent". We could rename in Grapher but not sure the effects of that. @@ -586,9 +586,7 @@ const columnDefFromOwidVariable = ( nonRedistributable, sourceLink: source?.link ?? - (origins && origins.length > 0 - ? origins[0].datasetUrlMain - : undefined), + (origins && origins.length > 0 ? origins[0].urlMain : undefined), sourceName: source?.name, dataPublishedBy: source?.dataPublishedBy, dataPublisherSource: source?.dataPublisherSource, @@ -599,7 +597,7 @@ const columnDefFromOwidVariable = ( attribution, titlePublic, titleVariant, - producerShort, + attributionShort, owidVariableId: variable.id, type: isContinent ? ColumnTypeNames.Continent diff --git a/packages/@ourworldindata/grapher/src/modal/SourcesModal.tsx b/packages/@ourworldindata/grapher/src/modal/SourcesModal.tsx index cf000c53c16..2d82fc131a8 100644 --- a/packages/@ourworldindata/grapher/src/modal/SourcesModal.tsx +++ b/packages/@ourworldindata/grapher/src/modal/SourcesModal.tsx @@ -1,11 +1,10 @@ import { MarkdownTextWrap, - linkify, - Bounds, - DEFAULT_BOUNDS, OwidOrigin, uniq, excludeNullish, + Bounds, + DEFAULT_BOUNDS, } from "@ourworldindata/utils" import React from "react" import { action, computed } from "mobx" @@ -15,9 +14,6 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" import { CoreColumn, OwidColumnDef } from "@ourworldindata/core-table" import { Modal } from "./Modal" -const formatText = (s: string): string => - linkify(s).replace(/(?:\r\n|\r|\n)/g, "
") - export interface SourcesModalManager { adminBaseUrl?: string columnsWithSources: CoreColumn[] @@ -26,6 +22,25 @@ export interface SourcesModalManager { tabBounds?: Bounds } +// TODO: remove this component once all backported indicators +// etc have switched from HTML to markdown for their sources +const HtmlOrMarkdownText = (props: { + text: string + fontSize: number +}): JSX.Element => { + // check the text for closing a, li or p tags. If + // one is found, render using dangerouslySetInnerHTML, + // othewise use MarkdownTextWrap + const { text, fontSize } = props + const htmlRegex = /<\/(a|li|p)>/ + const match = text.match(htmlRegex) + if (match) { + return + } else { + return + } +} + @observer export class SourcesModal extends React.Component<{ manager: SourcesModalManager @@ -80,12 +95,12 @@ export class SourcesModal extends React.Component<{ ? column.def.origins[0].dateAccessed : undefined) - const citationProducer = + const citationFull = column.def.origins && column.def.origins.length ? excludeNullish( uniq( column.def.origins.map( - (origin: OwidOrigin) => origin.citationProducer + (origin: OwidOrigin) => origin.citationFull ) ) ) @@ -118,11 +133,12 @@ export class SourcesModal extends React.Component<{ // metadata V2 shortDescription Variable description - + + + ) : null} {coverage ? ( @@ -143,62 +159,71 @@ export class SourcesModal extends React.Component<{ {column.unitConversionFactor} ) : null} - {citationProducer.length === 1 ? ( + {citationFull.length === 1 ? ( Data published by - {citationProducer[0]} + + + ) : null} - {citationProducer.length > 1 ? ( + {citationFull.length > 1 ? ( Data published by
    - {citationProducer.map( + {citationFull.map( ( - producer: string, + citation: string, index: number ) => ( -
  • {producer}
    • +
  • + +
    • ) )}
) : null} - {(!citationProducer || citationProducer.length === 0) && + {(!citationFull || citationFull.length === 0) && source.dataPublishedBy ? ( Data published by - + + + ) : null} {source.dataPublisherSource ? ( Data publisher's source - + + + ) : null} {source.link ? ( Link - + + + ) : null} {retrievedDate ? ( @@ -210,12 +235,12 @@ export class SourcesModal extends React.Component<{ {source.additionalInfo && ( -

+

+ +

)} ) diff --git a/packages/@ourworldindata/utils/package.json b/packages/@ourworldindata/utils/package.json index b98fa39d3cf..d89c65a65e4 100644 --- a/packages/@ourworldindata/utils/package.json +++ b/packages/@ourworldindata/utils/package.json @@ -25,7 +25,7 @@ "parsimmon": "^1.18.1", "react": "^16.14.0", "react-select": "^5.7.3", - "react-tag-autocomplete": "^6.3.0", + "react-tag-autocomplete": "^7.1.0", "s-expression": "^3.1.1", "string-pixel-width": "^1.10.0", "striptags": "^3.2.0", diff --git a/packages/@ourworldindata/utils/src/GdocsUtils.ts b/packages/@ourworldindata/utils/src/GdocsUtils.ts index 0848ec4a2ce..5f3e391e2c7 100644 --- a/packages/@ourworldindata/utils/src/GdocsUtils.ts +++ b/packages/@ourworldindata/utils/src/GdocsUtils.ts @@ -1,8 +1,14 @@ import { spansToUnformattedPlainText } from "./Util.js" import { gdocUrlRegex } from "./GdocsConstants.js" -import { OwidGdocLinkJSON, Span } from "./owidTypes.js" +import { EnrichedBlockText, OwidGdocLinkJSON, Span } from "./owidTypes.js" import { Url } from "./urls/Url.js" import urlSlug from "url-slug" +import { + EveryMarkdownNode, + MarkdownRoot, + mdParser, +} from "./MarkdownTextWrap/parser.js" +import { P, match } from "ts-pattern" export function getLinkType(urlString: string): OwidGdocLinkJSON["linkType"] { const url = Url.fromURL(urlString) @@ -40,3 +46,149 @@ export function getUrlTarget(urlString: string): string { export function convertHeadingTextToId(headingText: Span[]): string { return urlSlug(spansToUnformattedPlainText(headingText)) } + +const convertMarkdownNodeToSpan = (node: EveryMarkdownNode): Span[] => { + return match(node) + .with( + { + type: "text", + }, + (n) => [ + { + spanType: "span-simple-text" as const, + text: n.value, + } as Span, + ] + ) + .with( + { + type: "textSegments", + }, + (n) => n.children.flatMap(convertMarkdownNodeToSpan) as Span[] + ) + .with( + { + type: "newline", + }, + () => [ + { + spanType: "span-simple-text" as const, + text: "\n", + } as Span, + ] + ) + .with( + { + type: "whitespace", + }, + () => [ + { + spanType: "span-simple-text" as const, + text: " ", + } as Span, + ] + ) + .with( + { + type: "detailOnDemand", + }, + (n) => [ + { + spanType: "span-dod" as const, + id: n.term, + children: n.children.flatMap(convertMarkdownNodeToSpan), + } as Span, + ] + ) + .with( + { + type: "markdownLink", + }, + (n) => [ + { + spanType: "span-link" as const, + url: n.href, + children: n.children.flatMap(convertMarkdownNodeToSpan), + } as Span, + ] + ) + .with( + { + type: "plainUrl", + }, + (n) => [ + { + spanType: "span-link" as const, + url: n.href, + children: [ + { + spanType: "span-simple-text" as const, + text: n.href, + }, + ], + } as Span, + ] + ) + .with( + { + type: "bold", + }, + (n) => [ + { + spanType: "span-bold" as const, + children: n.children.flatMap(convertMarkdownNodeToSpan), + } as Span, + ] + ) + .with( + { + type: P.union("italic", "plainItalic", "italicWithoutBold"), + }, + (n) => [ + { + spanType: "span-italic" as const, + children: n.children.flatMap(convertMarkdownNodeToSpan), + } as Span, + ] + ) + .with( + { + type: P.union("bold", "plainBold", "boldWithoutItalic"), + }, + (n) => [ + { + spanType: "span-bold" as const, + children: n.children.flatMap(convertMarkdownNodeToSpan), + } as Span, + ] + ) + .exhaustive() + //.otherwise(() => ({ spanType: "span-simple-text" as const, text: "" })) +} + +const convertMarkdownNodesToSpans = (nodes: MarkdownRoot): Span[] => + nodes.children.flatMap(convertMarkdownNodeToSpan) + +export const markdownToEnrichedTextBlock = ( + markdown: string +): EnrichedBlockText => { + const parsedMarkdown = mdParser.markdown.parse(markdown) + if (parsedMarkdown.status) { + const spans = convertMarkdownNodesToSpans(parsedMarkdown.value) + return { + type: "text", + value: spans, + parseErrors: [], + } + } else + return { + type: "text", + value: [], + parseErrors: [ + { + message: `Failed to parse markdown - expected ${parsedMarkdown.expected} at ${parsedMarkdown.index}`, + isWarning: false, + }, + ], + } +} diff --git a/packages/@ourworldindata/utils/src/OwidOrigin.ts b/packages/@ourworldindata/utils/src/OwidOrigin.ts index 45bcecad7c7..28349ee48c1 100644 --- a/packages/@ourworldindata/utils/src/OwidOrigin.ts +++ b/packages/@ourworldindata/utils/src/OwidOrigin.ts @@ -1,18 +1,18 @@ import { OwidLicense } from "./OwidVariable.js" export interface OwidOrigin { id?: number - datasetTitleProducer?: string - datasetTitleOwid?: string + title?: string + titleSnapshot?: string attribution?: string attributionShort?: string versionProducer?: string license?: OwidLicense - datasetDescriptionOwid?: string - datasetDescriptionProducer?: string + descriptionSnapshot?: string + description?: string producer?: string - citationProducer?: string - datasetUrlMain?: string - datasetUrlDownload?: string + citationFull?: string + urlMain?: string + urlDownload?: string dateAccessed?: Date datePublished?: string } diff --git a/packages/@ourworldindata/utils/src/OwidVariable.ts b/packages/@ourworldindata/utils/src/OwidVariable.ts index 58a7d6ea1a9..5d6974ee85e 100644 --- a/packages/@ourworldindata/utils/src/OwidVariable.ts +++ b/packages/@ourworldindata/utils/src/OwidVariable.ts @@ -94,7 +94,7 @@ export interface OwidLicense { export interface OwidVariablePresentation { titlePublic?: string titleVariant?: string - producerShort?: string + attributionShort?: string attribution?: string topicTagsLinks?: string[] faqs?: FaqLink[] diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 981f8b7483a..a2e4a382346 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -161,6 +161,7 @@ import { EnrichedScrollerItem, EnrichedBlockKeyInsightsSlide, UserCountryInformation, + SpanLink, } from "./owidTypes.js" import { OwidVariableWithSource } from "./OwidVariable.js" import { PointVector } from "./PointVector.js" @@ -613,7 +614,8 @@ export const getIdealGridParams = ({ // Also Desmos graph: https://www.desmos.com/calculator/tmajzuq5tm const ratio = containerAspectRatio / idealAspectRatio // Prefer vertical grid for count=2. - if (count === 2 && ratio < 2) return { rows: 2, columns: 1 } + if (count === 2 && containerAspectRatio < 2.8) + return { rows: 2, columns: 1 } // Otherwise, optimize for closest to the ideal aspect ratio. const initialColumns = Math.min(Math.round(Math.sqrt(count * ratio)), count) const rows = Math.ceil(count / initialColumns) @@ -1620,7 +1622,8 @@ export function traverseEnrichedBlocks( "sdg-grid", "sdg-toc", "topic-page-intro", - "all-charts" + "all-charts", + "entry-summary" ), }, callback @@ -1632,6 +1635,10 @@ export function checkNodeIsSpan(node: NodeWithUrl): node is Span { return "spanType" in node } +export function checkNodeIsSpanLink(node: unknown): node is SpanLink { + return isObject(node) && "spanType" in node && node.spanType === "span-link" +} + export function spansToUnformattedPlainText(spans: Span[]): string { return spans .map((span) => @@ -1747,5 +1754,20 @@ export function getAttributionFromVariable( variable.origins ) const sourceName = variable.source?.name - return uniq(compact([sourceName, ...originAttributionFragments])).join(", ") + return uniq(compact([sourceName, ...originAttributionFragments])).join("; ") +} + +interface ETLPathComponents { + channel: string + producer: string + version: string + dataset: string + table: string + indicator: string +} + +export const getETLPathComponents = (path: string): ETLPathComponents => { + const [channel, producer, version, dataset, table, indicator] = + path.split("/") + return { channel, producer, version, dataset, table, indicator } } diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts index 71d7dc2a93b..e593d408ad4 100644 --- a/packages/@ourworldindata/utils/src/index.ts +++ b/packages/@ourworldindata/utils/src/index.ts @@ -73,6 +73,8 @@ export { type EnrichedRecircLink, type EnrichedScrollerItem, type EnrichedSDGGridItem, + type EnrichedBlockEntrySummary, + type EnrichedBlockEntrySummaryItem, type EntryMeta, type EntryNode, EPOCH_DATE, @@ -153,6 +155,8 @@ export { type RawRecircLink, type RawDetail, type RawSDGGridItem, + type RawBlockEntrySummary, + type RawBlockEntrySummaryItem, type RelatedChart, type Ref, type RefDictionary, @@ -320,6 +324,7 @@ export { recursivelyMapArticleContent, traverseEnrichedBlocks, checkNodeIsSpan, + checkNodeIsSpanLink, spansToUnformattedPlainText, findDuplicates, checkIsOwidGdocType, @@ -333,6 +338,7 @@ export { getOriginAttributionFragments, getAttributionFromVariable, copyToClipboard, + getETLPathComponents, } from "./Util.js" export { @@ -608,6 +614,7 @@ export { getUrlTarget, checkIsInternalLink, convertHeadingTextToId, + markdownToEnrichedTextBlock, } from "./GdocsUtils.js" export { diff --git a/packages/@ourworldindata/utils/src/owidTypes.ts b/packages/@ourworldindata/utils/src/owidTypes.ts index 78f6ba45599..a94cab6a328 100644 --- a/packages/@ourworldindata/utils/src/owidTypes.ts +++ b/packages/@ourworldindata/utils/src/owidTypes.ts @@ -1,4 +1,3 @@ -import { Tag as TagReactTagAutocomplete } from "react-tag-autocomplete" import { ImageMetadata } from "./image.js" import { Static, Type } from "@sinclair/typebox" import { gdocUrlRegex } from "./GdocsConstants.js" @@ -195,8 +194,11 @@ export enum KeyChartLevel { Top = 3, // chart will show at the top of the all charts block } -export interface Tag extends TagReactTagAutocomplete { +export interface Tag { + id: number + name: string keyChartLevel?: KeyChartLevel + isApproved?: boolean } export interface EntryMeta { @@ -963,8 +965,13 @@ export type RawBlockResearchAndWritingRow = { export type RawBlockResearchAndWriting = { type: "research-and-writing" value: { - primary?: RawBlockResearchAndWritingLink - secondary?: RawBlockResearchAndWritingLink + // We're migrating these to be arrays, but have to support the old use-case until it's done + primary?: + | RawBlockResearchAndWritingLink + | RawBlockResearchAndWritingLink[] + secondary?: + | RawBlockResearchAndWritingLink + | RawBlockResearchAndWritingLink[] more?: RawBlockResearchAndWritingRow rows?: RawBlockResearchAndWritingRow[] } @@ -987,9 +994,9 @@ export type EnrichedBlockResearchAndWritingRow = { export type EnrichedBlockResearchAndWriting = { type: "research-and-writing" - primary: EnrichedBlockResearchAndWritingLink - secondary: EnrichedBlockResearchAndWritingLink - more: EnrichedBlockResearchAndWritingRow + primary: EnrichedBlockResearchAndWritingLink[] + secondary: EnrichedBlockResearchAndWritingLink[] + more?: EnrichedBlockResearchAndWritingRow rows: EnrichedBlockResearchAndWritingRow[] } & EnrichedBlockWithParseErrors @@ -1049,6 +1056,32 @@ export type EnrichedBlockAlign = { content: OwidEnrichedGdocBlock[] } & EnrichedBlockWithParseErrors +export type RawBlockEntrySummaryItem = { + text?: string + slug?: string +} + +// This block renders via the TableOfContents component, same as the sdg-toc block. +// Because the summary headings can differ from the actual headings in the document, +// we need to serialize the text and slug explicitly, instead of programmatically generating them +// by analyzing the document (like we do for the sdg-toc block) +export type RawBlockEntrySummary = { + type: "entry-summary" + value: { + items?: RawBlockEntrySummaryItem[] + } +} + +export type EnrichedBlockEntrySummaryItem = { + text: string + slug: string +} + +export type EnrichedBlockEntrySummary = { + type: "entry-summary" + items: EnrichedBlockEntrySummaryItem[] +} & EnrichedBlockWithParseErrors + export type Ref = { id: string // Can be -1 @@ -1093,6 +1126,7 @@ export type OwidRawGdocBlock = | RawBlockTopicPageIntro | RawBlockKeyInsights | RawBlockAlign + | RawBlockEntrySummary export type OwidEnrichedGdocBlock = | EnrichedBlockAllCharts @@ -1126,6 +1160,7 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockKeyInsights | EnrichedBlockResearchAndWriting | EnrichedBlockAlign + | EnrichedBlockEntrySummary export enum OwidGdocPublicationContext { unlisted = "unlisted", @@ -1385,7 +1420,7 @@ export interface DataPageDataV2 { status: "published" | "draft" title: string titleVariant?: string - producerShort?: string + attributionShort?: string topicTagsLinks?: string[] attribution: string descriptionShort?: string diff --git a/site/DataPageV2Content.tsx b/site/DataPageV2Content.tsx index a3fdd92879a..982e4b0cf96 100644 --- a/site/DataPageV2Content.tsx +++ b/site/DataPageV2Content.tsx @@ -10,13 +10,9 @@ import { ArticleBlocks } from "./gdocs/ArticleBlocks.js" import { RelatedCharts } from "./blocks/RelatedCharts.js" import { DataPageV2ContentFields, - mdParser, - MarkdownRoot, - EveryMarkdownNode, - Span, - EnrichedBlockText, excludeNullish, slugify, + markdownToEnrichedTextBlock, } from "@ourworldindata/utils" import { AttachmentsContext, DocumentContext } from "./gdocs/OwidGdoc.js" import StickyNav from "./blocks/StickyNav.js" @@ -24,158 +20,12 @@ import cx from "classnames" import { DebugProvider } from "./gdocs/DebugContext.js" import { CodeSnippet } from "./blocks/CodeSnippet.js" import dayjs from "dayjs" -import { P, match } from "ts-pattern" declare global { interface Window { _OWID_DATAPAGEV2_PROPS: DataPageV2ContentFields _OWID_GRAPHER_CONFIG: GrapherInterface } } - -const convertMarkdownNodeToSpan = (node: EveryMarkdownNode): Span[] => { - return match(node) - .with( - { - type: "text", - }, - (n) => [ - { - spanType: "span-simple-text" as const, - text: n.value, - } as Span, - ] - ) - .with( - { - type: "textSegments", - }, - (n) => n.children.flatMap(convertMarkdownNodeToSpan) as Span[] - ) - .with( - { - type: "newline", - }, - () => [ - { - spanType: "span-simple-text" as const, - text: "\n", - } as Span, - ] - ) - .with( - { - type: "whitespace", - }, - () => [ - { - spanType: "span-simple-text" as const, - text: " ", - } as Span, - ] - ) - .with( - { - type: "detailOnDemand", - }, - (n) => [ - { - spanType: "span-dod" as const, - id: n.term, - children: n.children.flatMap(convertMarkdownNodeToSpan), - } as Span, - ] - ) - .with( - { - type: "markdownLink", - }, - (n) => [ - { - spanType: "span-link" as const, - url: n.href, - children: n.children.flatMap(convertMarkdownNodeToSpan), - } as Span, - ] - ) - .with( - { - type: "plainUrl", - }, - (n) => [ - { - spanType: "span-link" as const, - url: n.href, - children: [ - { - spanType: "span-simple-text" as const, - text: n.href, - }, - ], - } as Span, - ] - ) - .with( - { - type: "bold", - }, - (n) => [ - { - spanType: "span-bold" as const, - children: n.children.flatMap(convertMarkdownNodeToSpan), - } as Span, - ] - ) - .with( - { - type: P.union("italic", "plainItalic", "italicWithoutBold"), - }, - (n) => [ - { - spanType: "span-italic" as const, - children: n.children.flatMap(convertMarkdownNodeToSpan), - } as Span, - ] - ) - .with( - { - type: P.union("bold", "plainBold", "boldWithoutItalic"), - }, - (n) => [ - { - spanType: "span-bold" as const, - children: n.children.flatMap(convertMarkdownNodeToSpan), - } as Span, - ] - ) - .exhaustive() - //.otherwise(() => ({ spanType: "span-simple-text" as const, text: "" })) -} - -const convertMarkdownNodesToSpans = (nodes: MarkdownRoot) => - nodes.children.flatMap(convertMarkdownNodeToSpan) - -const markdownToEnrichedTextBlock = (markdown: string): EnrichedBlockText => { - const parsedMarkdown = mdParser.markdown.parse(markdown) - if (parsedMarkdown.status) { - const spans = convertMarkdownNodesToSpans(parsedMarkdown.value) - return { - type: "text", - value: spans, - parseErrors: [], - } - } else - return { - type: "text", - value: [], - parseErrors: [ - { - message: `Failed to parse markdown - expected ${parsedMarkdown.expected} at ${parsedMarkdown.index}`, - isWarning: false, - }, - ], - } -} - export const OWID_DATAPAGE_CONTENT_ROOT_ID = "owid-datapageJson-root" export const DataPageV2Content = ({ @@ -189,9 +39,9 @@ export const DataPageV2Content = ({ const [grapher, setGrapher] = React.useState(undefined) const sourceShortName = - datapageData.producerShort && datapageData.titleVariant - ? `${datapageData.producerShort} - ${datapageData.titleVariant}` - : datapageData.producerShort || datapageData.titleVariant + datapageData.attributionShort && datapageData.titleVariant + ? `${datapageData.attributionShort} - ${datapageData.titleVariant}` + : datapageData.attributionShort || datapageData.titleVariant // Initialize the grapher for client-side rendering const mergedGrapherConfig = grapherConfig @@ -244,15 +94,12 @@ export const DataPageV2Content = ({ ]).year() const citationShort = `${producers} — ${processedAdapted} OWID (${yearOfUpdate})` const originsLong = datapageData.origins - .map( - (o) => - `${o.producer}, ${o.datasetTitleProducer ?? o.datasetTitleOwid}` - ) + .map((o) => `${o.producer}, ${o.title ?? o.titleSnapshot}`) .join("; ") const dateAccessed = datapageData.origins[0].dateAccessed ? dayjs(datapageData.origins[0].dateAccessed).format("MMMM D, YYYY") : "" - const urlAccessed = datapageData.origins[0].datasetUrlDownload + const urlAccessed = datapageData.origins[0].urlDownload const citationLong = `${citationShort}. ${datapageData.title}. ${originsLong}, ${processedAdapted} by Our World In Data. Retrieved ${dateAccessed} from ${urlAccessed}` const processedAdaptedText = datapageData.owidProcessingLevel === "minor" @@ -370,7 +217,7 @@ export const DataPageV2Content = ({ )} {datapageData.descriptionFromProducer && ( - {source.datasetDescriptionProducer && ( + {source.description && ( )} {(source.dateAccessed || - source.datasetUrlDownload) && ( + source.urlDownload) && (
)} - {source.datasetUrlDownload && ( + {source.urlDownload && (
Retrieved @@ -672,19 +519,19 @@ export const DataPageV2Content = ({
)} - {source.citationProducer && ( + {source.citationFull && (
{ } return ( // They key= in here makes it so that the chart is re-loaded when the slug changes. - // This is especially important for SearchResults, where the preview chart can change as - // the search query changes.
{this.bounds && ( li { list-style-type: none; + white-space: nowrap; > a, .SiteNavigationToggle__button { @include body-3-medium; @@ -100,6 +101,7 @@ .site-search-cta { display: flex; flex-direction: row; + flex-grow: 1; align-items: center; justify-content: end; height: $search-cta-height; @@ -178,6 +180,9 @@ } &.search-active { + .site-primary-links { + display: none; + } @include sm-only { .site-logos { display: none; @@ -187,35 +192,6 @@ .site-search-cta { flex: 1 1 100%; } - .site-primary-links { - display: none; - } - } - } - - .SearchResults { - position: absolute; - top: calc(100% + $header-border-height); - height: calc(100vh - $header-height-sm); - left: 0; - margin: 0 -16px; - padding: 0 16px 24px; - max-width: 100vw; // force chart to resize and not overflow when rotating - /* Needs to go over the charts */ - z-index: $zindex-search-overlay; - overflow-y: scroll; - overscroll-behavior: contain; - background: white; - @include sm-up { - margin: 0 #{-$padding-x-sm}; - padding-left: $padding-x-sm; - padding-right: $padding-x-sm; - } - @include md-up { - height: calc(100vh - $header-height-md); - width: calc(100% + #{2 * $padding-x-md}); - margin: 0 #{-$padding-x-md}; - padding: 0 $padding-x-md 40px; } } } diff --git a/site/SiteNavigation.tsx b/site/SiteNavigation.tsx index 0e19c97dca3..577efd080c6 100644 --- a/site/SiteNavigation.tsx +++ b/site/SiteNavigation.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react" +import React, { useCallback, useEffect, useState } from "react" import ReactDOM from "react-dom" import { faListUl, @@ -21,6 +21,7 @@ import { SiteNavigationToggle } from "./SiteNavigationToggle.js" import classnames from "classnames" import { useTriggerOnEscape } from "./hooks.js" import { BAKED_BASE_URL } from "../settings/clientSettings.js" +import { AUTOCOMPLETE_CONTAINER_ID } from "./search/Autocomplete.js" export enum Menu { Topics = "topics", @@ -41,10 +42,38 @@ export const SiteNavigation = ({ baseUrl }: { baseUrl: string }) => { menu !== null && [Menu.Topics, Menu.Resources, Menu.About].includes(menu) - const closeOverlay = () => { + // useCallback so as to not trigger a re-render for SiteSearchNavigation, which remounts + // Autocomplete and breaks it + const closeOverlay = useCallback(() => { setActiveMenu(null) setQuery("") - } + }, []) + + // Same SiteSearchNavigation re-rendering case as above + const setSearchAsActiveMenu = useCallback(() => { + setActiveMenu(Menu.Search) + // Forced DOM manipulation of the algolia autocomplete panel position 🙃 + // Without this, the panel initially renders at the same width as the shrunk search input + // Fortunately we only have to do this when it mounts - it takes care of resizes + setTimeout(() => { + const [panel, autocompleteContainer] = [ + ".aa-Panel", + AUTOCOMPLETE_CONTAINER_ID, + ].map((className) => document.querySelector(className)) + if (panel && autocompleteContainer) { + const bounds = autocompleteContainer.getBoundingClientRect() + panel.style.left = `${bounds.left}px` + } + }, 10) + + setTimeout(() => { + const input = document.querySelector(".aa-Input") + if (input) { + input.focus() + input.setAttribute("required", "true") + } + }, 10) + }, []) const toggleMenu = (root: Menu) => { if (menu === root) { @@ -150,11 +179,9 @@ export const SiteNavigation = ({ baseUrl }: { baseUrl: string }) => {
setActiveMenu(Menu.Search)} + onActivate={setSearchAsActiveMenu} /> a { display: block; diff --git a/site/SiteSearchNavigation.scss b/site/SiteSearchNavigation.scss index 96e561d36aa..ad0c811acc9 100644 --- a/site/SiteSearchNavigation.scss +++ b/site/SiteSearchNavigation.scss @@ -1,8 +1,8 @@ -.SiteSearchNavigation { - position: relative; - display: none; +.site-navigation-bar.search-active .SiteSearchNavigation { width: 100%; +} +.SiteSearchNavigation { @include lg-up { display: block; width: 300px; @@ -12,31 +12,6 @@ display: block; } - > input { - height: $search-cta-height; - width: inherit; - padding: 0 40px 0 16px; - background-color: $blue-90; - border: 1px solid $blue-90; - color: $white; - outline: none; - - &:hover { - border: 1px solid $blue-60; - } - - &:focus, - &.active { - border: 1px solid $blue-40; - } - - &::placeholder { - @include body-3-medium; - color: $blue-30; - transition: color 150ms ease; - } - } - > .icon { position: absolute; top: 0; @@ -59,21 +34,3 @@ } } } - -.SiteSearchNavigation__mobile-toggle { - display: flex; - align-items: center; - height: $search-cta-height; - margin-right: -8px; - background: none; - border: none; - color: $white; - cursor: pointer; -} - -.SiteSearchNavigation__mobile-toggle { - @include mobile-toggle-icon; - &:hover { - color: $blue-40; - } -} diff --git a/site/SiteSearchNavigation.tsx b/site/SiteSearchNavigation.tsx index a6e6a0763fd..165b0fa93c1 100644 --- a/site/SiteSearchNavigation.tsx +++ b/site/SiteSearchNavigation.tsx @@ -1,87 +1,19 @@ -import React, { useEffect } from "react" -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" -import { faSearch, faXmark } from "@fortawesome/free-solid-svg-icons" -import classnames from "classnames" -import { siteSearch } from "./search/searchClient.js" -import { SearchResults } from "./search/SearchResults.js" -import { SiteSearchResults } from "./search/searchTypes.js" +import React from "react" +import cx from "classnames" +import { Autocomplete } from "./search/Autocomplete.js" export const SiteSearchNavigation = ({ - query, - setQuery, isActive, onClose, onActivate, }: { - query: string - setQuery: (query: string) => void isActive: boolean onClose: VoidFunction onActivate: VoidFunction }) => { - const [results, setResults] = React.useState(null) - const inputRef = React.useRef(null) - - // Run search - React.useEffect(() => { - const runSearch = async () => { - if (query) { - setResults(await siteSearch(query)) - } else { - setResults(null) - } - } - runSearch() - }, [query]) - - // Focus input when active (needs to happen after render, hence useEffect) - useEffect(() => { - if (isActive && inputRef.current) { - inputRef.current.focus() - } - }, [isActive]) - return ( - <> -
- setQuery(e.currentTarget.value)} - onFocus={onActivate} - className={classnames({ active: isActive })} - value={query} - ref={inputRef} - /> -
- {isActive ? ( - - ) : ( - - )} -
-
- {!isActive && ( - - )} - {isActive && results && } - +
+ +
) } diff --git a/site/blocks/related-charts.scss b/site/blocks/related-charts.scss index 2627d6a10d8..3ef53ee84c8 100644 --- a/site/blocks/related-charts.scss +++ b/site/blocks/related-charts.scss @@ -60,7 +60,7 @@ grid-template-columns: repeat(2, 1fr); gap: $vertical-spacing; - // Hack to work raround the top border shadow being cropped for the first + // Hack to work around the top border shadow being cropped for the first // two thumbnails. See also unsatisfactory solution above. li { &:nth-child(1), diff --git a/site/blocks/research-and-writing.scss b/site/blocks/research-and-writing.scss index ab9e68a3b61..851e285ce08 100644 --- a/site/blocks/research-and-writing.scss +++ b/site/blocks/research-and-writing.scss @@ -56,19 +56,10 @@ margin-top: 16px; } - .title { - @include h2-bold; - color: $blue-90; - } - .description { line-height: 1.55; } - a { - text-decoration: none; - } - a:hover .title { text-decoration: underline; } @@ -88,10 +79,14 @@ .wp-block-owid-card { .title { - font-size: 18px; + @include owid-link-90; + @include h2-bold; + margin-top: 16px; + margin-bottom: 8px; + text-decoration: none; } .description { - font-size: 14px; + @include body-1-regular; } } @@ -172,6 +167,8 @@ .title { @include h3-bold; + margin-top: 16px; + margin-bottom: 8px; } img { border-color: #fff; @@ -186,15 +183,6 @@ } .research-and-writing__top { - .wp-block-owid-card { - .title { - font-size: 24px; - } - .description { - font-size: 18px; - } - } - > .wp-block-owid-card { grid-column: 1 / 13; } diff --git a/site/css/content.scss b/site/css/content.scss index 4f80f0bd764..8ddb6784bd4 100644 --- a/site/css/content.scss +++ b/site/css/content.scss @@ -30,8 +30,7 @@ @include figure-margin; } -.article-content figure[data-grapher-src], -.SearchResults figure[data-grapher-src] { +.article-content figure[data-grapher-src] { @include figure-grapher-reset; > a { @@ -89,13 +88,11 @@ } } -.article-content figure[data-grapher-src].grapherPreview, -.SearchResults figure[data-grapher-src].grapherPreview { +.article-content figure[data-grapher-src].grapherPreview { padding: 1em 0; } -.article-content figure[data-grapher-src]:not(.grapherPreview), -.SearchResults figure[data-grapher-src]:not(.grapherPreview) { +.article-content figure[data-grapher-src]:not(.grapherPreview) { height: $grapher-height; } diff --git a/site/gdocs/ArticleBlock.tsx b/site/gdocs/ArticleBlock.tsx index 9f9f282cf85..1c658427d8d 100644 --- a/site/gdocs/ArticleBlock.tsx +++ b/site/gdocs/ArticleBlock.tsx @@ -22,7 +22,7 @@ import { BlockErrorBoundary, BlockErrorFallback } from "./BlockErrorBoundary.js" import { match } from "ts-pattern" import { renderSpans } from "./utils.js" import Paragraph from "./Paragraph.js" -import SDGTableOfContents from "./SDGTableOfContents.js" +import TableOfContents from "./TableOfContents.js" import urlSlug from "url-slug" import { MissingData } from "./MissingData.js" import { AdditionalCharts } from "./AdditionalCharts.js" @@ -76,13 +76,13 @@ const layouts: { [key in Container]: Layouts} = { ["research-and-writing"]: "col-start-2 span-cols-12", ["scroller"]: "grid span-cols-12 col-start-2", ["sdg-grid"]: "grid col-start-2 span-cols-12 col-lg-start-3 span-lg-cols-10 span-sm-cols-12 col-sm-start-2", - ["sdg-toc"]: "grid grid-cols-8 col-start-4 span-cols-8 grid-md-cols-10 col-md-start-3 span-md-cols-10 grid-sm-cols-12 span-sm-cols-12 col-sm-start-2", + ["toc"]: "grid grid-cols-8 col-start-4 span-cols-8 grid-md-cols-10 col-md-start-3 span-md-cols-10 grid-sm-cols-12 span-sm-cols-12 col-sm-start-2", ["side-by-side"]: "grid span-cols-12 col-start-2", - ["sticky-left-left-column"]: "grid grid-cols-7 span-cols-7 span-md-cols-12 grid-md-cols-12", - ["sticky-left-right-column"]: "grid grid-cols-5 span-cols-5 span-md-cols-12 grid-md-cols-12", + ["sticky-left-left-column"]: "grid grid-cols-7 span-cols-7 span-md-cols-10 grid-md-cols-10", + ["sticky-left-right-column"]: "grid grid-cols-5 span-cols-5 span-md-cols-10 grid-md-cols-10", ["sticky-left"]: "grid span-cols-12 col-start-2", - ["sticky-right-left-column"]: "grid span-cols-5 grid grid-cols-5 span-md-cols-12 grid-md-cols-12", - ["sticky-right-right-column"]: "span-cols-7 span-md-cols-12", + ["sticky-right-left-column"]: "grid span-cols-5 grid grid-cols-5 span-md-cols-10 grid-md-cols-10 col-md-start-2 span-sm-cols-12 grid-sm-cols-12 col-sm-start-1", + ["sticky-right-right-column"]: "span-cols-7 grid-cols-7 span-md-cols-10 grid-md-cols-10 col-md-start-2 span-sm-cols-12 grid-sm-cols-12 col-sm-start-1", ["sticky-right"]: "grid span-cols-12 col-start-2", ["text"]: "col-start-5 span-cols-6 col-md-start-3 span-md-cols-10 span-sm-cols-12 col-sm-start-2", ["topic-page-intro"]: "grid col-start-2 span-cols-12", @@ -495,12 +495,26 @@ export default function ArticleBlock({ )) .with({ type: "sdg-toc" }, () => { return toc ? ( - ) : null }) + .with({ type: "entry-summary" }, (block) => { + return ( + ({ + ...item, + title: item.text, + isSubheading: false, + }))} + className={getLayout("toc", containerType)} + /> + ) + }) .with({ type: "missing-data" }, () => ( )) diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index 4e42d9f5b78..c7945c6446e 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -15,7 +15,10 @@ import { OwidGdocType, } from "@ourworldindata/utils" import { CodeSnippet } from "../blocks/CodeSnippet.js" -import { BAKED_BASE_URL } from "../../settings/clientSettings.js" +import { + ADMIN_BASE_URL, + BAKED_BASE_URL, +} from "../../settings/clientSettings.js" import { formatAuthors } from "../clientFormatting.js" import { DebugProvider } from "./DebugContext.js" import { OwidGdocHeader } from "./OwidGdocHeader.js" @@ -52,6 +55,7 @@ const citationDescriptionsByArticleType: Record = { } export function OwidGdoc({ + id, content, publishedAt, slug, @@ -92,6 +96,23 @@ export function OwidGdoc({ }} > +
) : null} {!!breadcrumbs?.length && ( -
+
-
+
{content.supertitle ? (

{content.supertitle} @@ -70,11 +72,11 @@ function OwidArticleHeader({

{content.subtitle ? ( -

+

{content.subtitle}

) : null} -
+
{"By: "} diff --git a/site/gdocs/ResearchAndWriting.scss b/site/gdocs/ResearchAndWriting.scss index 97409c4b81c..b037fa47bd4 100644 --- a/site/gdocs/ResearchAndWriting.scss +++ b/site/gdocs/ResearchAndWriting.scss @@ -2,7 +2,6 @@ margin-bottom: 40px; > h1 { - text-align: center; font-size: 32px; margin-bottom: 40px; @@ -16,55 +15,26 @@ } } -.research-and-writing-more { - padding: 24px; - background: $gray-10; - h5 { - margin: 0; - color: $blue-50; - } +.research-and-writing-row h2 { + margin-top: 32px; + margin-bottom: 24px; + color: $blue-50; +} - @include md-down { - margin-top: 24px; +.research-and-writing-row__link-container { + @include sm-only { + display: flex; + overflow-x: auto; + scrollbar-width: thin; } - .research-and-writing-link { - display: block; h3 { @include h3-bold; margin-top: 16px; margin-bottom: 8px; } - - p { - margin-bottom: 8px; - } - - &:not(:last-child) { - border-bottom: 1px solid $blue-10; - } - - &:last-child p { - margin-bottom: 0; - } - } -} - -.research-and-writing-row { - h5 { - text-align: center; - margin-top: 40px; - margin-bottom: 24px; - } - .research-and-writing-row__link-container { - overflow-x: auto; - scrollbar-width: thin; - .research-and-writing-link { - // distributes 1280px between 4 tiles + 3 column gaps - min-width: calc((1280px - calc(3 * var(--grid-gap))) / 4); - h3 { - font-size: 1.125rem; - } + @include sm-only { + flex: 1 0 80%; } } } diff --git a/site/gdocs/ResearchAndWriting.tsx b/site/gdocs/ResearchAndWriting.tsx index 2f9dbbbc9d0..1f740bab7f9 100644 --- a/site/gdocs/ResearchAndWriting.tsx +++ b/site/gdocs/ResearchAndWriting.tsx @@ -93,33 +93,28 @@ export function ResearchAndWriting(props: ResearchAndWritingProps) { Research & Writing - - -
-
-
{more.heading}
- {more.articles.map((link, i) => ( +
+
+ {primary.map((link, i) => ( ))} + {secondary.map((link, i) => ( + + ))}
{rows.map((row, i) => (
-
{row.heading}
-
- {/* center the two thumbnails with a filler element */} - {row.articles.length === 2 ?
: null} +

{row.heading}

+
{row.articles.map((link, i) => (
))} + {more ? ( +
+

{more.heading}

+
+ {more.articles.map((link, i) => ( + + ))} +
+
+ ) : null}
) } diff --git a/site/gdocs/SDGTableOfContents.scss b/site/gdocs/TableOfContents.scss similarity index 95% rename from site/gdocs/SDGTableOfContents.scss rename to site/gdocs/TableOfContents.scss index 6d484b5b5be..9fa51c2c8e8 100644 --- a/site/gdocs/SDGTableOfContents.scss +++ b/site/gdocs/TableOfContents.scss @@ -1,4 +1,4 @@ -.sdg-toc { +.toc { padding: 40px 0; margin: 32px 0; background-color: $beige; @@ -12,7 +12,7 @@ margin-right: var(--grid-gap); } - .sdg-toc-toggle { + .toc-toggle { @include h2-bold; margin: 0; padding: 0; @@ -25,8 +25,8 @@ } // Have to hard-code this because the span-cols-x overrides col-start-x - .sdg-toc-toggle, - .sdg-toc-content { + .toc-toggle, + .toc-content { grid-column-start: 2; } diff --git a/site/gdocs/SDGTableOfContents.tsx b/site/gdocs/TableOfContents.tsx similarity index 78% rename from site/gdocs/SDGTableOfContents.tsx rename to site/gdocs/TableOfContents.tsx index fbaa5e726e0..addba3d5e46 100644 --- a/site/gdocs/SDGTableOfContents.tsx +++ b/site/gdocs/TableOfContents.tsx @@ -7,12 +7,14 @@ import AnimateHeight from "react-animate-height" // See ARIA roles: https://w3c.github.io/aria-practices/examples/menu-button/menu-button-links.html -export default function SDGTableOfContents({ +export default function TableOfContents({ toc, className = "", + title, }: { toc: TocHeadingWithTitleSupertitle[] className?: string + title: string }) { const [height, setHeight] = useState<"auto" | 0>(0) const [isOpen, setIsOpen] = useState(false) @@ -23,28 +25,28 @@ export default function SDGTableOfContents({ return (