From 8dafdefdfb83264f7a1d85c903c83a6cbb5f6849 Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Mon, 25 Sep 2023 17:16:04 +0200
Subject: [PATCH 01/10] fix(algolia): index explorers even if there isn't any
 text

---
 baker/algolia/indexExplorersToAlgolia.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts
index 160fafab58c..1cd7f91a494 100644
--- a/baker/algolia/indexExplorersToAlgolia.ts
+++ b/baker/algolia/indexExplorersToAlgolia.ts
@@ -107,9 +107,14 @@ const getExplorerRecords = async (): Promise<ExplorerRecord[]> => {
                     [...uniqueTextTokens].join(" "),
                     1000
                 )
+
+                // In case we don't have any text for this explorer, we still want to index it
+                const textChunksForIteration = textChunks.length
+                    ? textChunks
+                    : [""]
                 const explorerRecords = []
                 let i = 0
-                for (const chunk of textChunks) {
+                for (const chunk of textChunksForIteration) {
                     explorerRecords.push({
                         slug,
                         title: getNullishJSONValueAsPlaintext(title),

From 18521e6db5c14f971bf8aab1f6b82d9e4fc1fccc Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Mon, 25 Sep 2023 17:18:13 +0200
Subject: [PATCH 02/10] enhance(algolia): extract text from `grapherId`
 explorers

---
 baker/algolia/indexExplorersToAlgolia.ts | 57 +++++++++++++++++++++---
 1 file changed, 51 insertions(+), 6 deletions(-)

diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts
index 1cd7f91a494..49add49e324 100644
--- a/baker/algolia/indexExplorersToAlgolia.ts
+++ b/baker/algolia/indexExplorersToAlgolia.ts
@@ -1,13 +1,18 @@
 import cheerio from "cheerio"
 import { isArray } from "lodash"
 import { match } from "ts-pattern"
-import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils"
+import {
+    checkIsPlainObjectWithGuard,
+    identity,
+    keyBy,
+} from "@ourworldindata/utils"
 import { getAlgoliaClient } from "./configureAlgolia.js"
 import * as db from "../../db/db.js"
 import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
 import { Pageview } from "../../db/model/Pageview.js"
 import { chunkParagraphs } from "../chunk.js"
 import { SearchIndexName } from "../../site/search/searchTypes.js"
+import { Chart } from "../../db/model/Chart.js"
 
 type ExplorerBlockColumns = {
     type: "columns"
@@ -17,8 +22,9 @@ type ExplorerBlockColumns = {
 type ExplorerBlockGraphers = {
     type: "graphers"
     block: {
-        title: string
+        title?: string
         subtitle?: string
+        grapherId?: number
     }[]
 }
 
@@ -38,7 +44,10 @@ type ExplorerRecord = {
     text: string
 }
 
-function extractTextFromExplorer(blocksString: string): string {
+function extractTextFromExplorer(
+    blocksString: string,
+    graphersUsedInExplorers: Record<number, Chart | null>
+): string {
     const blockText = new Set<string>()
     const blocks = JSON.parse(blocksString)
 
@@ -61,9 +70,28 @@ function extractTextFromExplorer(blocksString: string): string {
                         { type: "graphers" },
                         (graphers: ExplorerBlockGraphers) => {
                             graphers.block.forEach(
-                                ({ title = "", subtitle = "" }) => {
+                                ({
+                                    title = "",
+                                    subtitle = "",
+                                    grapherId = undefined,
+                                }) => {
                                     blockText.add(title)
                                     blockText.add(subtitle)
+
+                                    if (grapherId !== undefined) {
+                                        const chartConfig =
+                                            graphersUsedInExplorers[grapherId]
+                                                ?.config
+
+                                        if (chartConfig) {
+                                            blockText.add(
+                                                chartConfig.title ?? ""
+                                            )
+                                            blockText.add(
+                                                chartConfig.subtitle ?? ""
+                                            )
+                                        }
+                                    }
                                 }
                             )
                         }
@@ -76,7 +104,7 @@ function extractTextFromExplorer(blocksString: string): string {
         }
     }
 
-    return [...blockText].join(" ")
+    return [...blockText].filter(identity).join(" ")
 }
 
 function getNullishJSONValueAsPlaintext(value: string): string {
@@ -86,6 +114,20 @@ function getNullishJSONValueAsPlaintext(value: string): string {
 const getExplorerRecords = async (): Promise<ExplorerRecord[]> => {
     const pageviews = await Pageview.getViewsByUrlObj()
 
+    // Fetch info about all charts used in explorers, as linked by the explorer_charts table
+    const graphersUsedInExplorers = await db
+        .queryMysql(
+            `
+        SELECT DISTINCT chartId
+        FROM explorer_charts
+        `
+        )
+        .then((results: { chartId: number }[]) =>
+            results.map(({ chartId }) => chartId)
+        )
+        .then((ids) => Promise.all(ids.map((id) => Chart.findOneBy({ id }))))
+        .then((charts) => keyBy(charts, "id"))
+
     const explorerRecords = await db
         .queryMysql(
             `
@@ -99,7 +141,10 @@ const getExplorerRecords = async (): Promise<ExplorerRecord[]> => {
         )
         .then((results: ExplorerEntry[]) =>
             results.flatMap(({ slug, title, subtitle, blocks }) => {
-                const textFromExplorer = extractTextFromExplorer(blocks)
+                const textFromExplorer = extractTextFromExplorer(
+                    blocks,
+                    graphersUsedInExplorers
+                )
                 const uniqueTextTokens = new Set([
                     ...textFromExplorer.split(" "),
                 ])

From fa563dcc043fb37b8c22746ce04b82d8d6a26d5b Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Mon, 25 Sep 2023 18:33:10 +0200
Subject: [PATCH 03/10] enhance(explorer): always show an explicit "sort by
 name" option in explorer entity picker

---
 explorer/Explorer.tsx                         |  5 +++-
 .../controls/entityPicker/EntityPicker.tsx    | 23 ++++++++++++-------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/explorer/Explorer.tsx b/explorer/Explorer.tsx
index 155a5ba4b86..d5adebf26c6 100644
--- a/explorer/Explorer.tsx
+++ b/explorer/Explorer.tsx
@@ -7,6 +7,7 @@ import {
     extractPotentialDataSlugsFromTransform,
     OwidColumnDef,
     OwidTable,
+    OwidTableSlugs,
     SortOrder,
     TableSlug,
 } from "@ourworldindata/core-table"
@@ -1036,7 +1037,9 @@ export class Explorer
             ])
             return allColumnDefs.filter(
                 (def) =>
-                    def.type === undefined || !discardColumnTypes.has(def.type)
+                    (def.type === undefined ||
+                        !discardColumnTypes.has(def.type)) &&
+                    def.slug !== undefined
             )
         }
     }
diff --git a/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx b/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
index ee1a221c194..bc73ee45e76 100644
--- a/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
+++ b/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
@@ -21,6 +21,8 @@ import {
     getUserCountryInformation,
     regions,
     sortBy,
+    upperFirst,
+    compact,
 } from "@ourworldindata/utils"
 import { VerticalScrollContainer } from "../../controls/VerticalScrollContainer"
 import { SortIcon } from "../../controls/SortIcon"
@@ -120,8 +122,16 @@ export class EntityPicker extends React.Component<{
         label: string
         value: string | undefined
     }[] {
-        return [
+        const entityNameColumn = this.grapherTable?.entityNameColumn
+        const entityNameColumnInPickerColumnDefs = !!this.pickerColumnDefs.find(
+            (col) => col.slug === entityNameColumn?.slug
+        )
+        return compact([
             { label: "Relevance", value: undefined },
+            !entityNameColumnInPickerColumnDefs && {
+                label: upperFirst(this.manager.entityType) ?? "Name",
+                value: entityNameColumn?.slug,
+            },
             ...this.pickerColumnDefs.map(
                 (
                     col
@@ -135,11 +145,13 @@ export class EntityPicker extends React.Component<{
                     }
                 }
             ),
-        ]
+        ])
     }
 
     private getColumn(slug: ColumnSlug | undefined): CoreColumn | undefined {
         if (slug === undefined) return undefined
+        if (slug === OwidTableSlugs.entityName)
+            return this.manager.grapherTable?.entityNameColumn
         return this.manager.entityPickerTable?.get(slug)
     }
 
@@ -461,12 +473,7 @@ export class EntityPicker extends React.Component<{
     }
 
     private get pickerMenu(): JSX.Element | null {
-        if (
-            this.isDropdownMenu ||
-            !this.manager.entityPickerColumnDefs ||
-            this.manager.entityPickerColumnDefs.length === 0
-        )
-            return null
+        if (this.isDropdownMenu) return null
         return (
             <div className="MetricSettings">
                 <span className="mainLabel">Sort by</span>

From d4a9e98c704b493e6b454b40fe67f982e8d93c29 Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Mon, 25 Sep 2023 18:37:00 +0200
Subject: [PATCH 04/10] style: remove unused import

---
 explorer/Explorer.tsx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/explorer/Explorer.tsx b/explorer/Explorer.tsx
index d5adebf26c6..0c148acb3e1 100644
--- a/explorer/Explorer.tsx
+++ b/explorer/Explorer.tsx
@@ -7,7 +7,6 @@ import {
     extractPotentialDataSlugsFromTransform,
     OwidColumnDef,
     OwidTable,
-    OwidTableSlugs,
     SortOrder,
     TableSlug,
 } from "@ourworldindata/core-table"

From 860e0c743d4ad09ff6fe2578f115808f22e46ced Mon Sep 17 00:00:00 2001
From: Ike Saunders <ikesau@protonmail.com>
Date: Mon, 25 Sep 2023 19:04:04 +0000
Subject: [PATCH 05/10] =?UTF-8?q?=E2=9C=A8=20Correct=20typography=20for=20?=
 =?UTF-8?q?R&W=20headings?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 site/gdocs/ResearchAndWriting.scss | 3 ++-
 site/gdocs/ResearchAndWriting.tsx  | 9 ++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/site/gdocs/ResearchAndWriting.scss b/site/gdocs/ResearchAndWriting.scss
index b037fa47bd4..86fe3380735 100644
--- a/site/gdocs/ResearchAndWriting.scss
+++ b/site/gdocs/ResearchAndWriting.scss
@@ -40,15 +40,16 @@
 }
 
 .research-and-writing-link {
+    h2,
     h3 {
         @include owid-link-90;
-        @include h2-bold;
         margin-top: 16px;
         margin-bottom: 8px;
         text-decoration: none;
     }
 
     &:hover {
+        h2,
         h3 {
             text-decoration: underline;
         }
diff --git a/site/gdocs/ResearchAndWriting.tsx b/site/gdocs/ResearchAndWriting.tsx
index 1f740bab7f9..cfec6508461 100644
--- a/site/gdocs/ResearchAndWriting.tsx
+++ b/site/gdocs/ResearchAndWriting.tsx
@@ -49,6 +49,12 @@ function ResearchAndWritingLinkContainer(
         filename = linkedDocument.content["featured-image"] || filename
     }
 
+    const heading = React.createElement(
+        isSmall ? "h3" : "h2",
+        { className: isSmall ? "h3-bold" : "h2-bold" },
+        title
+    )
+
     return (
         <a
             href={url}
@@ -66,7 +72,7 @@ function ResearchAndWritingLinkContainer(
                         />
                     </figure>
                 ) : null}
-                <h3>{title}</h3>
+                {heading}
                 {subtitle && !shouldHideSubtitle ? (
                     <p className="research-and-writing-link__description body-1-regular">
                         {subtitle}
@@ -135,6 +141,7 @@ export function ResearchAndWriting(props: ResearchAndWritingProps) {
                             <ResearchAndWritingLinkContainer
                                 shouldHideThumbnail
                                 shouldHideSubtitle
+                                isSmall
                                 className="span-cols-1"
                                 key={i}
                                 {...link}

From d8a592aea8cd1d95d694550d93552b3758062244 Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Tue, 26 Sep 2023 09:44:25 +0200
Subject: [PATCH 06/10] refactor: use map instead of for-loop

---
 baker/algolia/indexExplorersToAlgolia.ts | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts
index 49add49e324..812d9486086 100644
--- a/baker/algolia/indexExplorersToAlgolia.ts
+++ b/baker/algolia/indexExplorersToAlgolia.ts
@@ -157,21 +157,15 @@ const getExplorerRecords = async (): Promise<ExplorerRecord[]> => {
                 const textChunksForIteration = textChunks.length
                     ? textChunks
                     : [""]
-                const explorerRecords = []
-                let i = 0
-                for (const chunk of textChunksForIteration) {
-                    explorerRecords.push({
-                        slug,
-                        title: getNullishJSONValueAsPlaintext(title),
-                        subtitle: getNullishJSONValueAsPlaintext(subtitle),
-                        views_7d:
-                            pageviews[`/explorers/${slug}`]?.views_7d || 0,
-                        text: chunk,
-                        objectID: `${slug}-${i}`,
-                    })
-                    i++
-                }
-                return explorerRecords
+
+                return textChunksForIteration.map((chunk, i) => ({
+                    slug,
+                    title: getNullishJSONValueAsPlaintext(title),
+                    subtitle: getNullishJSONValueAsPlaintext(subtitle),
+                    views_7d: pageviews[`/explorers/${slug}`]?.views_7d ?? 0,
+                    text: chunk,
+                    objectID: `${slug}-${i}`,
+                }))
             })
         )
 

From f168b4b635990344775e5a055733378d27d583e0 Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Tue, 26 Sep 2023 11:01:51 +0200
Subject: [PATCH 07/10] refactor: various fixes for entity name handling

---
 .../controls/entityPicker/EntityPicker.tsx    | 36 +++++++++++--------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx b/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
index bc73ee45e76..98179241abd 100644
--- a/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
+++ b/packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
@@ -128,10 +128,11 @@ export class EntityPicker extends React.Component<{
         )
         return compact([
             { label: "Relevance", value: undefined },
-            !entityNameColumnInPickerColumnDefs && {
-                label: upperFirst(this.manager.entityType) ?? "Name",
-                value: entityNameColumn?.slug,
-            },
+            !entityNameColumnInPickerColumnDefs &&
+                entityNameColumn && {
+                    label: upperFirst(this.manager.entityType) || "Name",
+                    value: entityNameColumn?.slug,
+                },
             ...this.pickerColumnDefs.map(
                 (
                     col
@@ -148,15 +149,21 @@ export class EntityPicker extends React.Component<{
         ])
     }
 
-    private getColumn(slug: ColumnSlug | undefined): CoreColumn | undefined {
-        if (slug === undefined) return undefined
-        if (slug === OwidTableSlugs.entityName)
-            return this.manager.grapherTable?.entityNameColumn
-        return this.manager.entityPickerTable?.get(slug)
+    @computed private get metricTable(): OwidTable | undefined {
+        if (this.metric === undefined) return undefined
+
+        // If the slug is "entityName", then try to get it from grapherTable first, because it might
+        // not be present in pickerTable (for indicator-powered explorers, for example).
+        if (
+            this.metric === OwidTableSlugs.entityName &&
+            this.grapherTable?.has(this.metric)
+        )
+            return this.grapherTable
+        return this.pickerTable
     }
 
     @computed private get activePickerMetricColumn(): CoreColumn | undefined {
-        return this.getColumn(this.metric)
+        return this.metricTable?.get(this.metric)
     }
 
     @computed private get availableEntitiesForCurrentView(): Set<string> {
@@ -190,13 +197,13 @@ export class EntityPicker extends React.Component<{
 
     @computed
     private get entitiesWithMetricValue(): EntityOptionWithMetricValue[] {
-        const { pickerTable, selection, localEntityNames } = this
+        const { metricTable, selection, localEntityNames } = this
         const col = this.activePickerMetricColumn
         const entityNames = selection.availableEntityNames.slice().sort()
         return entityNames.map((entityName) => {
             const plotValue =
-                col && pickerTable
-                    ? (pickerTable.getLatestValueForEntity(
+                col && metricTable?.has(col.slug)
+                    ? (metricTable.getLatestValueForEntity(
                           entityName,
                           col.slug
                       ) as string | number)
@@ -447,7 +454,7 @@ export class EntityPicker extends React.Component<{
     }
 
     @action private updateMetric(columnSlug: ColumnSlug): void {
-        const col = this.getColumn(columnSlug)
+        const col = this.pickerTable?.get(columnSlug)
 
         this.manager.setEntityPicker?.({
             metric: columnSlug,
@@ -465,6 +472,7 @@ export class EntityPicker extends React.Component<{
         return (
             // If columnSlug is undefined, we're sorting by relevance, which is (mostly) by country name.
             columnSlug !== undefined &&
+            columnSlug !== OwidTableSlugs.entityName &&
             // If the column is currently missing (not loaded yet), assume it is numeric.
             (col === undefined ||
                 col.isMissing ||

From 027561d9366e8c2f7a56b9984deab03876b5a22f Mon Sep 17 00:00:00 2001
From: Marcel Gerber <mgerber59@gmail.com>
Date: Tue, 26 Sep 2023 11:02:49 +0200
Subject: [PATCH 08/10] enhance: eagerly provide current grapher table as
 pickerTable

so sorting by entity name immediately infers the correct alphabetical sort order
---
 explorer/Explorer.tsx | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/explorer/Explorer.tsx b/explorer/Explorer.tsx
index 0c148acb3e1..4ae8052105d 100644
--- a/explorer/Explorer.tsx
+++ b/explorer/Explorer.tsx
@@ -952,14 +952,14 @@ export class Explorer
     })
 
     private updateEntityPickerTable(): void {
-        if (this.entityPickerMetric) {
-            this.entityPickerTableIsLoading = true
-            this.futureEntityPickerTable.set(
-                this.tableLoader.get(
-                    this.getTableSlugOfColumnSlug(this.entityPickerMetric)
-                )
-            )
-        }
+        // If we don't currently have a entity picker metric, then set pickerTable to the currently-used table anyways,
+        // so that when we start sorting by entity name we can infer that the column is a string column immediately.
+        const tableSlugToLoad = this.entityPickerMetric
+            ? this.getTableSlugOfColumnSlug(this.entityPickerMetric)
+            : this.explorerProgram.grapherConfig.tableSlug
+
+        this.entityPickerTableIsLoading = true
+        this.futureEntityPickerTable.set(this.tableLoader.get(tableSlugToLoad))
     }
 
     setEntityPicker({

From 233117e3453ebc7c5fdcd8647404a5479e3beea4 Mon Sep 17 00:00:00 2001
From: sophiamersmann <sophia.mersmann1@gmail.com>
Date: Tue, 26 Sep 2023 14:22:37 +0000
Subject: [PATCH 09/10] fix(explorer): fix data loading issue in mixed-content
 explorers

---
 explorer/Explorer.tsx | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/explorer/Explorer.tsx b/explorer/Explorer.tsx
index 4ae8052105d..00be6fcad85 100644
--- a/explorer/Explorer.tsx
+++ b/explorer/Explorer.tsx
@@ -461,6 +461,7 @@ export class Explorer
             ...this.explorerProgram.grapherConfigOnlyGrapherProps,
             bakedGrapherURL: BAKED_GRAPHER_URL,
             hideEntityControls: this.showExplorerControls,
+            manuallyProvideData: false,
         }
 
         grapher.setAuthoredVersion(config)
@@ -498,6 +499,7 @@ export class Explorer
             ...this.explorerProgram.grapherConfigOnlyGrapherProps,
             bakedGrapherURL: BAKED_GRAPHER_URL,
             hideEntityControls: this.showExplorerControls,
+            manuallyProvideData: false,
         }
 
         // set given variable IDs as dimensions to make Grapher

From 008f996512a5aa5e42c730dfdb9519d40f7f95dc Mon Sep 17 00:00:00 2001
From: Mojmir Vinkler <mojmir.vinkler@gmail.com>
Date: Tue, 26 Sep 2023 16:48:14 +0200
Subject: [PATCH 10/10] :bug: show source publishedBy alongside
 origin.fullCitation (#2663)

---
 .../grapher/src/sourcesTab/SourcesTab.tsx     | 26 +++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/packages/@ourworldindata/grapher/src/sourcesTab/SourcesTab.tsx b/packages/@ourworldindata/grapher/src/sourcesTab/SourcesTab.tsx
index e740a01958a..78e27ca4daa 100644
--- a/packages/@ourworldindata/grapher/src/sourcesTab/SourcesTab.tsx
+++ b/packages/@ourworldindata/grapher/src/sourcesTab/SourcesTab.tsx
@@ -97,6 +97,12 @@ export class SourcesTab extends React.Component<{
                       )
                   )
                 : []
+
+        const publishedByArray = [
+            ...(source.dataPublishedBy ? [source.dataPublishedBy] : []),
+            ...citationFull,
+        ]
+
         return (
             <div key={slug} className="datasource-wrapper">
                 <h2>
@@ -151,23 +157,23 @@ export class SourcesTab extends React.Component<{
                                 <td>{column.unitConversionFactor}</td>
                             </tr>
                         ) : null}
-                        {citationFull.length === 1 ? (
+                        {publishedByArray.length === 1 ? (
                             <tr>
                                 <td>Data published by</td>
                                 <td>
                                     <MarkdownTextWrap
-                                        text={citationFull[0]}
+                                        text={publishedByArray[0]}
                                         fontSize={12}
                                     />
                                 </td>
                             </tr>
                         ) : null}
-                        {citationFull.length > 1 ? (
+                        {publishedByArray.length > 1 ? (
                             <tr>
                                 <td>Data published by</td>
                                 <td>
                                     <ul>
-                                        {citationFull.map(
+                                        {publishedByArray.map(
                                             (
                                                 citation: string,
                                                 index: number
@@ -184,18 +190,6 @@ export class SourcesTab extends React.Component<{
                                 </td>
                             </tr>
                         ) : null}
-                        {(!citationFull || citationFull.length === 0) &&
-                        source.dataPublishedBy ? (
-                            <tr>
-                                <td>Data published by</td>
-                                <td>
-                                    <HtmlOrMarkdownText
-                                        text={source.dataPublishedBy}
-                                        fontSize={12}
-                                    />
-                                </td>
-                            </tr>
-                        ) : null}
                         {source.dataPublisherSource ? (
                             <tr>
                                 <td>Data publisher's source</td>