Skip to content

Commit

Permalink
Merge branch 'master' into index-updated-at
Browse files Browse the repository at this point in the history
  • Loading branch information
@ikesau
ikesau authored Sep 26, 2023
2 parents 9d6b18f + 14ef2b0 commit 1c13803
Show file tree
Hide file tree
Showing 6 changed files with 129 additions and 64 deletions.
86 changes: 65 additions & 21 deletions baker/algolia/indexExplorersToAlgolia.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
import cheerio from "cheerio"
import { isArray } from "lodash"
import { match } from "ts-pattern"
import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils"
import {
checkIsPlainObjectWithGuard,
identity,
keyBy,
} from "@ourworldindata/utils"
import { getAlgoliaClient } from "./configureAlgolia.js"
import * as db from "../../db/db.js"
import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
import { Pageview } from "../../db/model/Pageview.js"
import { chunkParagraphs } from "../chunk.js"
import { SearchIndexName } from "../../site/search/searchTypes.js"
import { Chart } from "../../db/model/Chart.js"

type ExplorerBlockColumns = {
type: "columns"
Expand All @@ -17,8 +22,9 @@ type ExplorerBlockColumns = {
type ExplorerBlockGraphers = {
type: "graphers"
block: {
title: string
title?: string
subtitle?: string
grapherId?: number
}[]
}

Expand All @@ -38,7 +44,10 @@ type ExplorerRecord = {
text: string
}

function extractTextFromExplorer(blocksString: string): string {
function extractTextFromExplorer(
blocksString: string,
graphersUsedInExplorers: Record<number, Chart | null>
): string {
const blockText = new Set<string>()
const blocks = JSON.parse(blocksString)

Expand All @@ -61,9 +70,28 @@ function extractTextFromExplorer(blocksString: string): string {
{ type: "graphers" },
(graphers: ExplorerBlockGraphers) => {
graphers.block.forEach(
({ title = "", subtitle = "" }) => {
({
title = "",
subtitle = "",
grapherId = undefined,
}) => {
blockText.add(title)
blockText.add(subtitle)

if (grapherId !== undefined) {
const chartConfig =
graphersUsedInExplorers[grapherId]
?.config

if (chartConfig) {
blockText.add(
chartConfig.title ?? ""
)
blockText.add(
chartConfig.subtitle ?? ""
)
}
}
}
)
}
Expand All @@ -76,7 +104,7 @@ function extractTextFromExplorer(blocksString: string): string {
}
}

return [...blockText].join(" ")
return [...blockText].filter(identity).join(" ")
}

function getNullishJSONValueAsPlaintext(value: string): string {
Expand All @@ -86,6 +114,20 @@ function getNullishJSONValueAsPlaintext(value: string): string {
const getExplorerRecords = async (): Promise<ExplorerRecord[]> => {
const pageviews = await Pageview.getViewsByUrlObj()

// Fetch info about all charts used in explorers, as linked by the explorer_charts table
const graphersUsedInExplorers = await db
.queryMysql(
`
SELECT DISTINCT chartId
FROM explorer_charts
`
)
.then((results: { chartId: number }[]) =>
results.map(({ chartId }) => chartId)
)
.then((ids) => Promise.all(ids.map((id) => Chart.findOneBy({ id }))))
.then((charts) => keyBy(charts, "id"))

const explorerRecords = await db
.queryMysql(
`
Expand All @@ -99,29 +141,31 @@ const getExplorerRecords = async (): Promise<ExplorerRecord[]> => {
)
.then((results: ExplorerEntry[]) =>
results.flatMap(({ slug, title, subtitle, blocks }) => {
const textFromExplorer = extractTextFromExplorer(blocks)
const textFromExplorer = extractTextFromExplorer(
blocks,
graphersUsedInExplorers
)
const uniqueTextTokens = new Set([
...textFromExplorer.split(" "),
])
const textChunks = chunkParagraphs(
[...uniqueTextTokens].join(" "),
1000
)
const explorerRecords = []
let i = 0
for (const chunk of textChunks) {
explorerRecords.push({
slug,
title: getNullishJSONValueAsPlaintext(title),
subtitle: getNullishJSONValueAsPlaintext(subtitle),
views_7d:
pageviews[`/explorers/${slug}`]?.views_7d || 0,
text: chunk,
objectID: `${slug}-${i}`,
})
i++
}
return explorerRecords

// In case we don't have any text for this explorer, we still want to index it
const textChunksForIteration = textChunks.length
? textChunks
: [""]

return textChunksForIteration.map((chunk, i) => ({
slug,
title: getNullishJSONValueAsPlaintext(title),
subtitle: getNullishJSONValueAsPlaintext(subtitle),
views_7d: pageviews[`/explorers/${slug}`]?.views_7d ?? 0,
text: chunk,
objectID: `${slug}-${i}`,
}))
})
)

Expand Down
22 changes: 13 additions & 9 deletions explorer/Explorer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,7 @@ export class Explorer
...this.explorerProgram.grapherConfigOnlyGrapherProps,
bakedGrapherURL: BAKED_GRAPHER_URL,
hideEntityControls: this.showExplorerControls,
manuallyProvideData: false,
}

grapher.setAuthoredVersion(config)
Expand Down Expand Up @@ -498,6 +499,7 @@ export class Explorer
...this.explorerProgram.grapherConfigOnlyGrapherProps,
bakedGrapherURL: BAKED_GRAPHER_URL,
hideEntityControls: this.showExplorerControls,
manuallyProvideData: false,
}

// set given variable IDs as dimensions to make Grapher
Expand Down Expand Up @@ -952,14 +954,14 @@ export class Explorer
})

private updateEntityPickerTable(): void {
if (this.entityPickerMetric) {
this.entityPickerTableIsLoading = true
this.futureEntityPickerTable.set(
this.tableLoader.get(
this.getTableSlugOfColumnSlug(this.entityPickerMetric)
)
)
}
// If we don't currently have a entity picker metric, then set pickerTable to the currently-used table anyways,
// so that when we start sorting by entity name we can infer that the column is a string column immediately.
const tableSlugToLoad = this.entityPickerMetric
? this.getTableSlugOfColumnSlug(this.entityPickerMetric)
: this.explorerProgram.grapherConfig.tableSlug

this.entityPickerTableIsLoading = true
this.futureEntityPickerTable.set(this.tableLoader.get(tableSlugToLoad))
}

setEntityPicker({
Expand Down Expand Up @@ -1036,7 +1038,9 @@ export class Explorer
])
return allColumnDefs.filter(
(def) =>
def.type === undefined || !discardColumnTypes.has(def.type)
(def.type === undefined ||
!discardColumnTypes.has(def.type)) &&
def.slug !== undefined
)
}
}
Expand Down
47 changes: 31 additions & 16 deletions packages/@ourworldindata/grapher/src/controls/entityPicker/EntityPicker.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import {
getUserCountryInformation,
regions,
sortBy,
upperFirst,
compact,
} from "@ourworldindata/utils"
import { VerticalScrollContainer } from "../../controls/VerticalScrollContainer"
import { SortIcon } from "../../controls/SortIcon"
Expand Down Expand Up @@ -120,8 +122,17 @@ export class EntityPicker extends React.Component<{
label: string
value: string | undefined
}[] {
return [
const entityNameColumn = this.grapherTable?.entityNameColumn
const entityNameColumnInPickerColumnDefs = !!this.pickerColumnDefs.find(
(col) => col.slug === entityNameColumn?.slug
)
return compact([
{ label: "Relevance", value: undefined },
!entityNameColumnInPickerColumnDefs &&
entityNameColumn && {
label: upperFirst(this.manager.entityType) || "Name",
value: entityNameColumn?.slug,
},
...this.pickerColumnDefs.map(
(
col
Expand All @@ -135,16 +146,24 @@ export class EntityPicker extends React.Component<{
}
}
),
]
])
}

private getColumn(slug: ColumnSlug | undefined): CoreColumn | undefined {
if (slug === undefined) return undefined
return this.manager.entityPickerTable?.get(slug)
@computed private get metricTable(): OwidTable | undefined {
if (this.metric === undefined) return undefined

// If the slug is "entityName", then try to get it from grapherTable first, because it might
// not be present in pickerTable (for indicator-powered explorers, for example).
if (
this.metric === OwidTableSlugs.entityName &&
this.grapherTable?.has(this.metric)
)
return this.grapherTable
return this.pickerTable
}

@computed private get activePickerMetricColumn(): CoreColumn | undefined {
return this.getColumn(this.metric)
return this.metricTable?.get(this.metric)
}

@computed private get availableEntitiesForCurrentView(): Set<string> {
Expand Down Expand Up @@ -178,13 +197,13 @@ export class EntityPicker extends React.Component<{

@computed
private get entitiesWithMetricValue(): EntityOptionWithMetricValue[] {
const { pickerTable, selection, localEntityNames } = this
const { metricTable, selection, localEntityNames } = this
const col = this.activePickerMetricColumn
const entityNames = selection.availableEntityNames.slice().sort()
return entityNames.map((entityName) => {
const plotValue =
col && pickerTable
? (pickerTable.getLatestValueForEntity(
col && metricTable?.has(col.slug)
? (metricTable.getLatestValueForEntity(
entityName,
col.slug
) as string | number)
Expand Down Expand Up @@ -435,7 +454,7 @@ export class EntityPicker extends React.Component<{
}

@action private updateMetric(columnSlug: ColumnSlug): void {
const col = this.getColumn(columnSlug)
const col = this.pickerTable?.get(columnSlug)

this.manager.setEntityPicker?.({
metric: columnSlug,
Expand All @@ -453,6 +472,7 @@ export class EntityPicker extends React.Component<{
return (
// If columnSlug is undefined, we're sorting by relevance, which is (mostly) by country name.
columnSlug !== undefined &&
columnSlug !== OwidTableSlugs.entityName &&
// If the column is currently missing (not loaded yet), assume it is numeric.
(col === undefined ||
col.isMissing ||
Expand All @@ -461,12 +481,7 @@ export class EntityPicker extends React.Component<{
}

private get pickerMenu(): JSX.Element | null {
if (
this.isDropdownMenu ||
!this.manager.entityPickerColumnDefs ||
this.manager.entityPickerColumnDefs.length === 0
)
return null
if (this.isDropdownMenu) return null
return (
<div className="MetricSettings">
<span className="mainLabel">Sort by</span>
Expand Down
26 changes: 10 additions & 16 deletions packages/@ourworldindata/grapher/src/sourcesTab/SourcesTab.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ export class SourcesTab extends React.Component<{
)
)
: []

const publishedByArray = [
...(source.dataPublishedBy ? [source.dataPublishedBy] : []),
...citationFull,
]

return (
<div key={slug} className="datasource-wrapper">
<h2>
Expand Down Expand Up @@ -151,23 +157,23 @@ export class SourcesTab extends React.Component<{
<td>{column.unitConversionFactor}</td>
</tr>
) : null}
{citationFull.length === 1 ? (
{publishedByArray.length === 1 ? (
<tr>
<td>Data published by</td>
<td>
<MarkdownTextWrap
text={citationFull[0]}
text={publishedByArray[0]}
fontSize={12}
/>
</td>
</tr>
) : null}
{citationFull.length > 1 ? (
{publishedByArray.length > 1 ? (
<tr>
<td>Data published by</td>
<td>
<ul>
{citationFull.map(
{publishedByArray.map(
(
citation: string,
index: number
Expand All @@ -184,18 +190,6 @@ export class SourcesTab extends React.Component<{
</td>
</tr>
) : null}
{(!citationFull || citationFull.length === 0) &&
source.dataPublishedBy ? (
<tr>
<td>Data published by</td>
<td>
<HtmlOrMarkdownText
text={source.dataPublishedBy}
fontSize={12}
/>
</td>
</tr>
) : null}
{source.dataPublisherSource ? (
<tr>
<td>Data publisher's source</td>
Expand Down
3 changes: 2 additions & 1 deletion site/gdocs/ResearchAndWriting.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,16 @@
}

.research-and-writing-link {
h2,
h3 {
@include owid-link-90;
@include h2-bold;
margin-top: 16px;
margin-bottom: 8px;
text-decoration: none;
}

&:hover {
h2,
h3 {
text-decoration: underline;
}
Expand Down
Loading

0 comments on commit 1c13803

Please sign in to comment.