experiment(algolia): make country names optional for pages and explorers

.eslintrc.cjs

@@ -1,3 +1,5 @@
+const path = require('node:path');
+
 module.exports = {
     extends: [
         "plugin:react/recommended",
@@ -12,7 +14,7 @@ module.exports = {
         ecmaFeatures: {
             jsx: true,
         },
-        project: "./tsconfig.eslint.json",
+        project: path.join(__dirname, "tsconfig.eslint.json"),
         ecmaVersion: "latest",
     },
     overrides: [

baker/algolia/configureAlgolia.ts

@@ -9,7 +9,7 @@ import {
     ALGOLIA_INDEXING,
     ALGOLIA_SECRET_KEY,
 } from "../../settings/serverSettings.js"
-import { countries } from "@ourworldindata/utils"
+import { countries, regions, excludeUndefined } from "@ourworldindata/utils"
 import { SearchIndexName } from "../../site/search/searchTypes.js"
 import { getIndexName } from "../../site/search/searchClient.js"
 
@@ -25,6 +25,11 @@ export const getAlgoliaClient = (): SearchClient | undefined => {
     return client
 }
 
+const allCountryNamesAndVariants = regions.flatMap((country) => [
+    country.name,
+    ...(("variantNames" in country && country.variantNames) || []),
+])
+
 // This function initializes and applies settings to the Algolia search indices
 // Algolia settings should be configured here rather than in the Algolia dashboard UI, as then
 // they are recorded and transferrable across dev/prod instances
@@ -164,6 +169,7 @@ export const configureAlgolia = async () => {
         attributeForDistinct: "explorerSlug",
         distinct: 4,
         minWordSizefor1Typo: 6,
+        optionalWords: allCountryNamesAndVariants,
     })
 
     const synonyms = [
@@ -307,12 +313,6 @@ export const configureAlgolia = async () => {
         ["funding", "funded"],
     ]
 
-    // Send all our country variant names to algolia as synonyms
-    for (const country of countries) {
-        if (country.variantNames)
-            synonyms.push([country.name].concat(country.variantNames))
-    }
-
     const algoliaSynonyms = synonyms.map((s) => {
         return {
             objectID: s.join("-"),
@@ -321,6 +321,22 @@ export const configureAlgolia = async () => {
         } as Synonym
     })
 
+    // Add synonyms for all country names and their variants, e.g. "US" <-> "USA" <-> "United States"
+    for (const country of countries) {
+        const alternatives = excludeUndefined([
+            country.shortName,
+            ...(country.variantNames ?? []),
+        ])
+        if (alternatives.length) {
+            const synonyms = [country.name, ...alternatives]
+            algoliaSynonyms.push({
+                objectID: synonyms.join("-"),
+                type: "synonym",
+                synonyms: synonyms,
+            })
+        }
+    }
+
     await pagesIndex.saveSynonyms(algoliaSynonyms, {
         replaceExistingSynonyms: true,
     })

baker/algolia/indexChartsToAlgolia.ts

@@ -8,6 +8,9 @@ import {
     OwidGdocLinkType,
     excludeNullish,
     isNil,
+    countries,
+    orderBy,
+    removeTrailingParenthetical,
 } from "@ourworldindata/utils"
 import { MarkdownTextWrap } from "@ourworldindata/components"
 import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js"
@@ -20,6 +23,35 @@ const computeScore = (record: Omit<ChartRecord, "score">): number => {
     return numRelatedArticles * 500 + views_7d
 }
 
+const countriesWithVariantNames = new Set(
+    countries
+        .filter((country) => country.variantNames?.length || country.shortName)
+        .map((country) => country.name)
+)
+
+const processAvailableEntities = (availableEntities: string[] | null) => {
+    if (!availableEntities) return []
+
+    // Algolia is a bit weird with synonyms:
+    // If we have a synonym "USA" -> "United States", and we search for "USA",
+    // then it seems that Algolia can only find that within `availableEntities`
+    // if "USA" is within the first 100-or-so entries of the array.
+    // So, the easy solution is to sort the entities to ensure that countries
+    // with variant names are at the top.
+    // - @marcelgerber, 2024-03-25
+    return orderBy(
+        availableEntities,
+        [
+            (entityName) =>
+                countriesWithVariantNames.has(
+                    removeTrailingParenthetical(entityName)
+                ),
+            (entityName) => entityName,
+        ],
+        ["desc", "asc"]
+    )
+}
+
 const getChartsRecords = async (
     knex: db.KnexReadonlyTransaction
 ): Promise<ChartRecord[]> => {
@@ -81,14 +113,15 @@ const getChartsRecords = async (
             if (c.entityNames.length < 12000)
                 c.entityNames = excludeNullish(
                     JSON.parse(c.entityNames as string) as (string | null)[]
-                )
+                ) as string[]
             else {
                 console.info(
                     `Chart ${c.id} has too many entities, skipping its entities`
                 )
                 c.entityNames = []
             }
         }
+        c.entityNames = processAvailableEntities(c.entityNames)
 
         c.tags = JSON.parse(c.tags)
         c.keyChartForTags = JSON.parse(c.keyChartForTags as string).filter(

packages/@ourworldindata/utils/src/Util.ts

@@ -1856,6 +1856,12 @@ export function cartesian<T>(matrix: T[][]): T[][] {
     )
 }
 
+// Remove any parenthetical content from _the end_ of a string
+// E.g. "Africa (UN)" -> "Africa"
+export function removeTrailingParenthetical(str: string): string {
+    return str.replace(/\s*\(.*\)$/, "")
+}
+
 export function isElementHidden(element: Element | null): boolean {
     if (!element) return false
     const computedStyle = window.getComputedStyle(element)

packages/@ourworldindata/utils/src/index.ts

@@ -35,6 +35,7 @@ export {
     isTouchDevice,
     type Json,
     csvEscape,
+    escapeRegExp,
     urlToSlug,
     trimObject,
     fetchText,
@@ -119,6 +120,7 @@ export {
     checkIsDataInsight,
     checkIsAuthor,
     cartesian,
+    removeTrailingParenthetical,
     isElementHidden,
 } from "./Util.js"
 
@@ -240,6 +242,7 @@ export {
     countries,
     type Country,
     getCountryBySlug,
+    getRegionByNameOrVariantName,
     isCountryName,
     continents,
     type Continent,

packages/@ourworldindata/utils/src/regions.ts

@@ -67,6 +67,18 @@ const countriesBySlug: Record<string, Country> = Object.fromEntries(
     countries.map((country) => [country.slug, country])
 )
 
+const regionsByNameOrVariantNameLowercase: Map<string, Region> = new Map(
+    regions.flatMap((region) => {
+        const names = [region.name.toLowerCase()]
+        if ("variantNames" in region && region.variantNames) {
+            names.push(
+                ...region.variantNames.map((variant) => variant.toLowerCase())
+            )
+        }
+        return names.map((name) => [name, region])
+    })
+)
+
 const currentAndHistoricalCountryNames = regions
     .filter(({ regionType }) => regionType === "country")
     .map(({ name }) => name.toLowerCase())
@@ -76,3 +88,8 @@ export const isCountryName = (name: string): boolean =>
 
 export const getCountryBySlug = (slug: string): Country | undefined =>
     countriesBySlug[slug]
+
+export const getRegionByNameOrVariantName = (
+    nameOrVariantName: string
+): Region | undefined =>
+    regionsByNameOrVariantNameLowercase.get(nameOrVariantName.toLowerCase())

site/SiteAnalytics.ts

@@ -30,11 +30,15 @@ export class SiteAnalytics extends GrapherAnalytics {
         position,
         url,
         positionInSection,
+        cardPosition,
+        positionWithinCard,
         filter,
     }: {
         query: string
         position: string
         positionInSection: string
+        cardPosition?: string
+        positionWithinCard?: string
         url: string
         filter: SearchCategoryFilter
     }) {
@@ -45,6 +49,8 @@ export class SiteAnalytics extends GrapherAnalytics {
                 query,
                 position,
                 positionInSection,
+                cardPosition,
+                positionWithinCard,
                 filter,
             }),
             eventTarget: url,

site/search/Autocomplete.scss

@@ -215,9 +215,14 @@ section[data-autocomplete-source-id="recentSearchesPlugin"]
     }
 }
 
-section[data-autocomplete-source-id="autocomplete"]
+section[data-autocomplete-source-id="autocomplete"] {
+    .aa-ItemWrapper {
+        text-wrap: pretty;
+    }
+
     .aa-ItemWrapper__contentType {
-    color: $blue-50;
+        color: $blue-50;
+    }
 }
 
 section[data-autocomplete-source-id="runSearch"] {

site/search/Autocomplete.tsx

@@ -76,7 +76,12 @@ const getItemUrl: AutocompleteSource<BaseItem>["getItemUrl"] = ({ item }) =>
 const prependSubdirectoryToAlgoliaItemUrl = (item: BaseItem): string => {
     const indexName = parseIndexName(item.__autocomplete_indexName as string)
     const subdirectory = indexNameToSubdirectoryMap[indexName]
-    return `${subdirectory}/${item.slug}`
+    switch (indexName) {
+        case SearchIndexName.ExplorerViews:
+            return `${subdirectory}/${item.explorerSlug}${item.viewQueryParams}`
+        default:
+            return `${subdirectory}/${item.slug}`
+    }
 }
 
 const FeaturedSearchesSource: AutocompleteSource<BaseItem> = {
@@ -142,7 +147,7 @@ const AlgoliaSource: AutocompleteSource<BaseItem> = {
                     },
                 },
                 {
-                    indexName: getIndexName(SearchIndexName.Explorers),
+                    indexName: getIndexName(SearchIndexName.ExplorerViews),
                     query,
                     params: {
                         hitsPerPage: 1,
@@ -162,10 +167,13 @@ const AlgoliaSource: AutocompleteSource<BaseItem> = {
             const indexLabel =
                 index === SearchIndexName.Charts
                     ? "Chart"
-                    : index === SearchIndexName.Explorers
+                    : index === SearchIndexName.ExplorerViews
                       ? "Explorer"
                       : pageTypeDisplayNames[item.type as PageType]
 
+            const mainAttribute =
+                index === SearchIndexName.ExplorerViews ? "viewTitle" : "title"
+
             return (
                 <div
                     className="aa-ItemWrapper"
@@ -175,7 +183,7 @@ const AlgoliaSource: AutocompleteSource<BaseItem> = {
                     <span>
                         <components.Highlight
                             hit={item}
-                            attribute="title"
+                            attribute={mainAttribute}
                             tagName="strong"
                         />
                     </span>