Merge pull request #3337 from owid/prefixed-algolia-index

🐝 Add the ability to prefix algolia index names

.env.example-full

@@ -41,3 +41,14 @@ IMAGE_HOSTING_SPACE_ACCESS_KEY_ID=''
 IMAGE_HOSTING_SPACE_SECRET_ACCESS_KEY=''
 
 OPENAI_API_KEY=''
+
+# enable search (readonly)
+ALGOLIA_ID=''   # optional
+ALGOLIA_SEARCH_KEY=''  # optional
+
+# write records to a (staging) search index
+ALGOLIA_INDEX_PREFIX=''  # optional
+ALGOLIA_SECRET_KEY=''  # optional
+ALGOLIA_INDEXING=false  # optional
+
+DATA_API_URL='' # optional

Makefile

@@ -18,7 +18,7 @@ ifneq (,$(wildcard ./.env))
 	export
 endif
 
-.PHONY: help up up.full down down.full refresh refresh.wp refresh.full migrate svgtest
+.PHONY: help up up.full down down.full refresh refresh.wp refresh.full migrate svgtest itsJustJavascript
 
 help:
 	@echo 'Available commands:'
@@ -39,6 +39,7 @@ help:
 	@echo '  make refresh.wp        download a new wordpress snapshot and update MySQL'
 	@echo '  make refresh.full      do a full MySQL update of both wordpress and grapher'
 	@echo '  make sync-images       sync all images from the remote master'
+	@echo '  make reindex			reindex (or initialise) search in Algolia'
 	@echo
 	@echo '  OPS (staff-only)'
 	@echo '  make deploy            Deploy your local site to production'
@@ -335,3 +336,23 @@ svgtest: ../owid-grapher-svgs
 ../owid-content:
 	@echo '==> Cloning owid-content to ../owid-content'
 	cd .. && git clone [email protected]:owid/owid-content
+
+node_modules: package.json yarn.lock yarn.config.cjs
+	@echo '==> Installing packages'
+	yarn install
+
+itsJustJavascript: node_modules
+	@echo '==> Compiling TS'
+	yarn lerna run build
+	yarn run tsc -b
+	touch $@
+
+reindex: itsJustJavascript
+	@echo '==> Reindexing search in Algolia'
+	node --enable-source-maps itsJustJavascript/baker/algolia/configureAlgolia.js
+	node --enable-source-maps itsJustJavascript/baker/algolia/indexToAlgolia.js
+	node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js
+	node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorersToAlgolia.js
+
+clean:
+	rm -rf node_modules itsJustJavascript

baker/algolia/configureAlgolia.ts

@@ -11,8 +11,9 @@ import {
 } from "../../settings/serverSettings.js"
 import { countries } from "@ourworldindata/utils"
 import { SearchIndexName } from "../../site/search/searchTypes.js"
+import { getIndexName } from "../../site/search/searchClient.js"
 
-export const CONTENT_GRAPH_ALGOLIA_INDEX = "graph"
+export const CONTENT_GRAPH_ALGOLIA_INDEX = getIndexName("graph")
 
 export const getAlgoliaClient = (): SearchClient | undefined => {
     if (!ALGOLIA_ID || !ALGOLIA_SECRET_KEY) {
@@ -58,7 +59,7 @@ export const configureAlgolia = async () => {
         unretrievableAttributes: ["views_7d", "score"],
     }
 
-    const chartsIndex = client.initIndex(SearchIndexName.Charts)
+    const chartsIndex = client.initIndex(getIndexName(SearchIndexName.Charts))
 
     await chartsIndex.setSettings({
         ...baseSettings,
@@ -89,7 +90,7 @@ export const configureAlgolia = async () => {
         optionalWords: ["vs"],
     })
 
-    const pagesIndex = client.initIndex(SearchIndexName.Pages)
+    const pagesIndex = client.initIndex(getIndexName(SearchIndexName.Pages))
 
     await pagesIndex.setSettings({
         ...baseSettings,
@@ -112,7 +113,9 @@ export const configureAlgolia = async () => {
         disableExactOnAttributes: ["tags"],
     })
 
-    const explorersIndex = client.initIndex(SearchIndexName.Explorers)
+    const explorersIndex = client.initIndex(
+        getIndexName(SearchIndexName.Explorers)
+    )
 
     await explorersIndex.setSettings({
         ...baseSettings,

baker/algolia/indexChartsToAlgolia.ts

@@ -9,6 +9,7 @@ import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js"
 import { Link } from "../../db/model/Link.js"
 import { getRelatedArticles } from "../../db/model/Post.js"
 import { Knex } from "knex"
+import { getIndexName } from "../../site/search/searchClient.js"
 
 const computeScore = (record: Omit<ChartRecord, "score">): number => {
     const { numRelatedArticles, views_7d } = record
@@ -114,7 +115,7 @@ const indexChartsToAlgolia = async () => {
         return
     }
 
-    const index = client.initIndex(SearchIndexName.Charts)
+    const index = client.initIndex(getIndexName(SearchIndexName.Charts))
 
     await db.getConnection()
     const records = await getChartsRecords(db.knexInstance())

baker/algolia/indexExplorersToAlgolia.ts

@@ -2,18 +2,21 @@ import cheerio from "cheerio"
 import { isArray } from "lodash"
 import { match } from "ts-pattern"
 import {
+    DbEnrichedChart,
+    DbRawChart,
     checkIsPlainObjectWithGuard,
     identity,
     keyBy,
+    parseChartsRow,
 } from "@ourworldindata/utils"
 import { getAlgoliaClient } from "./configureAlgolia.js"
 import * as db from "../../db/db.js"
 import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js"
 import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js"
 import { chunkParagraphs } from "../chunk.js"
 import { SearchIndexName } from "../../site/search/searchTypes.js"
-import { Chart } from "../../db/model/Chart.js"
 import { Knex } from "knex"
+import { getIndexName } from "../../site/search/searchClient.js"
 
 type ExplorerBlockColumns = {
     type: "columns"
@@ -47,7 +50,7 @@ type ExplorerRecord = {
 
 function extractTextFromExplorer(
     blocksString: string,
-    graphersUsedInExplorers: Record<number, Chart | null>
+    graphersUsedInExplorers: Record<number, DbEnrichedChart | null>
 ): string {
     const blockText = new Set<string>()
     const blocks = JSON.parse(blocksString)
@@ -119,17 +122,17 @@ const getExplorerRecords = async (
 
     // Fetch info about all charts used in explorers, as linked by the explorer_charts table
     const graphersUsedInExplorers = await db
-        .knexRaw<{ chartId: number }>(
+        .knexRaw<DbRawChart>(
             `
-        SELECT DISTINCT chartId
-        FROM explorer_charts
+        SELECT * FROM charts
+        INNER JOIN (
+            SELECT DISTINCT chartId AS id FROM explorer_charts
+        ) AS ec
+        USING (id)
         `,
             knex
         )
-        .then((results: { chartId: number }[]) =>
-            results.map(({ chartId }) => chartId)
-        )
-        .then((ids) => Promise.all(ids.map((id) => Chart.findOneBy({ id }))))
+        .then((charts) => charts.map((c) => parseChartsRow(c)))
         .then((charts) => keyBy(charts, "id"))
 
     const explorerRecords = await db
@@ -193,7 +196,7 @@ const indexExplorersToAlgolia = async () => {
     }
 
     try {
-        const index = client.initIndex(SearchIndexName.Explorers)
+        const index = client.initIndex(getIndexName(SearchIndexName.Explorers))
 
         const knex = db.knexInstance()
         const records = await getExplorerRecords(knex)

baker/algolia/indexToAlgolia.tsx

@@ -32,6 +32,7 @@ import {
     getPostsFromSnapshots,
 } from "../../db/model/Post.js"
 import { Knex } from "knex"
+import { getIndexName } from "../../site/search/searchClient.js"
 
 interface TypeAndImportance {
     type: PageType
@@ -232,7 +233,7 @@ const indexToAlgolia = async (knex: Knex<any, any[]>) => {
         console.error(`Failed indexing pages (Algolia client not initialized)`)
         return
     }
-    const index = client.initIndex(SearchIndexName.Pages)
+    const index = client.initIndex(getIndexName(SearchIndexName.Pages))
 
     await db.getConnection()
     const records = await getPagesRecords(knex)

settings/clientSettings.ts

@@ -44,6 +44,8 @@ export const WORDPRESS_URL: string = process.env.WORDPRESS_URL ?? ""
 
 export const ALGOLIA_ID: string = process.env.ALGOLIA_ID ?? ""
 export const ALGOLIA_SEARCH_KEY: string = process.env.ALGOLIA_SEARCH_KEY ?? ""
+export const ALGOLIA_INDEX_PREFIX: string =
+    process.env.ALGOLIA_INDEX_PREFIX ?? ""
 
 export const DONATE_API_URL: string =
     process.env.DONATE_API_URL ?? "http://localhost:8788/donation/donate"

site/search/Autocomplete.tsx

@@ -20,7 +20,11 @@ import {
 } from "../../settings/clientSettings.js"
 import { faSearch } from "@fortawesome/free-solid-svg-icons"
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js"
-import { DEFAULT_SEARCH_PLACEHOLDER } from "./searchClient.js"
+import {
+    DEFAULT_SEARCH_PLACEHOLDER,
+    getIndexName,
+    parseIndexName,
+} from "./searchClient.js"
 
 type BaseItem = Record<string, unknown>
 
@@ -64,7 +68,7 @@ const getItemUrl: AutocompleteSource<BaseItem>["getItemUrl"] = ({ item }) =>
 // The slugs we index to Algolia don't include the /grapher/ or /explorers/ directories
 // Prepend them with this function when we need them
 const prependSubdirectoryToAlgoliaItemUrl = (item: BaseItem): string => {
-    const indexName = item.__autocomplete_indexName as SearchIndexName
+    const indexName = parseIndexName(item.__autocomplete_indexName as string)
     const subdirectory = indexNameToSubdirectoryMap[indexName]
     return `${subdirectory}/${item.slug}`
 }
@@ -111,23 +115,23 @@ const AlgoliaSource: AutocompleteSource<BaseItem> = {
             searchClient,
             queries: [
                 {
-                    indexName: SearchIndexName.Pages,
+                    indexName: getIndexName(SearchIndexName.Pages),
                     query,
                     params: {
                         hitsPerPage: 2,
                         distinct: true,
                     },
                 },
                 {
-                    indexName: SearchIndexName.Charts,
+                    indexName: getIndexName(SearchIndexName.Charts),
                     query,
                     params: {
                         hitsPerPage: 2,
                         distinct: true,
                     },
                 },
                 {
-                    indexName: SearchIndexName.Explorers,
+                    indexName: getIndexName(SearchIndexName.Explorers),
                     query,
                     params: {
                         hitsPerPage: 1,
@@ -141,7 +145,9 @@ const AlgoliaSource: AutocompleteSource<BaseItem> = {
     templates: {
         header: () => <h5 className="overline-black-caps">Top Results</h5>,
         item: ({ item, components }) => {
-            const index = item.__autocomplete_indexName as SearchIndexName
+            const index = parseIndexName(
+                item.__autocomplete_indexName as string
+            )
             const indexLabel =
                 index === SearchIndexName.Charts
                     ? "Chart"