🔨 Add "make bench.search" to evaluate search performance

It fetches a dataset of synthetic queries and evaluates the extent to
which we surface good articles for the given queries.

The scoring algorithm chosen for articles is `precision@4`, meaning the
the proportion of the first four results that are relevant, averaged
over a ton of queries.

This is chosen since at most four articles are
presented un-collapsed, and the value of getting those four right is
much much higher than getting any right further down in the ranking.

It does not yet score chart or explorer search.

Makefile

@@ -40,6 +40,7 @@ help:
 	@echo '  make refresh.full      do a full MySQL update of both wordpress and grapher'
 	@echo '  make sync-images       sync all images from the remote master'
 	@echo '  make reindex			reindex (or initialise) search in Algolia'
+	@echo '  make bench.search      run search benchmarks'
 	@echo
 	@echo '  OPS (staff-only)'
 	@echo '  make deploy            Deploy your local site to production'
@@ -354,5 +355,9 @@ reindex: itsJustJavascript
 	node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js
 	node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorersToAlgolia.js
 
+bench.search: itsJustJavascript
+	@echo '==> Running search benchmarks'
+	@node --enable-source-maps itsJustJavascript/site/search/evaluateSearch.js
+
 clean:
 	rm -rf node_modules itsJustJavascript

settings/serverSettings.ts

@@ -205,3 +205,7 @@ export const OPENAI_API_KEY: string = serverSettings.OPENAI_API_KEY ?? ""
 
 export const SLACK_BOT_OAUTH_TOKEN: string =
     serverSettings.SLACK_BOT_OAUTH_TOKEN ?? ""
+
+// search evaluation
+export const SEARCH_EVAL_URL: string =
+    "https://pub-ec761fe0df554b02bc605610f3296000.r2.dev"

site/search/evaluateSearch.ts

@@ -0,0 +1,149 @@
+/**
+ * Simulate searches against our Algolia index and evaluate the results.
+ */
+
+import {
+    ALGOLIA_ID,
+    ALGOLIA_SEARCH_KEY,
+} from "../../settings/clientSettings.js"
+import { SEARCH_EVAL_URL } from "../../settings/serverSettings.js"
+import { getIndexName } from "./searchClient.js"
+import algoliasearch from "algoliasearch"
+
+/* eslint-disable no-console */
+
+// this many articles are displayed un-collapsed, only score this many results
+const N_ARTICLES_DISPLAYED = 4
+
+const CONCURRENT_QUERIES = 10
+
+type QueryDataset = {
+    name: string
+    queries: Query[]
+}
+
+type Query = {
+    query: string
+    slugs: string[]
+}
+
+type ScoredQuery = {
+    query: string
+    expected: string[]
+    actual: string[]
+    precision: number
+}
+
+type SearchResults = {
+    name: string
+    scope: "articles" | "charts" | "all"
+    meanPrecision: number
+    numQueries: number
+}
+
+const QUERY_FILES = {
+    single: "synthetic-queries-single-2024-03-25.json",
+    multi: "synthetic-queries-2024-03-25.json",
+}
+
+const main = async (): Promise<void> => {
+    await evaluateAndPrint(QUERY_FILES.single)
+    await evaluateAndPrint(QUERY_FILES.multi)
+}
+
+const evaluateAndPrint = async (name: string): Promise<void> => {
+    const results = await evaluateArticleSearch(name)
+    console.log(JSON.stringify(results, null, 2))
+}
+
+const evaluateArticleSearch = async (name: string): Promise<SearchResults> => {
+    const ds = await fetchQueryDataset(name)
+    const indexName = getIndexName("pages")
+
+    // make a search client
+    const client = getClient()
+    const index = client.initIndex(indexName)
+
+    // run the evaluation
+    const results = await simulateQueries(index, ds.queries)
+    const meanPrecision =
+        results.map((r) => r.precision).reduce((a, b) => a + b) / results.length
+
+    // print the results to two decimal places
+    return {
+        name: ds.name,
+        scope: "articles",
+        meanPrecision: parseFloat(meanPrecision.toFixed(3)),
+        numQueries: ds.queries.length,
+    }
+}
+
+const getClient = (): any => {
+    const client = algoliasearch(ALGOLIA_ID, ALGOLIA_SEARCH_KEY)
+    return client
+}
+
+const fetchQueryDataset = async (name: string): Promise<QueryDataset> => {
+    const url: string = `${SEARCH_EVAL_URL}/${name}`
+    const resp = await fetch(url)
+    const jsonData = await resp.json()
+    return { name, queries: jsonData }
+}
+
+const simulateQuery = async (
+    index: any,
+    query: Query
+): Promise<ScoredQuery> => {
+    const { hits } = await index.search(query.query)
+    const actual = hits.map((h: any) => h.slug)
+    const precision = calculatePrecision(query.slugs, actual)
+    return { query: query.query, expected: query.slugs, actual, precision }
+}
+
+const calculatePrecision = (expected: string[], actual: string[]): number => {
+    const actualTruncated = actual.slice(0, N_ARTICLES_DISPLAYED)
+    const n = actualTruncated.length
+    if (n === 0) {
+        return 0
+    }
+    const correct = actualTruncated.filter((a) => expected.includes(a)).length
+    return correct / n
+}
+
+const simulateQueries = async (
+    index: any,
+    queries: Query[]
+): Promise<ScoredQuery[]> => {
+    // NOTE: should be a rate-limited version of:
+    //
+    // const scores = await Promise.all(
+    //     queries.map((query) => simulateQuery(index, query))
+    // )
+
+    let activeQueries = 0
+    let i = 0
+    const scores: ScoredQuery[] = []
+
+    const next = async () => {
+        if (i >= queries.length) return
+        const query = queries[i++]
+        activeQueries++
+        const score = await simulateQuery(index, query)
+        scores.push(score)
+        activeQueries--
+        if (i < queries.length) {
+            await next()
+        }
+    }
+
+    const promises = []
+    while (activeQueries < CONCURRENT_QUERIES && i < queries.length) {
+        promises.push(next())
+    }
+
+    await Promise.all(promises)
+
+    return scores
+}
+
+main()