From 35544f1ce9515003bfa3ecaa0f632edbe3b119fc Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Wed, 13 Mar 2024 11:15:12 +0100 Subject: [PATCH 1/8] feat(algolia): index explorer views to Algolia --- Makefile | 1 + baker/algolia/configureAlgolia.ts | 16 ++ baker/algolia/indexExplorerViewsToAlgolia.ts | 199 +++++++++++++++++++ baker/algolia/indexExplorersToAlgolia.ts | 2 +- explorer/ExplorerDecisionMatrix.ts | 12 +- site/search/searchTypes.ts | 2 + 6 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 baker/algolia/indexExplorerViewsToAlgolia.ts diff --git a/Makefile b/Makefile index 5c20e5cc1de..fc3bc5644e0 100644 --- a/Makefile +++ b/Makefile @@ -358,6 +358,7 @@ reindex: itsJustJavascript node --enable-source-maps itsJustJavascript/baker/algolia/indexToAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorersToAlgolia.js + node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorerViewsToAlgolia.js clean: rm -rf node_modules itsJustJavascript diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 21c4b8f79d5..399b24ce95e 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -143,6 +143,22 @@ export const configureAlgolia = async () => { disableTypoToleranceOnAttributes: ["text"], }) + const explorerViewsIndex = client.initIndex( + getIndexName(SearchIndexName.ExplorerViews) + ) + + await explorerViewsIndex.setSettings({ + ...baseSettings, + searchableAttributes: [ + "unordered(viewTitle)", + "unordered(viewSettings)", + ], + customRanking: ["desc(score)", "asc(viewIndexWithinExplorer)"], + attributeForDistinct: "viewTitleAndExplorerSlug", + distinct: true, + minWordSizefor1Typo: 6, + }) + const synonyms = [ ["owid", "our world in data"], ["kids", "children"], diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts new file mode 100644 index 00000000000..630283c7c41 --- /dev/null +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -0,0 +1,199 @@ +import * as db from "../../db/db.js" +import { ExplorerBlockGraphers } from "./indexExplorersToAlgolia.js" +import { DecisionMatrix } from "../../explorer/ExplorerDecisionMatrix.js" +import { tsvFormat } from "d3-dsv" +import { + ExplorerChoiceParams, + ExplorerControlType, +} from "../../explorer/ExplorerConstants.js" +import { GridBoolean } from "../../gridLang/GridLangConstants.js" +import { getAnalyticsPageviewsByUrlObj } from "../../db/model/Pageview.js" +import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" +import { getAlgoliaClient } from "./configureAlgolia.js" +import { getIndexName } from "../../site/search/searchClient.js" +import { SearchIndexName } from "../../site/search/searchTypes.js" + +interface ExplorerViewEntry { + viewTitle: string + viewSubtitle: string + viewSettings: string[] + viewQueryParams: string + + viewGrapherId?: number + + // Potential ranking criteria + viewIndexWithinExplorer: number + titleLength: number + numNonDefaultSettings: number + // viewViews_7d: number +} + +interface ExplorerViewEntryWithExplorerInfo extends ExplorerViewEntry { + explorerSlug: string + explorerTitle: string + explorerViews_7d: number + viewTitleAndExplorerSlug: string // used for deduplication: `viewTitle | explorerSlug` + + score: number + + objectID?: string +} + +const explorerChoiceToViewSettings = ( + choices: ExplorerChoiceParams, + decisionMatrix: DecisionMatrix +): string[] => { + return Object.entries(choices).map(([choiceName, choiceValue]) => { + const choiceControlType = + decisionMatrix.choiceNameToControlTypeMap.get(choiceName) + if (choiceControlType === ExplorerControlType.Checkbox) + return choiceValue === GridBoolean.true ? choiceName : "" + else return choiceValue + }) +} + +const getExplorerViewRecordsForExplorerSlug = async ( + trx: db.KnexReadonlyTransaction, + slug: string +): Promise => { + const explorerConfig = await trx + .table("explorers") + .select("config") + .where({ slug }) + .first() + .then((row) => JSON.parse(row.config) as any) + + const explorerGrapherBlock: ExplorerBlockGraphers = + explorerConfig.blocks.filter( + (block: any) => block.type === "graphers" + )[0] as ExplorerBlockGraphers + + if (explorerGrapherBlock === undefined) + throw new Error(`Explorer ${slug} has no grapher block`) + + // TODO: Maybe make DecisionMatrix accept JSON directly + const tsv = tsvFormat(explorerGrapherBlock.block) + const explorerDecisionMatrix = new DecisionMatrix(tsv) + + console.log( + `Processing explorer ${slug} (${explorerDecisionMatrix.numRows} rows)` + ) + + const defaultSettings = explorerDecisionMatrix.defaultSettings + + const records = explorerDecisionMatrix + .allDecisionsAsQueryParams() + .map((choice, i) => { + explorerDecisionMatrix.setValuesFromChoiceParams(choice) + + // Check which choices are non-default, i.e. are not the first available option in a dropdown/radio + const nonDefaultSettings = Object.entries( + explorerDecisionMatrix.availableChoiceOptions + ).filter(([choiceName, choiceOptions]) => { + // Keep only choices which are not the default, which is: + // - either the options marked as `default` in the decision matrix + // - or the first available option in the decision matrix + return ( + choiceOptions.length > 1 && + !(defaultSettings[choiceName] !== undefined + ? defaultSettings[choiceName] === choice[choiceName] + : choice[choiceName] === choiceOptions[0]) + ) + }) + + // TODO: Handle grapherId and fetch title/subtitle + // TODO: Handle indicator-based explorers + + const record: ExplorerViewEntry = { + viewTitle: explorerDecisionMatrix.selectedRow.title, + viewSubtitle: explorerDecisionMatrix.selectedRow.subtitle, + viewSettings: explorerChoiceToViewSettings( + choice, + explorerDecisionMatrix + ), + viewGrapherId: explorerDecisionMatrix.selectedRow.grapherId, + viewQueryParams: explorerDecisionMatrix.toString(), + + viewIndexWithinExplorer: i, + titleLength: explorerDecisionMatrix.selectedRow.title?.length, + numNonDefaultSettings: nonDefaultSettings.length, + } + return record + }) + + return records +} + +const getExplorerViewRecords = async ( + trx: db.KnexReadonlyTransaction +): Promise => { + const publishedExplorers = Object.values( + await db.getPublishedExplorersBySlug(trx) + ) + + const pageviews = await getAnalyticsPageviewsByUrlObj(trx) + + let records = [] as ExplorerViewEntryWithExplorerInfo[] + for (const explorerInfo of publishedExplorers) { + const explorerViewRecords = await getExplorerViewRecordsForExplorerSlug( + trx, + explorerInfo.slug + ) + + const explorerPageviews = + pageviews[`/explorers/${explorerInfo.slug}`]?.views_7d ?? 0 + records = records.concat( + explorerViewRecords.map( + (record, i): ExplorerViewEntryWithExplorerInfo => ({ + ...record, + explorerSlug: explorerInfo.slug, + explorerTitle: explorerInfo.title, + explorerViews_7d: explorerPageviews, + viewTitleAndExplorerSlug: `${record.viewTitle} | ${explorerInfo.slug}`, + // Scoring function + score: + explorerPageviews * 10 - + record.numNonDefaultSettings * 50 - + record.titleLength, + + objectID: `${explorerInfo.slug}-${i}`, + }) + ) + ) + } + + return records +} + +const indexExplorerViewsToAlgolia = async () => { + if (!ALGOLIA_INDEXING) return + + const client = getAlgoliaClient() + if (!client) { + console.error( + `Failed indexing explorer views (Algolia client not initialized)` + ) + return + } + + try { + const index = client.initIndex( + getIndexName(SearchIndexName.ExplorerViews) + ) + + const records = await db.knexReadonlyTransaction( + getExplorerViewRecords, + db.TransactionCloseMode.Close + ) + await index.replaceAllObjects(records) + } catch (e) { + console.log("Error indexing explorer views to Algolia:", e) + } +} + +process.on("unhandledRejection", (e) => { + console.error(e) + process.exit(1) +}) + +void indexExplorerViewsToAlgolia() diff --git a/baker/algolia/indexExplorersToAlgolia.ts b/baker/algolia/indexExplorersToAlgolia.ts index 2ac7580a1fd..9085cf1dab9 100644 --- a/baker/algolia/indexExplorersToAlgolia.ts +++ b/baker/algolia/indexExplorersToAlgolia.ts @@ -22,7 +22,7 @@ type ExplorerBlockColumns = { block: { name: string; additionalInfo?: string }[] } -type ExplorerBlockGraphers = { +export type ExplorerBlockGraphers = { type: "graphers" block: { title?: string diff --git a/explorer/ExplorerDecisionMatrix.ts b/explorer/ExplorerDecisionMatrix.ts index 0b0ff8b802e..2f2f454df5b 100644 --- a/explorer/ExplorerDecisionMatrix.ts +++ b/explorer/ExplorerDecisionMatrix.ts @@ -86,7 +86,7 @@ export class DecisionMatrix { table: CoreTable @observable currentParams: ExplorerChoiceParams = {} constructor(delimited: string, hash = "") { - this.choices = makeChoicesMap(delimited) + this.choiceNameToControlTypeMap = makeChoicesMap(delimited) this.table = new CoreTable(parseDelimited(dropColumnTypes(delimited)), [ // todo: remove col def? { @@ -141,7 +141,7 @@ export class DecisionMatrix { ) } - private choices: Map + choiceNameToControlTypeMap: Map hash: string toConstrainedOptions(): ExplorerChoiceParams { @@ -243,7 +243,7 @@ export class DecisionMatrix { } @computed private get choiceNames(): ChoiceName[] { - return Array.from(this.choices.keys()) + return Array.from(this.choiceNameToControlTypeMap.keys()) } @computed private get allChoiceOptions(): ChoiceMap { @@ -256,7 +256,7 @@ export class DecisionMatrix { return choiceMap } - @computed private get availableChoiceOptions(): ChoiceMap { + @computed get availableChoiceOptions(): ChoiceMap { const result: ChoiceMap = {} this.choiceNames.forEach((choiceName) => { result[choiceName] = this.allChoiceOptions[choiceName].filter( @@ -317,7 +317,7 @@ export class DecisionMatrix { } // The first row with defaultView column value of "true" determines the default view to use - private get defaultSettings() { + get defaultSettings() { const hits = this.rowsWith({ [GrapherGrammar.defaultView.keyword]: "true", }) @@ -373,7 +373,7 @@ export class DecisionMatrix { constrainedOptions ) ) - const type = this.choices.get(title)! + const type = this.choiceNameToControlTypeMap.get(title)! return { title, diff --git a/site/search/searchTypes.ts b/site/search/searchTypes.ts index bb23b325138..1491bb6c190 100644 --- a/site/search/searchTypes.ts +++ b/site/search/searchTypes.ts @@ -68,6 +68,7 @@ export type IChartHit = Hit & ChartRecord export enum SearchIndexName { Explorers = "explorers", + ExplorerViews = "explorer-views", Charts = "charts", Pages = "pages", } @@ -85,4 +86,5 @@ export const indexNameToSubdirectoryMap: Record = { [SearchIndexName.Pages]: "", [SearchIndexName.Charts]: "/grapher", [SearchIndexName.Explorers]: "/explorers", + [SearchIndexName.ExplorerViews]: "/explorers", } From 612651ec26bef89a1b0df487dfbfcf552cc085e2 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 14 Mar 2024 18:44:37 +0100 Subject: [PATCH 2/8] enhance(algolia): explorer views based on grapherIds are handled --- baker/algolia/indexExplorerViewsToAlgolia.ts | 45 ++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index 630283c7c41..e9c9f89b3d5 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -12,6 +12,7 @@ import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" import { getAlgoliaClient } from "./configureAlgolia.js" import { getIndexName } from "../../site/search/searchClient.js" import { SearchIndexName } from "../../site/search/searchTypes.js" +import { keyBy } from "lodash" interface ExplorerViewEntry { viewTitle: string @@ -39,6 +40,9 @@ interface ExplorerViewEntryWithExplorerInfo extends ExplorerViewEntry { objectID?: string } +// Creates a search-ready string from a choice. +// Special handling is pretty much only necessary for checkboxes: If they are not ticked, then their name is not included. +// Imagine a "Per capita" checkbox, for example. If it's not ticked, then we don't want searches for "per capita" to wrongfully match it. const explorerChoiceToViewSettings = ( choices: ExplorerChoiceParams, decisionMatrix: DecisionMatrix @@ -101,9 +105,6 @@ const getExplorerViewRecordsForExplorerSlug = async ( ) }) - // TODO: Handle grapherId and fetch title/subtitle - // TODO: Handle indicator-based explorers - const record: ExplorerViewEntry = { viewTitle: explorerDecisionMatrix.selectedRow.title, viewSubtitle: explorerDecisionMatrix.selectedRow.subtitle, @@ -121,6 +122,44 @@ const getExplorerViewRecordsForExplorerSlug = async ( return record }) + // Enrich `grapherId`-powered views with title/subtitle + const grapherIds = records + .filter((record) => record.viewGrapherId !== undefined) + .map((record) => record.viewGrapherId as number) + + if (grapherIds.length) { + console.log( + `Fetching grapher info from ${grapherIds.length} graphers for explorer ${slug}` + ) + const grapherIdToTitle = await trx + .table("charts") + .select( + "id", + trx.raw("config->>'$.title' as title"), + trx.raw("config->>'$.subtitle' as subtitle") + ) + .whereIn("id", grapherIds) + .andWhereRaw("config->>'$.isPublished' = 'true'") + .then((rows) => keyBy(rows, "id")) + + for (const record of records) { + if (record.viewGrapherId !== undefined) { + const grapherInfo = grapherIdToTitle[record.viewGrapherId] + if (grapherInfo === undefined) { + console.warn( + `Grapher id ${record.viewGrapherId} not found for explorer ${slug}` + ) + continue + } + record.viewTitle = grapherInfo.title + record.viewSubtitle = grapherInfo.subtitle + record.titleLength = grapherInfo.title?.length + } + } + } + + // TODO: Handle indicator-based explorers + return records } From b149187cd7c0a5878e01d8d5382e3418f834cca2 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Fri, 29 Mar 2024 12:29:58 +0100 Subject: [PATCH 3/8] enhance(algolia): index `viewTitleIndexWithinExplorer`, `numViewsWithinExplorer` --- baker/algolia/indexExplorerViewsToAlgolia.ts | 73 +++++++++++++++----- 1 file changed, 54 insertions(+), 19 deletions(-) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index e9c9f89b3d5..a9e1ce9e5c6 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -12,7 +12,7 @@ import { ALGOLIA_INDEXING } from "../../settings/serverSettings.js" import { getAlgoliaClient } from "./configureAlgolia.js" import { getIndexName } from "../../site/search/searchClient.js" import { SearchIndexName } from "../../site/search/searchTypes.js" -import { keyBy } from "lodash" +import { groupBy, keyBy, orderBy } from "lodash" interface ExplorerViewEntry { viewTitle: string @@ -22,6 +22,13 @@ interface ExplorerViewEntry { viewGrapherId?: number + /** + * We often have several views with the same title within an explorer, e.g. "Population". + * In order to only display _one_ of these views in search results, we need a way to demote duplicates. + * This attribute is used for that: The highest-scored such view will be given a value of 0, the second-highest 1, etc. + */ + viewTitleIndexWithinExplorer: number + // Potential ranking criteria viewIndexWithinExplorer: number titleLength: number @@ -34,6 +41,7 @@ interface ExplorerViewEntryWithExplorerInfo extends ExplorerViewEntry { explorerTitle: string explorerViews_7d: number viewTitleAndExplorerSlug: string // used for deduplication: `viewTitle | explorerSlug` + numViewsWithinExplorer: number score: number @@ -56,6 +64,11 @@ const explorerChoiceToViewSettings = ( }) } +const computeScore = (record: Partial) => + (record.explorerViews_7d ?? 0) * 10 - + (record.numNonDefaultSettings ?? 0) * 50 - + (record.titleLength ?? 0) + const getExplorerViewRecordsForExplorerSlug = async ( trx: db.KnexReadonlyTransaction, slug: string @@ -105,7 +118,10 @@ const getExplorerViewRecordsForExplorerSlug = async ( ) }) - const record: ExplorerViewEntry = { + const record: Omit< + ExplorerViewEntry, + "viewTitleIndexWithinExplorer" + > = { viewTitle: explorerDecisionMatrix.selectedRow.title, viewSubtitle: explorerDecisionMatrix.selectedRow.subtitle, viewSettings: explorerChoiceToViewSettings( @@ -158,9 +174,28 @@ const getExplorerViewRecordsForExplorerSlug = async ( } } + // Compute viewTitleIndexWithinExplorer: + // First, sort by score descending (ignoring views_7d, which is not relevant _within_ an explorer). + // Then, group by viewTitle. + // Finally, ungroup again, and keep track of the index of each element within the group. + const recordsSortedByScore = orderBy( + records, + (record) => computeScore(record), + "desc" + ) + const recordsGroupedByViewTitle = groupBy(recordsSortedByScore, "viewTitle") + const recordsWithIndexWithinExplorer = Object.values( + recordsGroupedByViewTitle + ).flatMap((recordsGroup) => + recordsGroup.map((record, i) => ({ + ...record, + viewTitleIndexWithinExplorer: i, + })) + ) + // TODO: Handle indicator-based explorers - return records + return recordsWithIndexWithinExplorer } const getExplorerViewRecords = async ( @@ -181,23 +216,23 @@ const getExplorerViewRecords = async ( const explorerPageviews = pageviews[`/explorers/${explorerInfo.slug}`]?.views_7d ?? 0 + const unscoredRecords = explorerViewRecords.map( + (record, i): Omit => ({ + ...record, + explorerSlug: explorerInfo.slug, + explorerTitle: explorerInfo.title, + explorerViews_7d: explorerPageviews, + viewTitleAndExplorerSlug: `${record.viewTitle} | ${explorerInfo.slug}`, + numViewsWithinExplorer: explorerViewRecords.length, + + objectID: `${explorerInfo.slug}-${i}`, + }) + ) records = records.concat( - explorerViewRecords.map( - (record, i): ExplorerViewEntryWithExplorerInfo => ({ - ...record, - explorerSlug: explorerInfo.slug, - explorerTitle: explorerInfo.title, - explorerViews_7d: explorerPageviews, - viewTitleAndExplorerSlug: `${record.viewTitle} | ${explorerInfo.slug}`, - // Scoring function - score: - explorerPageviews * 10 - - record.numNonDefaultSettings * 50 - - record.titleLength, - - objectID: `${explorerInfo.slug}-${i}`, - }) - ) + unscoredRecords.map((record) => ({ + ...record, + score: computeScore(record), + })) ) } From d0e36411d97b1740a2ebf1490f6f07dc54e258e6 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Fri, 29 Mar 2024 12:51:22 +0100 Subject: [PATCH 4/8] enhance(algolia): rank by `viewTitleIndexWithinExplorer` --- baker/algolia/configureAlgolia.ts | 13 ++++++++++--- baker/algolia/indexExplorerViewsToAlgolia.ts | 9 ++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 399b24ce95e..57c53e4044d 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -152,10 +152,17 @@ export const configureAlgolia = async () => { searchableAttributes: [ "unordered(viewTitle)", "unordered(viewSettings)", + "unordered(explorerTitle)", ], - customRanking: ["desc(score)", "asc(viewIndexWithinExplorer)"], - attributeForDistinct: "viewTitleAndExplorerSlug", - distinct: true, + customRanking: [ + // For multiple explorer views with the same title, we want to avoid surfacing duplicates. + // So, rank a result with viewTitleIndexWithinExplorer=0 way more highly than one with 1, 2, etc. + "asc(viewTitleIndexWithinExplorer)", + "desc(score)", + "asc(viewIndexWithinExplorer)", + ], + attributeForDistinct: "explorerSlug", + distinct: 4, minWordSizefor1Typo: 6, }) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index a9e1ce9e5c6..51e398a4856 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -64,10 +64,13 @@ const explorerChoiceToViewSettings = ( }) } -const computeScore = (record: Partial) => +const computeScore = ( + record: Omit & + Partial +) => (record.explorerViews_7d ?? 0) * 10 - - (record.numNonDefaultSettings ?? 0) * 50 - - (record.titleLength ?? 0) + record.numNonDefaultSettings * 50 - + record.titleLength const getExplorerViewRecordsForExplorerSlug = async ( trx: db.KnexReadonlyTransaction, From 81c4a494346f7bda71353a9c04881e4164e3c773 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 2 Apr 2024 12:48:57 +0200 Subject: [PATCH 5/8] enhance(algolia): index explorer subtitles --- baker/algolia/indexExplorerViewsToAlgolia.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index 51e398a4856..0b73a1951de 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -39,6 +39,7 @@ interface ExplorerViewEntry { interface ExplorerViewEntryWithExplorerInfo extends ExplorerViewEntry { explorerSlug: string explorerTitle: string + explorerSubtitle: string explorerViews_7d: number viewTitleAndExplorerSlug: string // used for deduplication: `viewTitle | explorerSlug` numViewsWithinExplorer: number @@ -224,6 +225,7 @@ const getExplorerViewRecords = async ( ...record, explorerSlug: explorerInfo.slug, explorerTitle: explorerInfo.title, + explorerSubtitle: explorerInfo.subtitle, explorerViews_7d: explorerPageviews, viewTitleAndExplorerSlug: `${record.viewTitle} | ${explorerInfo.slug}`, numViewsWithinExplorer: explorerViewRecords.length, From bbb9e8b52dfdd6980dee12d962baeda6512f4d45 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 2 Apr 2024 13:39:18 +0200 Subject: [PATCH 6/8] chore(algolia): strip out markdown from view subtitles --- baker/algolia/indexExplorerViewsToAlgolia.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/baker/algolia/indexExplorerViewsToAlgolia.ts b/baker/algolia/indexExplorerViewsToAlgolia.ts index 0b73a1951de..b5288e3d27d 100644 --- a/baker/algolia/indexExplorerViewsToAlgolia.ts +++ b/baker/algolia/indexExplorerViewsToAlgolia.ts @@ -13,6 +13,7 @@ import { getAlgoliaClient } from "./configureAlgolia.js" import { getIndexName } from "../../site/search/searchClient.js" import { SearchIndexName } from "../../site/search/searchTypes.js" import { groupBy, keyBy, orderBy } from "lodash" +import { MarkdownTextWrap } from "@ourworldindata/components" interface ExplorerViewEntry { viewTitle: string @@ -178,6 +179,16 @@ const getExplorerViewRecordsForExplorerSlug = async ( } } + // Remove Markdown from viewSubtitle; do this after fetching grapher info above, as it might also contain Markdown + records.forEach((record) => { + if (record.viewSubtitle) { + record.viewSubtitle = new MarkdownTextWrap({ + text: record.viewSubtitle, + fontSize: 10, // doesn't matter, but is a mandatory field + }).plaintext + } + }) + // Compute viewTitleIndexWithinExplorer: // First, sort by score descending (ignoring views_7d, which is not relevant _within_ an explorer). // Then, group by viewTitle. From 1cc6b11f2d97157379a45fa046d0791cfc899db5 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 2 Apr 2024 14:27:20 +0200 Subject: [PATCH 7/8] enhance(algolia): set synonyms on `explorer-views` index --- baker/algolia/configureAlgolia.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 57c53e4044d..11e88f827d5 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -330,6 +330,9 @@ export const configureAlgolia = async () => { await explorersIndex.saveSynonyms(algoliaSynonyms, { replaceExistingSynonyms: true, }) + await explorerViewsIndex.saveSynonyms(algoliaSynonyms, { + replaceExistingSynonyms: true, + }) if (TOPICS_CONTENT_GRAPH) { const graphIndex = client.initIndex(CONTENT_GRAPH_ALGOLIA_INDEX) From 7dbbc49a4e34e3cb561eff0a28f2635f07f5e1cc Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Tue, 9 Apr 2024 14:21:28 +0200 Subject: [PATCH 8/8] enhance(algolia): search explorer title first --- baker/algolia/configureAlgolia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baker/algolia/configureAlgolia.ts b/baker/algolia/configureAlgolia.ts index 11e88f827d5..2a34c29bdd4 100644 --- a/baker/algolia/configureAlgolia.ts +++ b/baker/algolia/configureAlgolia.ts @@ -150,9 +150,9 @@ export const configureAlgolia = async () => { await explorerViewsIndex.setSettings({ ...baseSettings, searchableAttributes: [ + "unordered(explorerTitle)", "unordered(viewTitle)", "unordered(viewSettings)", - "unordered(explorerTitle)", ], customRanking: [ // For multiple explorer views with the same title, we want to avoid surfacing duplicates.