feat: Use local search only for peristent sessions

We now rely on the checkbox on login to determine whether or not the local search should be used. If the user enable this checkbox, it means he trusts this device and accept to store local data on it. Otherwise, we do not store any private data, and directly query the stack to build the search index. Note the update mechanism is the same and rely on realtime. The difference is that we do not replicate the document for non local search.
cozy · Nov 8, 2024 · dc2b38e · dc2b38e
1 parent 066163a
commit dc2b38e
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 85 deletions.
diff --git a/packages/cozy-dataproxy-lib/package.json b/packages/cozy-dataproxy-lib/package.json
@@ -63,7 +63,8 @@
     "prepublishOnly": "yarn build",
     "test": "jest --config=./tests/jest.config.js",
     "test:watch": "yarn test --watchAll",
-    "start": "yarn build:watch"
+    "start": "yarn build:watch",
+    "lint": "cd .. && yarn eslint --ext js,jsx,ts packages/cozy-dataproxy-lib"
   },
   "types": "dist/index.d.ts"
 }
diff --git a/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts b/packages/cozy-dataproxy-lib/src/search/SearchEngine.ts
@@ -1,6 +1,6 @@
 import FlexSearch from 'flexsearch'
 
-import CozyClient, { Q } from 'cozy-client'
+import CozyClient from 'cozy-client'
 import Minilog from 'cozy-minilog'
 import { RealtimePlugin } from 'cozy-realtime'
 
@@ -11,13 +11,14 @@ import {
   CONTACTS_DOCTYPE,
   DOCTYPE_ORDER,
   LIMIT_DOCTYPE_SEARCH,
-  SearchedDoctype
+  SearchedDoctype,
+  SEARCHABLE_DOCTYPES
 } from './consts'
 import { getPouchLink } from './helpers/client'
 import { getSearchEncoder } from './helpers/getSearchEncoder'
 import { addFilePaths, shouldKeepFile } from './helpers/normalizeFile'
 import { normalizeSearchResult } from './helpers/normalizeSearchResult'
-import { queryFilesForSearch, queryAllContacts, queryAllApps } from './queries'
+import { queryAllDocs, queryFilesForSearch } from './queries'
 import {
   CozyDoc,
   RawSearchResult,
@@ -41,27 +42,44 @@ export class SearchEngine {
   client: CozyClient
   searchIndexes: SearchIndexes
   debouncedReplication: () => void
+  isLocalSearch: boolean
 
   constructor(client: CozyClient) {
     this.client = client
     this.searchIndexes = {} as SearchIndexes
 
-    this.indexOnChanges()
+    this.isLocalSearch = !!getPouchLink(this.client)
+    log.info('Use local data on trusted device : ', this.isLocalSearch)
+
     this.debouncedReplication = (): void => {
       const pouchLink = getPouchLink(client)
       if (pouchLink) {
         pouchLink.startReplicationWithDebounce()
       }
     }
+    void this.indexDocuments()
   }
 
-  indexOnChanges(): void {
+  async indexDocuments(): Promise<void> {
     if (!this.client) {
       return
     }
+    if (!this.isLocalSearch) {
+      // In case of non-local search, force the indexing for all doctypes
+      // For local search, this will be done automatically after initial replication
+      for (const doctype of SEARCHABLE_DOCTYPES) {
+        this.searchIndexes[doctype] = await this.indexDocsForSearch(
+          doctype as keyof typeof SEARCH_SCHEMA
+        )
+      }
+    }
+
     this.client.on('pouchlink:doctypesync:end', async (doctype: string) => {
       if (isSearchedDoctype(doctype)) {
-        await this.indexDocsForSearch(doctype as keyof typeof SEARCH_SCHEMA)
+        // Index doctype after initial replication
+        this.searchIndexes[doctype] = await this.indexDocsForSearch(
+          doctype as keyof typeof SEARCH_SCHEMA
+        )
       }
     })
     this.client.on('pouchlink:sync:start', () => {
@@ -70,6 +88,7 @@ export class SearchEngine {
     this.client.on('pouchlink:sync:end', () => {
       log.debug('Ended pouch replication')
     })
+
     this.client.on('login', () => {
       // Ensure login is done before plugin register
       this.client.registerPlugin(RealtimePlugin, {})
@@ -105,7 +124,9 @@ export class SearchEngine {
     log.debug('[REALTIME] index doc after update : ', doc)
     this.addDocToIndex(searchIndex.index, doc)
 
-    this.debouncedReplication()
+    if (this.isLocalSearch) {
+      this.debouncedReplication()
+    }
   }
 
   handleDeletedDoc(doc: CozyDoc): void {
@@ -121,13 +142,17 @@ export class SearchEngine {
     log.debug('[REALTIME] remove doc from index after update : ', doc)
     this.searchIndexes[doctype].index.remove(doc._id!)
 
-    this.debouncedReplication()
+    if (this.isLocalSearch) {
+      this.debouncedReplication()
+    }
   }
 
   buildSearchIndex(
     doctype: keyof typeof SEARCH_SCHEMA,
     docs: CozyDoc[]
   ): FlexSearch.Document<CozyDoc, true> {
+    const startTimeIndex = performance.now()
+
     const fieldsToIndex = SEARCH_SCHEMA[doctype]
 
     const flexsearchIndex = new FlexSearch.Document<CozyDoc, true>({
@@ -146,6 +171,12 @@ export class SearchEngine {
       this.addDocToIndex(flexsearchIndex, doc)
     }
 
+    const endTimeIndex = performance.now()
+    log.debug(
+      `Create ${doctype} index took ${(endTimeIndex - startTimeIndex).toFixed(
+        2
+      )} ms`
+    )
     return flexsearchIndex
   }
 
@@ -165,51 +196,60 @@ export class SearchEngine {
     return true
   }
 
-  async indexDocsForSearch(
-    doctype: keyof typeof SEARCH_SCHEMA
-  ): Promise<SearchIndex | null> {
-    const searchIndex = this.searchIndexes[doctype]
-    const pouchLink = getPouchLink(this.client)
+  async getLocalLastSeq(doctype: keyof typeof SEARCH_SCHEMA): Promise<number> {
+    if (this.isLocalSearch) {
+      const pouchLink = getPouchLink(this.client)
+      const info = pouchLink ? await pouchLink.getDbInfo(doctype) : null
+      return info?.update_seq || 0
+    }
+    return -1
+  }
 
-    if (!pouchLink) {
-      return null
+  async queryLocalOrRemoteDocs(
+    doctype: keyof typeof SEARCH_SCHEMA
+  ): Promise<CozyDoc[]> {
+    let docs = []
+    const startTimeQ = performance.now()
+
+    if (!this.isLocalSearch && doctype === FILES_DOCTYPE) {
+      // Special case for stack's files
+      docs = await queryFilesForSearch(this.client)
+    } else {
+      docs = await queryAllDocs(this.client, doctype)
     }
+    const endTimeQ = performance.now()
+    log.debug(
+      `Query ${docs.length} ${doctype} took ${(endTimeQ - startTimeQ).toFixed(
+        2
+      )} ms`
+    )
+    return docs
+  }
 
-    if (!searchIndex) {
-      // First creation of search index
-      const startTimeQ = performance.now()
-      const docs = await this.client.queryAll<CozyDoc[]>(
-        Q(doctype).limitBy(null)
-      )
-      const endTimeQ = performance.now()
-      log.debug(
-        `Query ${docs.length} docs doctype ${doctype} took ${(
-          endTimeQ - startTimeQ
-        ).toFixed(2)} ms`
-      )
-
-      const startTimeIndex = performance.now()
-      const index = this.buildSearchIndex(doctype, docs)
-      const endTimeIndex = performance.now()
-      log.debug(
-        `Create ${doctype} index took ${(endTimeIndex - startTimeIndex).toFixed(
-          2
-        )} ms`
-      )
-      const info = await pouchLink.getDbInfo(doctype)
-
-      this.searchIndexes[doctype] = {
-        index,
-        lastSeq: info?.update_seq,
-        lastUpdated: new Date().toISOString()
-      }
-      return this.searchIndexes[doctype]
+  async initialIndexation(
+    doctype: keyof typeof SEARCH_SCHEMA
+  ): Promise<SearchIndex> {
+    const docs = await this.queryLocalOrRemoteDocs(doctype)
+    const index = this.buildSearchIndex(doctype, docs)
+    const lastSeq = await this.getLocalLastSeq(doctype)
+
+    this.searchIndexes[doctype] = {
+      index,
+      lastSeq,
+      lastUpdated: new Date().toISOString()
     }
+    return this.searchIndexes[doctype]
+  }
 
-    // Incremental index update
-    // At this point, the search index are supposed to be already up-to-date,
-    // thanks to the realtime.
-    // However, we check it is actually the case for safety, and update the lastSeq
+  async incrementalIndexation(
+    doctype: keyof typeof SEARCH_SCHEMA,
+    searchIndex: SearchIndex
+  ): Promise<SearchIndex> {
+    const pouchLink = getPouchLink(this.client)
+    if (!this.isLocalSearch || !pouchLink) {
+      // No need to handle incremental indexation for non-local search: it is already done through realtime
+      return searchIndex
+    }
     const lastSeq = searchIndex.lastSeq || 0
     const changes = await pouchLink.getChanges(doctype, {
       include_docs: true,
@@ -230,34 +270,20 @@ export class SearchEngine {
     return searchIndex
   }
 
-  initIndexesFromStack = async (): Promise<SearchIndexes> => {
-    log.debug('Initializing indexes')
-
-    const files = await queryFilesForSearch(this.client)
-    const filesIndex = this.buildSearchIndex('io.cozy.files', files)
-
-    const contacts = await queryAllContacts(this.client)
-    const contactsIndex = this.buildSearchIndex('io.cozy.contacts', contacts)
-
-    const apps = await queryAllApps(this.client)
-    const appsIndex = this.buildSearchIndex('io.cozy.apps', apps)
-
-    log.debug('Finished initializing indexes')
-    const currentDate = new Date().toISOString()
-    this.searchIndexes = {
-      [FILES_DOCTYPE]: {
-        index: filesIndex,
-        lastSeq: 0,
-        lastUpdated: currentDate
-      },
-      [CONTACTS_DOCTYPE]: {
-        index: contactsIndex,
-        lastSeq: 0,
-        lastUpdated: currentDate
-      },
-      [APPS_DOCTYPE]: { index: appsIndex, lastSeq: 0, lastUpdated: currentDate }
+  async indexDocsForSearch(
+    doctype: keyof typeof SEARCH_SCHEMA
+  ): Promise<SearchIndex> {
+    const searchIndex = this.searchIndexes[doctype]
+
+    if (!searchIndex) {
+      // First creation of search index
+      return this.initialIndexation(doctype)
     }
-    return this.searchIndexes
+
+    // At this point, the search index is supposed to be already up-to-date,
+    // thanks to the realtime.
+    // However, we check if it is actually the case for safety, and update the lastSeq
+    return this.incrementalIndexation(doctype, searchIndex)
   }
 
   search(query: string): SearchResult[] {
@@ -273,7 +299,10 @@ export class SearchEngine {
     const results = this.limitSearchResults(sortedResults)
 
     const normResults: SearchResult[] = []
-    const completedResults = addFilePaths(this.client, results)
+    // Special case for local files: the path is missing
+    const completedResults = this.isLocalSearch
+      ? addFilePaths(this.client, results)
+      : results
     for (const res of completedResults) {
       const normalizedRes = normalizeSearchResult(this.client, res, query)
       normResults.push(normalizedRes)

diff --git a/packages/cozy-dataproxy-lib/src/search/consts.ts b/packages/cozy-dataproxy-lib/src/search/consts.ts
@@ -1,4 +1,4 @@
-const SEARCHABLE_DOCTYPES = [
+export const SEARCHABLE_DOCTYPES = [
   'io.cozy.files',
   'io.cozy.contacts',
   'io.cozy.apps'

diff --git a/packages/cozy-dataproxy-lib/src/search/queries/index.ts b/packages/cozy-dataproxy-lib/src/search/queries/index.ts
@@ -1,7 +1,7 @@
 import CozyClient, { Q } from 'cozy-client'
 import { IOCozyFile } from 'cozy-client/types/types'
 
-import { CONTACTS_DOCTYPE, APPS_DOCTYPE, TYPE_DIRECTORY } from '../consts'
+import { TYPE_DIRECTORY } from '../consts'
 import {
   normalizeFileWithFolders,
   shouldKeepFile
@@ -37,10 +37,9 @@ export const queryFilesForSearch = async (
   return normalizedFiles
 }
 
-export const queryAllContacts = (client: CozyClient): Promise<CozyDoc[]> => {
-  return client.queryAll(Q(CONTACTS_DOCTYPE).limitBy(1000))
-}
-
-export const queryAllApps = (client: CozyClient): Promise<CozyDoc[]> => {
-  return client.queryAll(Q(APPS_DOCTYPE).limitBy(1000))
+export const queryAllDocs = async (
+  client: CozyClient,
+  doctype: string
+): Promise<CozyDoc[]> => {
+  return client.queryAll<CozyDoc[]>(Q(doctype).limitBy(null))
 }