Skip to content

Commit

Permalink
feat: Handle search on both path and name
Browse files Browse the repository at this point in the history 
Say you have a "Foo" directory with a "bar.txt" in it. Let's assume this
bar.txt is duplicated in many locations. You want to search specifically
for it by typing "Foo bar".
It wasn't possible before because of how flexsearch works: it searches
by indexed attributes, here the `path` and the `name`. In this example,
"Foo bar" will not match the name, neither the path.

As a workaround, we now force the path computing at indexing time for
all files. This path is not persisted in PouchDB, as the stack does not
store it for files (but it does for directories).
Note that even though the stack returns the file's path when we query
it, we need to add the file name, as it does not include it.
  • Loading branch information
@paultranvan
paultranvan committed Nov 8, 2024
1 parent a9e2ad5 commit 50bcdf3
Show file tree
Hide file tree
Showing 4 changed files with 180 additions and 68 deletions.
39 changes: 24 additions & 15 deletions packages/cozy-dataproxy-lib/src/search/SearchEngine.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@ import {
} from './consts'
import { getPouchLink } from './helpers/client'
import { getSearchEncoder } from './helpers/getSearchEncoder'
import { addFilePaths, shouldKeepFile } from './helpers/normalizeFile'
import {
addFilePaths,
computeFileFullpath,
shouldKeepFile
} from './helpers/normalizeFile'
import { normalizeSearchResult } from './helpers/normalizeSearchResult'
import { queryAllDocs, queryFilesForSearch } from './queries'
import {
Expand Down Expand Up @@ -131,8 +135,8 @@ export class SearchEngine {
// No index yet: it will be done by querying the local db after first replication
return
}
log.debug('[REALTIME] index doc after update : ', doc)
this.addDocToIndex(searchIndex.index, doc)
log.debug('[REALTIME] Update doc from index after update : ', doc)
void this.addDocToIndex(searchIndex.index, doc)

if (this.isLocalSearch) {
this.debouncedReplication()
Expand All @@ -149,7 +153,7 @@ export class SearchEngine {
// No index yet: it will be done by querying the local db after first replication
return
}
log.debug('[REALTIME] remove doc from index after update : ', doc)
log.debug('[REALTIME] Remove doc from index after update : ', doc)
this.searchIndexes[doctype].index.remove(doc._id!)

if (this.isLocalSearch) {
Expand Down Expand Up @@ -177,8 +181,12 @@ export class SearchEngine {
}
})

for (const doc of docs) {
this.addDocToIndex(flexsearchIndex, doc)
// There is no persisted path for files: we must add it
const completedDocs = this.isLocalSearch
? addFilePaths(this.client, docs)
: docs
for (const doc of completedDocs) {
void this.addDocToIndex(flexsearchIndex, doc)
}

const endTimeIndex = performance.now()
Expand All @@ -190,12 +198,17 @@ export class SearchEngine {
return flexsearchIndex
}

addDocToIndex(
async addDocToIndex(
flexsearchIndex: FlexSearch.Document<CozyDoc, true>,
doc: CozyDoc
): void {
): Promise<void> {
if (this.shouldIndexDoc(doc)) {
flexsearchIndex.add(doc)
let docToIndex = doc
if (isIOCozyFile(doc)) {
// Add path for files
docToIndex = await computeFileFullpath(this.client, doc)
}
flexsearchIndex.add(docToIndex)
}
}

Expand Down Expand Up @@ -271,7 +284,7 @@ export class SearchEngine {
searchIndex.index.remove(change.id)
} else {
const normalizedDoc = { ...change.doc, _type: doctype } as CozyDoc
this.addDocToIndex(searchIndex.index, normalizedDoc)
void this.addDocToIndex(searchIndex.index, normalizedDoc)
}
}

Expand Down Expand Up @@ -309,11 +322,7 @@ export class SearchEngine {
const results = this.limitSearchResults(sortedResults)

const normResults: SearchResult[] = []
// Special case for local files: the path is missing
const completedResults = this.isLocalSearch
? addFilePaths(this.client, results)
: results
for (const res of completedResults) {
for (const res of results) {
const normalizedRes = normalizeSearchResult(this.client, res, query)
normResults.push(normalizedRes)
}
Expand Down
130 changes: 90 additions & 40 deletions packages/cozy-dataproxy-lib/src/search/helpers/normalizeFile.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,18 @@ import { IOCozyFile } from 'cozy-client/types/types'

import {
addFilePaths,
computeFileFullpath,
normalizeFileWithFolders,
shouldKeepFile
} from './normalizeFile'
import { RawSearchResult } from '../types'
import { queryDocById } from '../queries'
import { CozyDoc } from '../types'

jest.mock('../queries', () => ({
queryDocById: jest.fn()
}))

const client = new CozyClient()

describe('normalizeFileWithFolders', () => {
test('should get path for directories', () => {
Expand Down Expand Up @@ -107,40 +115,31 @@ describe('addFilePaths', () => {
])
} as unknown as CozyClient

const results = [
const docs = [
{
doctype: 'io.cozy.files',
doc: {
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID'
}
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID'
}
] as unknown as RawSearchResult[]
] as CozyDoc[]

const result = addFilePaths(client, results)
const result = addFilePaths(client, docs)

expect(result).toStrictEqual([
{
doc: {
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID',
path: 'SOME/PARENT/PATH'
},
doctype: 'io.cozy.files'
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID',
path: 'SOME/PARENT/PATH'
}
])
})

test(`should handle no files in results`, () => {
const client = {} as unknown as CozyClient

const results = [] as unknown as RawSearchResult[]

const result = addFilePaths(client, results)
const result = addFilePaths(client, [])

expect(result).toStrictEqual([])
})
Expand All @@ -150,29 +149,23 @@ describe('addFilePaths', () => {
getCollectionFromState: jest.fn().mockReturnValue([])
} as unknown as CozyClient

const results = [
const docs = [
{
doctype: 'io.cozy.files',
doc: {
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID'
}
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID'
}
] as unknown as RawSearchResult[]
] as CozyDoc[]

const result = addFilePaths(client, results)
const result = addFilePaths(client, docs)

expect(result).toStrictEqual([
{
doc: {
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID'
},
doctype: 'io.cozy.files'
_id: 'SOME_FILE_ID',
_type: 'io.cozy.files',
type: 'file',
dir_id: 'SOME_DIR_ID'
}
])
})
Expand Down Expand Up @@ -243,3 +236,60 @@ describe('shouldKeepFile', () => {
expect(result).toBe(false)
})
})

describe('computeFileFullpath', () => {
const dir = {
_id: '123',
_type: 'io.cozy.files',
type: 'directory',
dir_id: 'ROOT',
name: 'MYDIR',
path: 'ROOT/MYDIR'
} as IOCozyFile
const fileWithFullpath = {
_id: '456',
_type: 'io.cozy.files',
type: 'file',
dir_id: '123',
name: 'file1',
path: 'ROOT/MYDIR/file1'
} as IOCozyFile
const fileWithStackPath = {
_id: '789',
_type: 'io.cozy.files',
type: 'file',
dir_id: '123',
name: 'file2',
path: 'ROOT/MYDIR'
} as IOCozyFile
const filewithNoPath = {
_id: '000',
_type: 'io.cozy.files',
type: 'file',
dir_id: '123',
name: 'file3'
} as IOCozyFile
it('should handle directory', async () => {
const res = await computeFileFullpath(client, dir)
expect(res).toEqual(dir)
})

it('should handle file with complete path', async () => {
const res = await computeFileFullpath(client, fileWithFullpath)
expect(res).toEqual(fileWithFullpath)
})

it('should compute fullpath for file with incomplete path', async () => {
const res = await computeFileFullpath(client, fileWithStackPath)
expect(res.path).toEqual('ROOT/MYDIR/file2')
})

it('should compute fullpath for file with no path', async () => {
// eslint-disable-next-line prettier/prettier
(queryDocById as jest.Mock).mockResolvedValue(dir)

const res = await computeFileFullpath(client, filewithNoPath)

expect(res.path).toEqual('ROOT/MYDIR/file3')
})
})
64 changes: 51 additions & 13 deletions packages/cozy-dataproxy-lib/src/search/helpers/normalizeFile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ import {
ROOT_DIR_ID,
SHARED_DRIVES_DIR_ID
} from '../consts'
import { CozyDoc, isIOCozyFile, RawSearchResult } from '../types'
import { queryDocById } from '../queries'
import { CozyDoc, isIOCozyFile } from '../types'

/**
* Normalize file for Front usage in <AutoSuggestion> component inside <BarSearchAutosuggest>
Expand Down Expand Up @@ -38,27 +39,28 @@ export const normalizeFileWithFolders = (

export const addFilePaths = (
client: CozyClient,
results: RawSearchResult[]
): RawSearchResult[] => {
const normResults = [...results]
const filesResults = normResults
.map(res => res.doc)
.filter(doc => isIOCozyFile(doc))
const files = filesResults.filter(file => file.type === TYPE_FILE)
docs: CozyDoc[]
): CozyDoc[] => {
const completedDocs = [...docs]
const filesOrDirs = completedDocs.filter(doc => isIOCozyFile(doc))
const files = filesOrDirs.filter(file => file.type === TYPE_FILE)

if (files.length > 0) {
const dirIds = files.map(file => file.dir_id)
const parentDirs = getDirsFromStore(client, dirIds)
if (parentDirs.length < 1) {
return completedDocs
}
for (const file of files) {
const dir = parentDirs.find(dir => dir._id === file.dir_id)
if (dir) {
const idx = normResults.findIndex(res => res.doc._id === file._id)
const idx = completedDocs.findIndex(doc => doc._id === file._id)
// @ts-expect-error We know that we are manipulating an IOCozyFile here so path exists
normResults[idx].doc.path = dir.path
completedDocs[idx].path = dir.path
}
}
}
return normResults
return completedDocs
}

const getDirsFromStore = (
Expand All @@ -68,8 +70,11 @@ const getDirsFromStore = (
// XXX querying from store is surprisingly slow: 100+ ms for 50 docs, while
// this approach takes 2-3ms... It should be investigated in cozy-client
const allFiles = client.getCollectionFromState(FILES_DOCTYPE) as IOCozyFile[]
const dirs = allFiles.filter(file => file.type === TYPE_DIRECTORY)
return dirs.filter(dir => dirIds.includes(dir._id))
if (allFiles) {
const dirs = allFiles.filter(file => file.type === TYPE_DIRECTORY)
return dirs.filter(dir => dirIds.includes(dir._id))
}
return []
}

export const shouldKeepFile = (file: IOCozyFile): boolean => {
Expand All @@ -81,3 +86,36 @@ export const shouldKeepFile = (file: IOCozyFile): boolean => {

return notInTrash && notRootDir && notSharedDrivesDir
}

export const computeFileFullpath = async (
client: CozyClient,
file: IOCozyFile
): Promise<IOCozyFile> => {
if (file.type === TYPE_DIRECTORY) {
// No need to compute directory path: it is always here
return file
}
if (file.path) {
// If a file path exists, check it is complete, i.e. it includes the name.
// The stack typically does not include the name in the path, which is useful to search on it
if (file.path?.includes(file.name)) {
return file
}
const newPath = `${file.path}/${file.name}`
return { ...file, path: newPath }
}
// If there is no path at all, let's compute it from the parent path

const fileWithPath = { ...file }
const parentDir = (await queryDocById(
client,
FILES_DOCTYPE,
file.dir_id
)) as IOCozyFile

if (parentDir) {
const path = `${parentDir.path}/${file.name}`
fileWithPath.path = path
}
return fileWithPath
}
15 changes: 15 additions & 0 deletions packages/cozy-dataproxy-lib/src/search/queries/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ interface AllDocsResponse {
rows: DBRow[]
}

interface QueryResponseSingleDoc {
data: CozyDoc
}

export const queryFilesForSearch = async (
client: CozyClient
): Promise<CozyDoc[]> => {
Expand All @@ -43,3 +47,14 @@ export const queryAllDocs = async (
): Promise<CozyDoc[]> => {
return client.queryAll<CozyDoc[]>(Q(doctype).limitBy(null))
}

export const queryDocById = async (
client: CozyClient,
doctype: string,
id: string
): Promise<CozyDoc> => {
const resp = (await client.query(Q(doctype).getById(id), {
singleDocData: true
})) as QueryResponseSingleDoc
return resp.data
}

0 comments on commit 50bcdf3

Please sign in to comment.