Merge pull request #1188 from XYOracleNetwork/feature/indexeddb-paylo…

…ad-indexes

Indexeddb Payload Indexes

packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/Diviner.ts

@@ -96,18 +96,18 @@ export class IndexedDbBoundWitnessDiviner<
     }
     // Collect results up to the limit
     while (cursor && results.length < parsedLimit) {
-      const bw = cursor.value
-      if (bw) {
+      const value = cursor.value
+      if (value) {
         // If we're filtering on more than just the schema
         if (valueFilters.length > 0) {
           // Ensure all filters pass
-          if (valueFilters.every((filter) => filter(bw))) {
+          if (valueFilters.every((filter) => filter(value))) {
             // Then save the value
-            results.push(bw)
+            results.push(value)
           }
         } else {
           // Otherwise just save the value
-          results.push(bw)
+          results.push(value)
         }
       }
       cursor = await cursor.continue()

packages/modules/packages/diviner/packages/indexeddb/packages/payload/package.json

@@ -10,7 +10,9 @@
     "url": "https://github.com/XYOracleNetwork/sdk-xyo-client-js/issues"
   },
   "dependencies": {
+    "@xylabs/array": "^2.13.23",
     "@xylabs/assert": "^2.13.23",
+    "@xylabs/exists": "^2.13.23",
     "@xyo-network/archivist-model": "workspace:~",
     "@xyo-network/diviner-model": "workspace:~",
     "@xyo-network/diviner-payload-abstract": "workspace:~",

packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/Diviner.ts

@@ -1,4 +1,6 @@
+import { containsAll } from '@xylabs/array'
 import { assertEx } from '@xylabs/assert'
+import { exists } from '@xylabs/exists'
 import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'
 import { IndexSeparator } from '@xyo-network/archivist-model'
 import { DivinerModule, DivinerModuleEventData } from '@xyo-network/diviner-model'
@@ -16,6 +18,20 @@ interface PayloadStore {
   [s: string]: Payload
 }
 
+type AnyPayload = Payload<Record<string, unknown>>
+
+type ValueFilter = (payload?: AnyPayload | null) => boolean
+
+const payloadValueFilter = (key: keyof AnyPayload, value?: unknown | unknown[]): ValueFilter | undefined => {
+  if (!value) return undefined
+  return (payload) => {
+    if (!payload) return false
+    const sourceValue = payload?.[key]
+    if (sourceValue === undefined) return false
+    return Array.isArray(sourceValue) && Array.isArray(value) ? containsAll(sourceValue, value) : sourceValue == value
+  }
+}
+
 export class IndexedDbPayloadDiviner<
   TParams extends IndexedDbPayloadDivinerParams = IndexedDbPayloadDivinerParams,
   TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload,
@@ -72,6 +88,11 @@ export class IndexedDbPayloadDiviner<
     const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next'
     const suggestedIndex = this.selectBestIndex(filter, store)
     const keyRangeValue = this.getKeyRangeValue(suggestedIndex, filter)
+    const valueFilters: ValueFilter[] = props
+      ? Object.entries(props)
+          .map(([key, value]) => payloadValueFilter(key, value))
+          .filter(exists)
+      : []
     let cursor = suggestedIndex
       ? // Conditionally filter on schemas
         await store.index(suggestedIndex).openCursor(IDBKeyRange.only(keyRangeValue), direction)
@@ -85,7 +106,20 @@ export class IndexedDbPayloadDiviner<
     }
     // Collect results up to the limit
     while (cursor && results.length < parsedLimit) {
-      results.push(cursor.value)
+      const value = cursor.value
+      if (value) {
+        // If we're filtering on more than just the schema
+        if (valueFilters.length > 0) {
+          // Ensure all filters pass
+          if (valueFilters.every((filter) => filter(value))) {
+            // Then save the value
+            results.push(value)
+          }
+        } else {
+          // Otherwise just save the value
+          results.push(value)
+        }
+      }
       cursor = await cursor.continue()
     }
     await tx.done

packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/spec/Diviner.spec.ts

@@ -55,6 +55,7 @@ describe('IndexedDbPayloadDiviner', () => {
   }
   const payloadB = {
     foo: ['bar', 'baz'],
+    other: 'value',
     schema: 'network.xyo.debug',
   }
   const payloads = [payloadA, payloadB]
@@ -110,13 +111,29 @@ describe('IndexedDbPayloadDiviner', () => {
     })
     describe('custom field', () => {
       describe('property', () => {
-        it('only returns payloads with that property', async () => {
-          type WithUrl = { url?: string }
-          const url = payloadA.url
-          const query = await new PayloadBuilder<PayloadDivinerQueryPayload & WithUrl>({ schema: PayloadDivinerQuerySchema }).fields({ url }).build()
-          const results = await sut.divine([query])
-          expect(results.length).toBeGreaterThan(0)
-          expect(results.every((result) => (result as WithUrl)?.url === url)).toBe(true)
+        describe('with index', () => {
+          it('only returns payloads with that property', async () => {
+            type WithUrl = { url?: string }
+            const url = payloadA.url
+            const query = await new PayloadBuilder<PayloadDivinerQueryPayload & WithUrl>({ schema: PayloadDivinerQuerySchema })
+              .fields({ url })
+              .build()
+            const results = await sut.divine([query])
+            expect(results.length).toBeGreaterThan(0)
+            expect(results.every((result) => (result as WithUrl)?.url === url)).toBe(true)
+          })
+        })
+        describe('without index', () => {
+          it('only returns payloads with that property', async () => {
+            type WithOther = { other?: string }
+            const other = payloadB.other
+            const query = await new PayloadBuilder<PayloadDivinerQueryPayload & WithOther>({ schema: PayloadDivinerQuerySchema })
+              .fields({ other })
+              .build()
+            const results = await sut.divine([query])
+            expect(results.length).toBeGreaterThan(0)
+            expect(results.every((result) => (result as WithOther)?.other === other)).toBe(true)
+          })
         })
       })
       describe.skip('array', () => {

yarn.lock

@@ -4222,7 +4222,9 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@xyo-network/diviner-payload-indexeddb@workspace:packages/modules/packages/diviner/packages/indexeddb/packages/payload"
   dependencies:
+    "@xylabs/array": "npm:^2.13.23"
     "@xylabs/assert": "npm:^2.13.23"
+    "@xylabs/exists": "npm:^2.13.23"
     "@xylabs/ts-scripts-yarn3": "npm:^3.2.33"
     "@xylabs/tsconfig": "npm:^3.2.33"
     "@xyo-network/account": "workspace:~"