Merge remote-tracking branch 'origin/feature/data-hash-update' into f…

…eature/data-hash-update

package.json

@@ -104,4 +104,4 @@
   },
   "resolutions_comment": "We set the above resolutions to make sure we pull in the latest versions of these packages even if some sub packages request earlier versions",
   "stableVersion": "3.5.2"
-}
+}

packages/modules/packages/diviner/packages/boundwitness/packages/model/src/Predicate.ts

@@ -1,6 +1,5 @@
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
 import type { PayloadDivinerPredicate } from '@xyo-network/diviner-payload-model'
-import type { Payload } from '@xyo-network/payload-model'
 
 export type WithoutSchemas<T> = Omit<Omit<T, 'schema'>, 'schemas'>
 
@@ -10,8 +9,7 @@ export type BoundWitnessDivinerPredicateFields = {
 }
 
 // TODO: Should we just accept "schema"/"schemas" here and infer that they mean "payload_schemas"?
-export type BoundWitnessDivinerPredicate = WithoutSchemas<PayloadDivinerPredicate
-  & Partial<BoundWitness
-  & Payload
-  & BoundWitnessDivinerPredicateFields>
+export type BoundWitnessDivinerPredicate = WithoutSchemas<
+  PayloadDivinerPredicate &
+  Partial<BoundWitness & BoundWitnessDivinerPredicateFields>
 >

packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/package.json

@@ -30,6 +30,7 @@
   "types": "dist/browser/index.d.ts",
   "dependencies": {
     "@xylabs/array": "^4.4.19",
+    "@xylabs/assert": "^4.4.21",
     "@xylabs/exists": "^4.4.19",
     "@xyo-network/archivist-indexeddb": "workspace:^",
     "@xyo-network/archivist-model": "workspace:^",

packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/Diviner.ts

@@ -1,13 +1,13 @@
 import { containsAll } from '@xylabs/array'
+import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
 import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { BoundWitnessSchema, isBoundWitnessWithStorageMeta } from '@xyo-network/boundwitness-model'
 import { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract'
 import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
 import { isBoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
-import { type Schema, SequenceConstants } from '@xyo-network/payload-model'
-import type { IDBPDatabase } from 'idb'
+import type { Schema, Sequence } from '@xyo-network/payload-model'
+import type { IDBPCursorWithValue, IDBPDatabase } from 'idb'
 import { openDB } from 'idb'
 
 import { IndexedDbBoundWitnessDivinerConfigSchema } from './Config.ts'
@@ -50,7 +50,7 @@ export class IndexedDbBoundWitnessDiviner<
   }
 
   /**
-   * The database version. If not supplied via config, it defaults to 1.
+   * The database version. If not supplied via config, it defaults to the archivist default version.
    */
   get dbVersion() {
     return this.config?.dbVersion ?? IndexedDbArchivist.defaultDbVersion
@@ -65,35 +65,41 @@ export class IndexedDbBoundWitnessDiviner<
   }
 
   protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {
-    const query = payloads?.filter(isBoundWitnessDivinerQueryPayload)?.pop()
+    const query = payloads?.find(isBoundWitnessDivinerQueryPayload)
     if (!query) return []
-
     const result = await this.tryUseDb(async (db) => {
       const {
         addresses, payload_hashes, payload_schemas, limit, cursor, order,
       } = query
       const tx = db.transaction(this.storeName, 'readonly')
       const store = tx.objectStore(this.storeName)
       const results: TOut[] = []
-      const parsedCursor = cursor ?? SequenceConstants.minLocalSequence
+      const parsedCursor = cursor
       const parsedLimit = limit ?? 10
-      const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next'
       const valueFilters: ValueFilter[] = [
         bwValueFilter('addresses', addresses),
         bwValueFilter('payload_hashes', payload_hashes),
         bwValueFilter('payload_schemas', payload_schemas),
       ].filter(exists)
-      // Only iterate over BWs
-      let dbCursor = await store.index(IndexedDbArchivist.schemaIndexName).openCursor(IDBKeyRange.only(BoundWitnessSchema), direction)
+      const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next'
 
-      // If we're filtering on more than just the schema, we need to
-      // iterate through all the results
-      if (valueFilters.length === 0) {
-        // Skip records until the offset is reached
-        while (dbCursor && parsedCursor && parsedCursor < dbCursor.value._sequence) {
+      // Iterate all records using the sequence index
+      const sequenceIndex = assertEx(store.index(IndexedDbArchivist.sequenceIndexName), () => 'Failed to get sequence index')
+      let dbCursor: IDBPCursorWithValue<BoundWitnessStore, [string], string, string, 'readonly'> | null
+      = assertEx(await sequenceIndex.openCursor(null, direction), () => `Failed to get cursor [${parsedCursor}, ${cursor}]`)
+
+      // If a cursor was supplied
+      if (parsedCursor !== undefined) {
+        let currentSequence: Sequence | undefined
+        // Skip records until the supplied cursor offset is reached
+        while (dbCursor && currentSequence !== parsedCursor) {
+          // Find the sequence of the current record
+          currentSequence = await dbCursor.value?.sequence
+          // Advance one record beyond the cursor
           dbCursor = await dbCursor.advance(1)
         }
       }
+
       // Collect results up to the limit
       while (dbCursor && results.length < parsedLimit) {
         const value = dbCursor.value
@@ -118,9 +124,7 @@ export class IndexedDbBoundWitnessDiviner<
       }
       await tx.done
       // Remove any metadata before returning to the client
-      return await Promise.all(
-        results.filter(isBoundWitnessWithStorageMeta),
-      )
+      return results
     })
     return result ?? []
   }

packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/spec/Diviner.spec.ts

@@ -1,13 +1,16 @@
 import '@xylabs/vitest-extended'
 
+import { filterAs } from '@xylabs/array'
+import { delay } from '@xylabs/delay'
 import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'
 import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
 import type { BoundWitness } from '@xyo-network/boundwitness-model'
-import { isBoundWitness } from '@xyo-network/boundwitness-model'
+import { asBoundWitnessWithStorageMeta, isBoundWitness } from '@xyo-network/boundwitness-model'
 import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model'
 import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model'
 import { MemoryNode } from '@xyo-network/node-memory'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
+import type { WithStorageMeta } from '@xyo-network/payload-model'
 import {
   IDBCursor,
   IDBCursorWithValue,
@@ -61,19 +64,24 @@ describe('IndexedDbBoundWitnessDiviner', () => {
     foo: ['bar', 'baz'],
     schema: 'network.xyo.debug',
   }
-  const boundWitnesses: BoundWitness[] = []
+  const boundWitnesses: WithStorageMeta<BoundWitness>[] = []
   beforeAll(async () => {
     const [boundWitnessA] = await (new BoundWitnessBuilder().payloads([payloadA])).build()
     const [boundWitnessB] = await (new BoundWitnessBuilder().payloads([payloadB])).build()
     const [boundWitnessC] = await (new BoundWitnessBuilder().payloads([payloadA, payloadB])).build()
-    boundWitnesses.push(boundWitnessA, boundWitnessB, boundWitnessC)
     archivist = await IndexedDbArchivist.create({
       account: 'random',
       config: {
         dbName, schema: IndexedDbArchivist.defaultConfigSchema, storeName,
       },
     })
     await archivist.insert(boundWitnesses)
+    for (const bw of [boundWitnessA, boundWitnessB, boundWitnessC]) {
+      await delay(2)
+      const inserted = await archivist.insert([bw])
+      const insertedBws = filterAs(inserted, asBoundWitnessWithStorageMeta)
+      boundWitnesses.push(...insertedBws)
+    }
     sut = await IndexedDbBoundWitnessDiviner.create({
       account: 'random',
       config: {
@@ -162,13 +170,13 @@ describe('IndexedDbBoundWitnessDiviner', () => {
       })
     })
   })
-  describe('with offset', () => {
+  describe('with cursor', () => {
     describe('when ascending order', () => {
       it('returns payloads from the beginning', async () => {
         for (const [i, boundWitness] of boundWitnesses.entries()) {
           const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
             .fields({
-              limit: 1, offset: i, order: 'asc',
+              limit: 1, cursor: boundWitnesses[i]._sequence, order: 'asc',
             })
             .build()
           const results = await sut.divine([query])
@@ -183,7 +191,7 @@ describe('IndexedDbBoundWitnessDiviner', () => {
         for (let i = 0; i < boundWitnesses.length; i++) {
           const query = new PayloadBuilder<BoundWitnessDivinerQueryPayload>({ schema: BoundWitnessDivinerQuerySchema })
             .fields({
-              limit: 1, offset: i, order: 'desc',
+              limit: 1, cursor: boundWitnesses[i]._sequence, order: 'desc',
             })
             .build()
           const results = await sut.divine([query])

packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/Diviner.ts

@@ -1,7 +1,6 @@
 import { containsAll } from '@xylabs/array'
 import { assertEx } from '@xylabs/assert'
 import { exists } from '@xylabs/exists'
-import type { Hash } from '@xylabs/hex'
 import { removeFields } from '@xylabs/object'
 import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'
 import type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model'
@@ -75,15 +74,12 @@ export class IndexedDbPayloadDiviner<
   }
 
   protected override async divineHandler(payloads?: TIn[]): Promise<TOut[]> {
-    const query = payloads?.find(isPayloadDivinerQueryPayload) as TIn
+    const query = payloads?.find(isPayloadDivinerQueryPayload)
     if (!query) return []
     const result = await this.tryUseDb(async (db) => {
       const {
         schemas, limit, cursor, order, ...props
-      } = removeFields(query as unknown as TIn & { sources?: Hash[] }, [
-        'hash',
-        'schema',
-      ])
+      } = removeFields(query, ['schema'])
       const tx = db.transaction(this.storeName, 'readonly')
       const store = tx.objectStore(this.storeName)
       const results: TOut[] = []

packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/Diviner.ts

@@ -82,9 +82,9 @@ export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<
             sourcePathExpression: '$.limit',
           },
           {
-            defaultValue: 0,
-            destinationField: 'offset',
-            sourcePathExpression: '$.offset',
+            // defaultValue: 0,
+            destinationField: 'cursor',
+            sourcePathExpression: '$.cursor',
           },
           {
             defaultValue: 'desc',
@@ -111,7 +111,7 @@ export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner<
           // TODO: Make sources not need to be deleted
           delete fields.sources
           // TODO: Add support for additional filters
-          return await new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()
+          return new PayloadBuilder<Payload>({ schema: this.indexQuerySchema }).fields(fields).build()
         }),
       )
     }

packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/spec/Diviner.spec.ts

@@ -10,7 +10,7 @@ import {
 } from '@xyo-network/diviner-temporal-indexing-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import type { Payload } from '@xyo-network/payload-model'
-import { isPayloadOfSchemaType } from '@xyo-network/payload-model'
+import { isPayloadOfSchemaType, SequenceConstants } from '@xyo-network/payload-model'
 import {
   beforeAll,
   describe, expect, it,
@@ -32,22 +32,19 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
         },
         {
           limit: 10,
-          offset: 10,
           order: 'asc',
           schema: PayloadDivinerQuerySchema,
         },
       ]
       const expected: PayloadDivinerQueryPayload[] = [
         {
           limit: 1,
-          offset: 0,
           order: 'desc',
           schema: 'network.xyo.diviner.payload.query',
           schemas: [TemporalIndexingDivinerResultIndexSchema],
         } as unknown as PayloadDivinerQueryPayload,
         {
           limit: 10,
-          offset: 10,
           order: 'asc',
           schema: 'network.xyo.diviner.payload.query',
           schemas: [TemporalIndexingDivinerResultIndexSchema],
@@ -85,7 +82,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
         },
         {
           limit: 10,
-          offset: 10,
           order: 'asc',
           schema: divinerQuerySchema,
           status: 200,
@@ -98,7 +94,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
           url,
         },
         {
-          offset: 10,
           schema: divinerQuerySchema,
           url,
         },
@@ -126,15 +121,13 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
       const expected = [
         {
           limit: 1,
-          offset: 0,
           order: 'desc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
           url,
         },
         {
           limit: 10,
-          offset: 10,
           order: 'asc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
@@ -144,31 +137,27 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
         },
         {
           limit: 10,
-          offset: 0,
           order: 'desc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
           url,
         },
         {
           limit: 1,
-          offset: 10,
           order: 'desc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
           url,
         },
         {
           limit: 1,
-          offset: 0,
           order: 'asc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
           url,
         },
         {
           limit: 1,
-          offset: 0,
           order: 'desc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
@@ -177,7 +166,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
         },
         {
           limit: 1,
-          offset: 0,
           order: 'desc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
@@ -186,7 +174,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
         },
         {
           limit: 1,
-          offset: 0,
           order: 'desc',
           schema: indexQuerySchema,
           schemas: [indexSchema],
@@ -207,9 +194,9 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => {
             sourcePathExpression: '$.limit',
           },
           {
-            defaultValue: 0,
-            destinationField: 'offset',
-            sourcePathExpression: '$.offset',
+            // defaultValue: 0,
+            destinationField: 'cursor',
+            sourcePathExpression: '$.cursor',
           },
           {
             defaultValue: 'desc',

packages/modules/packages/diviner/packages/payload/packages/generic/package.json

@@ -29,6 +29,7 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
+    "@xylabs/array": "^4.4.21",
     "@xylabs/assert": "^4.4.19",
     "@xylabs/forget": "^4.4.19",
     "@xylabs/hex": "^4.4.19",