Merge pull request #1184 from XYOracleNetwork/feature/temporal-indexi…

…ng-multi-indexing

Temporal Indexing Diviner Multi-Indexing

packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/StateToIndexCandidateDiviner/Diviner.ts

@@ -14,7 +14,8 @@ import {
 import { DivinerWrapper } from '@xyo-network/diviner-wrapper'
 import { isModuleState, Labels, ModuleState, ModuleStateSchema } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
-import { isPayloadOfSchemaType, Payload } from '@xyo-network/payload-model'
+import { Payload } from '@xyo-network/payload-model'
+import { intraBoundwitnessSchemaCombinations } from '@xyo-network/payload-utils'
 import { TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
 
 /**
@@ -128,13 +129,10 @@ export class TemporalIndexingDivinerStateToIndexCandidateDiviner<
   }
 
   protected async getPayloadsInBoundWitness(bw: BoundWitness, archivist: ArchivistInstance): Promise<IndexCandidate[] | undefined> {
-    const indexes = this.payload_schemas.map((schema) => bw.payload_schemas?.findIndex((s) => s === schema))
-    const hashes = indexes.map((index) => bw.payload_hashes?.[index])
-    const results = await archivist.get(hashes)
-    const indexCandidateIdentityFunctions = this.payload_schemas.map(isPayloadOfSchemaType)
-    const filteredResults = indexCandidateIdentityFunctions.map((is) => results.find(is))
-    if (filteredResults.includes(undefined)) return undefined
-    const indexCandidates: IndexCandidate[] = filteredResults.filter(exists) as IndexCandidate[]
+    const combinations = intraBoundwitnessSchemaCombinations(bw, this.payload_schemas).flat()
+    if (combinations.length === 0) return undefined
+    const hashes = new Set(combinations)
+    const indexCandidates = await archivist.get([...hashes])
     return [bw, ...indexCandidates]
   }
 }

packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/spec/Diviner.Multiple.spec.ts

@@ -0,0 +1,207 @@
+import { assertEx } from '@xylabs/assert'
+import { delay } from '@xylabs/delay'
+import { HDWallet } from '@xyo-network/account'
+import { MemoryArchivist } from '@xyo-network/archivist-memory'
+import { asArchivistInstance } from '@xyo-network/archivist-model'
+import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
+import { isBoundWitness } from '@xyo-network/boundwitness-model'
+import { MemoryBoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-memory'
+import { asDivinerInstance } from '@xyo-network/diviner-model'
+import { MemoryPayloadDiviner } from '@xyo-network/diviner-payload-memory'
+import { PayloadDivinerQueryPayload, PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model'
+import { isTemporalIndexingDivinerResultIndex } from '@xyo-network/diviner-temporal-indexing-model'
+import { PayloadHasher } from '@xyo-network/hash'
+import { ManifestWrapper, PackageManifest } from '@xyo-network/manifest'
+import { isModuleState, Labels, ModuleFactoryLocator } from '@xyo-network/module-model'
+import { MemoryNode } from '@xyo-network/node-memory'
+import { Payload } from '@xyo-network/payload-model'
+import { TimeStamp, TimestampSchema } from '@xyo-network/witness-timestamp'
+
+import { TemporalIndexingDiviner } from '../Diviner'
+import { TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner } from '../DivinerQueryToIndexQueryDiviner'
+import { TemporalIndexingDivinerIndexCandidateToIndexDiviner } from '../IndexCandidateToIndexDiviner'
+import { TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner } from '../IndexQueryResponseToDivinerQueryResponseDiviner'
+import { TemporalIndexingDivinerStateToIndexCandidateDiviner } from '../StateToIndexCandidateDiviner'
+import imageThumbnailDivinerManifest from './TemporalDiviner.json'
+
+type ImageThumbnail = Payload<{
+  http?: {
+    code?: string
+    ipAddress?: string
+    status?: number
+  }
+  // schema: 'network.xyo.image.thumbnail'
+  sourceHash?: string
+  sourceUrl: string
+  url?: string
+}>
+
+type Query = PayloadDivinerQueryPayload & { status?: number; success?: boolean; url?: string }
+
+/**
+ * @group slow
+ */
+describe('TemporalIndexingDiviner - Multiple', () => {
+  const sourceUrl = 'https://placekitten.com/200/300'
+  const thumbnailHttpSuccess: ImageThumbnail = {
+    http: {
+      status: 200,
+    },
+    schema: 'network.xyo.image.thumbnail',
+    sourceHash: '7f39363514d9d9b958a5a993edeba35cb44f912c7072ed9ddd628728ac0fd681',
+    sourceUrl,
+    url: 'data:image/png;base64,===',
+  }
+
+  const thumbnailHttpFail: ImageThumbnail = {
+    http: {
+      ipAddress: '104.17.96.13',
+      status: 429,
+    },
+    schema: 'network.xyo.image.thumbnail',
+    sourceUrl,
+  }
+
+  const thumbnailCodeFail: ImageThumbnail = {
+    http: {
+      code: 'FAILED',
+    },
+    schema: 'network.xyo.image.thumbnail',
+    sourceUrl,
+  }
+
+  const thumbnailWitnessFail: ImageThumbnail = {
+    http: {
+      ipAddress: '104.17.96.13',
+    },
+    schema: 'network.xyo.image.thumbnail',
+    sourceUrl,
+  }
+  const witnessedThumbnails = [thumbnailHttpSuccess, thumbnailHttpFail, thumbnailCodeFail, thumbnailWitnessFail]
+
+  let sut: TemporalIndexingDiviner
+  let node: MemoryNode
+
+  beforeAll(async () => {
+    const labels: Labels = {
+      'network.xyo.image.thumbnail': 'diviner',
+    }
+    const wallet = await HDWallet.random()
+    const locator = new ModuleFactoryLocator()
+    locator.register(MemoryArchivist)
+    locator.register(MemoryBoundWitnessDiviner)
+    locator.register(MemoryPayloadDiviner)
+    locator.register(TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner, labels)
+    locator.register(TemporalIndexingDivinerIndexCandidateToIndexDiviner, labels)
+    locator.register(TemporalIndexingDivinerIndexQueryResponseToDivinerQueryResponseDiviner, labels)
+    locator.register(TemporalIndexingDivinerStateToIndexCandidateDiviner, labels)
+    locator.register(TemporalIndexingDiviner, labels)
+    const manifest = imageThumbnailDivinerManifest as PackageManifest
+    const manifestWrapper = new ManifestWrapper(manifest, wallet, locator)
+    node = await manifestWrapper.loadNodeFromIndex(0)
+    await node.start()
+
+    const privateModules = manifest.nodes[0].modules?.private ?? []
+    const publicModules = manifest.nodes[0].modules?.public ?? []
+    const mods = await node.resolve()
+    expect(mods.length).toBe(privateModules.length + publicModules.length + 1)
+
+    // Insert previously witnessed payloads into thumbnail archivist
+    const timestamp: TimeStamp = { schema: TimestampSchema, timestamp: Date.now() }
+    const [boundWitness, payloads] = await new BoundWitnessBuilder().payloads([timestamp, ...witnessedThumbnails]).build()
+
+    const thumbnailArchivist = assertEx(asArchivistInstance<MemoryArchivist>(await node.resolve('ImageThumbnailArchivist')))
+    await thumbnailArchivist.insert([boundWitness, ...payloads])
+
+    sut = assertEx(asDivinerInstance<TemporalIndexingDiviner>(await node.resolve('ImageThumbnailDiviner')))
+
+    // Allow enough time for diviner to divine
+    await delay(300)
+  }, 40_000)
+  describe('diviner state', () => {
+    let stateArchivist: MemoryArchivist
+    beforeAll(async () => {
+      const mod = await node.resolve('AddressStateArchivist')
+      stateArchivist = assertEx(asArchivistInstance<MemoryArchivist>(mod))
+    })
+    it('has expected bound witnesses', async () => {
+      const payloads = await stateArchivist.all()
+      const stateBoundWitnesses = payloads.filter(isBoundWitness)
+      expect(stateBoundWitnesses).toBeArrayOfSize(2)
+      for (const stateBoundWitness of stateBoundWitnesses) {
+        expect(stateBoundWitness).toBeObject()
+        expect(stateBoundWitness.addresses).toBeArrayOfSize(1)
+        expect(stateBoundWitness.addresses).toContain(sut.address)
+      }
+    })
+    it('has expected state', async () => {
+      const payloads = await stateArchivist.all()
+      const statePayloads = payloads.filter(isModuleState)
+      expect(statePayloads).toBeArrayOfSize(2)
+      expect(statePayloads.at(-1)).toBeObject()
+      const statePayload = assertEx(statePayloads.at(-1))
+      expect(statePayload.state).toBeObject()
+      expect(statePayload.state?.offset).toBe(1)
+    })
+  })
+  describe('diviner index', () => {
+    let indexArchivist: MemoryArchivist
+    beforeAll(async () => {
+      const mod = await node.resolve('ImageThumbnailDivinerIndexArchivist')
+      indexArchivist = assertEx(asArchivistInstance<MemoryArchivist>(mod))
+    })
+    // NOTE: We're not signing indexes for performance reasons
+    it.skip('has expected bound witnesses', async () => {
+      const payloads = await indexArchivist.all()
+      const indexBoundWitnesses = payloads.filter(isBoundWitness)
+      expect(indexBoundWitnesses).toBeArrayOfSize(1)
+      const indexBoundWitness = indexBoundWitnesses[0]
+      expect(indexBoundWitness).toBeObject()
+      expect(indexBoundWitness.addresses).toBeArrayOfSize(1)
+      expect(indexBoundWitness.addresses).toContain(sut.address)
+    })
+    it('has expected index', async () => {
+      const payloads = await indexArchivist.all()
+      const indexPayloads = payloads.filter(isTemporalIndexingDivinerResultIndex)
+      expect(indexPayloads).toBeArrayOfSize(witnessedThumbnails.length)
+    })
+  })
+  describe('with no thumbnail for the provided URL', () => {
+    const url = 'https://does.not.exist.io'
+    const schema = PayloadDivinerQuerySchema
+    it('returns nothing', async () => {
+      const query: Query = { schema, url }
+      const result = await sut.divine([query])
+      expect(result).toBeArrayOfSize(0)
+    })
+  })
+  describe('with thumbnails for the provided URL', () => {
+    const url = sourceUrl
+    const schema = PayloadDivinerQuerySchema
+    describe('with no filter criteria', () => {
+      it('returns the most recent result', async () => {
+        const query: Query = { schema, url }
+        const results = await sut.divine([query])
+        const result = results.find(isTemporalIndexingDivinerResultIndex)
+        expect(result).toBeDefined()
+        const payload = assertEx(witnessedThumbnails.at(-1))
+        const expected = await PayloadHasher.hashAsync(payload)
+        expect(result?.sources).toContain(expected)
+      })
+    })
+    describe('with filter criteria', () => {
+      describe('for status code', () => {
+        const cases: ImageThumbnail[] = [thumbnailHttpSuccess, thumbnailHttpFail]
+        it.each(cases)('returns the most recent instance of that status code', async (payload) => {
+          const { status } = payload.http ?? {}
+          const query: Query = { schema, status, url }
+          const results = await sut.divine([query])
+          const result = results.find(isTemporalIndexingDivinerResultIndex)
+          expect(result).toBeDefined()
+          const expected = await PayloadHasher.hashAsync(payload)
+          expect(result?.sources).toContain(expected)
+        })
+      })
+    })
+  })
+})

packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/spec/Diviner.spec.ts

@@ -137,7 +137,7 @@ describe('TemporalIndexingDiviner', () => {
     sut = assertEx(asDivinerInstance<TemporalIndexingDiviner>(await node.resolve('ImageThumbnailDiviner')))
 
     // Allow enough time for diviner to divine
-    await delay(5000)
+    await delay(300)
   }, 40_000)
   describe('diviner state', () => {
     let stateArchivist: MemoryArchivist

packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/spec/TemporalDiviner.json

@@ -197,7 +197,7 @@
               },
               "language": "javascript",
               "name": "ImageThumbnailDiviner",
-              "pollFrequency": 1000,
+              "pollFrequency": 100,
               "schema": "network.xyo.diviner.indexing.config",
               "stateStore": {
                 "archivist": "AddressStateArchivist",