diff --git a/package.json b/package.json index bf31e7c312..910a0ea64e 100644 --- a/package.json +++ b/package.json @@ -104,4 +104,4 @@ }, "resolutions_comment": "We set the above resolutions to make sure we pull in the latest versions of these packages even if some sub packages request earlier versions", "stableVersion": "3.5.2" -} \ No newline at end of file +} diff --git a/packages/modules/packages/diviner/packages/boundwitness/packages/model/src/Predicate.ts b/packages/modules/packages/diviner/packages/boundwitness/packages/model/src/Predicate.ts index 8badf31d41..17964126de 100644 --- a/packages/modules/packages/diviner/packages/boundwitness/packages/model/src/Predicate.ts +++ b/packages/modules/packages/diviner/packages/boundwitness/packages/model/src/Predicate.ts @@ -1,6 +1,5 @@ import type { BoundWitness } from '@xyo-network/boundwitness-model' import type { PayloadDivinerPredicate } from '@xyo-network/diviner-payload-model' -import type { Payload } from '@xyo-network/payload-model' export type WithoutSchemas = Omit, 'schemas'> @@ -10,8 +9,7 @@ export type BoundWitnessDivinerPredicateFields = { } // TODO: Should we just accept "schema"/"schemas" here and infer that they mean "payload_schemas"? -export type BoundWitnessDivinerPredicate = WithoutSchemas +export type BoundWitnessDivinerPredicate = WithoutSchemas< + PayloadDivinerPredicate & + Partial > diff --git a/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/package.json b/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/package.json index ff5ec4a935..c155e57f46 100644 --- a/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/package.json +++ b/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/package.json @@ -30,6 +30,7 @@ "types": "dist/browser/index.d.ts", "dependencies": { "@xylabs/array": "^4.4.19", + "@xylabs/assert": "^4.4.21", "@xylabs/exists": "^4.4.19", "@xyo-network/archivist-indexeddb": "workspace:^", "@xyo-network/archivist-model": "workspace:^", diff --git a/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/Diviner.ts b/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/Diviner.ts index 63f5dd7ae4..18889cf0f6 100644 --- a/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/Diviner.ts +++ b/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/Diviner.ts @@ -1,13 +1,13 @@ import { containsAll } from '@xylabs/array' +import { assertEx } from '@xylabs/assert' import { exists } from '@xylabs/exists' import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb' import type { BoundWitness } from '@xyo-network/boundwitness-model' -import { BoundWitnessSchema, isBoundWitnessWithStorageMeta } from '@xyo-network/boundwitness-model' import { BoundWitnessDiviner } from '@xyo-network/diviner-boundwitness-abstract' import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model' import { isBoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model' -import { type Schema, SequenceConstants } from '@xyo-network/payload-model' -import type { IDBPDatabase } from 'idb' +import type { Schema, Sequence } from '@xyo-network/payload-model' +import type { IDBPCursorWithValue, IDBPDatabase } from 'idb' import { openDB } from 'idb' import { IndexedDbBoundWitnessDivinerConfigSchema } from './Config.ts' @@ -50,7 +50,7 @@ export class IndexedDbBoundWitnessDiviner< } /** - * The database version. If not supplied via config, it defaults to 1. + * The database version. If not supplied via config, it defaults to the archivist default version. */ get dbVersion() { return this.config?.dbVersion ?? IndexedDbArchivist.defaultDbVersion @@ -65,9 +65,8 @@ export class IndexedDbBoundWitnessDiviner< } protected override async divineHandler(payloads?: TIn[]): Promise { - const query = payloads?.filter(isBoundWitnessDivinerQueryPayload)?.pop() + const query = payloads?.find(isBoundWitnessDivinerQueryPayload) if (!query) return [] - const result = await this.tryUseDb(async (db) => { const { addresses, payload_hashes, payload_schemas, limit, cursor, order, @@ -75,25 +74,32 @@ export class IndexedDbBoundWitnessDiviner< const tx = db.transaction(this.storeName, 'readonly') const store = tx.objectStore(this.storeName) const results: TOut[] = [] - const parsedCursor = cursor ?? SequenceConstants.minLocalSequence + const parsedCursor = cursor const parsedLimit = limit ?? 10 - const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next' const valueFilters: ValueFilter[] = [ bwValueFilter('addresses', addresses), bwValueFilter('payload_hashes', payload_hashes), bwValueFilter('payload_schemas', payload_schemas), ].filter(exists) - // Only iterate over BWs - let dbCursor = await store.index(IndexedDbArchivist.schemaIndexName).openCursor(IDBKeyRange.only(BoundWitnessSchema), direction) + const direction: IDBCursorDirection = order === 'desc' ? 'prev' : 'next' - // If we're filtering on more than just the schema, we need to - // iterate through all the results - if (valueFilters.length === 0) { - // Skip records until the offset is reached - while (dbCursor && parsedCursor && parsedCursor < dbCursor.value._sequence) { + // Iterate all records using the sequence index + const sequenceIndex = assertEx(store.index(IndexedDbArchivist.sequenceIndexName), () => 'Failed to get sequence index') + let dbCursor: IDBPCursorWithValue | null + = assertEx(await sequenceIndex.openCursor(null, direction), () => `Failed to get cursor [${parsedCursor}, ${cursor}]`) + + // If a cursor was supplied + if (parsedCursor !== undefined) { + let currentSequence: Sequence | undefined + // Skip records until the supplied cursor offset is reached + while (dbCursor && currentSequence !== parsedCursor) { + // Find the sequence of the current record + currentSequence = await dbCursor.value?.sequence + // Advance one record beyond the cursor dbCursor = await dbCursor.advance(1) } } + // Collect results up to the limit while (dbCursor && results.length < parsedLimit) { const value = dbCursor.value @@ -118,9 +124,7 @@ export class IndexedDbBoundWitnessDiviner< } await tx.done // Remove any metadata before returning to the client - return await Promise.all( - results.filter(isBoundWitnessWithStorageMeta), - ) + return results }) return result ?? [] } diff --git a/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/spec/Diviner.spec.ts b/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/spec/Diviner.spec.ts index bf4d72b541..9df40afffa 100644 --- a/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/spec/Diviner.spec.ts +++ b/packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness/src/spec/Diviner.spec.ts @@ -1,13 +1,16 @@ import '@xylabs/vitest-extended' +import { filterAs } from '@xylabs/array' +import { delay } from '@xylabs/delay' import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb' import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder' import type { BoundWitness } from '@xyo-network/boundwitness-model' -import { isBoundWitness } from '@xyo-network/boundwitness-model' +import { asBoundWitnessWithStorageMeta, isBoundWitness } from '@xyo-network/boundwitness-model' import type { BoundWitnessDivinerQueryPayload } from '@xyo-network/diviner-boundwitness-model' import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitness-model' import { MemoryNode } from '@xyo-network/node-memory' import { PayloadBuilder } from '@xyo-network/payload-builder' +import type { WithStorageMeta } from '@xyo-network/payload-model' import { IDBCursor, IDBCursorWithValue, @@ -61,12 +64,11 @@ describe('IndexedDbBoundWitnessDiviner', () => { foo: ['bar', 'baz'], schema: 'network.xyo.debug', } - const boundWitnesses: BoundWitness[] = [] + const boundWitnesses: WithStorageMeta[] = [] beforeAll(async () => { const [boundWitnessA] = await (new BoundWitnessBuilder().payloads([payloadA])).build() const [boundWitnessB] = await (new BoundWitnessBuilder().payloads([payloadB])).build() const [boundWitnessC] = await (new BoundWitnessBuilder().payloads([payloadA, payloadB])).build() - boundWitnesses.push(boundWitnessA, boundWitnessB, boundWitnessC) archivist = await IndexedDbArchivist.create({ account: 'random', config: { @@ -74,6 +76,12 @@ describe('IndexedDbBoundWitnessDiviner', () => { }, }) await archivist.insert(boundWitnesses) + for (const bw of [boundWitnessA, boundWitnessB, boundWitnessC]) { + await delay(2) + const inserted = await archivist.insert([bw]) + const insertedBws = filterAs(inserted, asBoundWitnessWithStorageMeta) + boundWitnesses.push(...insertedBws) + } sut = await IndexedDbBoundWitnessDiviner.create({ account: 'random', config: { @@ -162,13 +170,13 @@ describe('IndexedDbBoundWitnessDiviner', () => { }) }) }) - describe('with offset', () => { + describe('with cursor', () => { describe('when ascending order', () => { it('returns payloads from the beginning', async () => { for (const [i, boundWitness] of boundWitnesses.entries()) { const query = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }) .fields({ - limit: 1, offset: i, order: 'asc', + limit: 1, cursor: boundWitnesses[i]._sequence, order: 'asc', }) .build() const results = await sut.divine([query]) @@ -183,7 +191,7 @@ describe('IndexedDbBoundWitnessDiviner', () => { for (let i = 0; i < boundWitnesses.length; i++) { const query = new PayloadBuilder({ schema: BoundWitnessDivinerQuerySchema }) .fields({ - limit: 1, offset: i, order: 'desc', + limit: 1, cursor: boundWitnesses[i]._sequence, order: 'desc', }) .build() const results = await sut.divine([query]) diff --git a/packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/Diviner.ts b/packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/Diviner.ts index 882cf58261..aa307d87e0 100644 --- a/packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/Diviner.ts +++ b/packages/modules/packages/diviner/packages/indexeddb/packages/payload/src/Diviner.ts @@ -1,7 +1,6 @@ import { containsAll } from '@xylabs/array' import { assertEx } from '@xylabs/assert' import { exists } from '@xylabs/exists' -import type { Hash } from '@xylabs/hex' import { removeFields } from '@xylabs/object' import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb' import type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model' @@ -75,15 +74,12 @@ export class IndexedDbPayloadDiviner< } protected override async divineHandler(payloads?: TIn[]): Promise { - const query = payloads?.find(isPayloadDivinerQueryPayload) as TIn + const query = payloads?.find(isPayloadDivinerQueryPayload) if (!query) return [] const result = await this.tryUseDb(async (db) => { const { schemas, limit, cursor, order, ...props - } = removeFields(query as unknown as TIn & { sources?: Hash[] }, [ - 'hash', - 'schema', - ]) + } = removeFields(query, ['schema']) const tx = db.transaction(this.storeName, 'readonly') const store = tx.objectStore(this.storeName) const results: TOut[] = [] diff --git a/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/Diviner.ts b/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/Diviner.ts index d0e7f14637..df6852120d 100644 --- a/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/Diviner.ts +++ b/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/Diviner.ts @@ -82,9 +82,9 @@ export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner< sourcePathExpression: '$.limit', }, { - defaultValue: 0, - destinationField: 'offset', - sourcePathExpression: '$.offset', + // defaultValue: 0, + destinationField: 'cursor', + sourcePathExpression: '$.cursor', }, { defaultValue: 'desc', @@ -111,7 +111,7 @@ export class TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner< // TODO: Make sources not need to be deleted delete fields.sources // TODO: Add support for additional filters - return await new PayloadBuilder({ schema: this.indexQuerySchema }).fields(fields).build() + return new PayloadBuilder({ schema: this.indexQuerySchema }).fields(fields).build() }), ) } diff --git a/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/spec/Diviner.spec.ts b/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/spec/Diviner.spec.ts index 41adc32732..fe76b0e343 100644 --- a/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/spec/Diviner.spec.ts +++ b/packages/modules/packages/diviner/packages/indexing/packages/temporal/packages/memory/src/DivinerQueryToIndexQueryDiviner/spec/Diviner.spec.ts @@ -10,7 +10,7 @@ import { } from '@xyo-network/diviner-temporal-indexing-model' import { PayloadBuilder } from '@xyo-network/payload-builder' import type { Payload } from '@xyo-network/payload-model' -import { isPayloadOfSchemaType } from '@xyo-network/payload-model' +import { isPayloadOfSchemaType, SequenceConstants } from '@xyo-network/payload-model' import { beforeAll, describe, expect, it, @@ -32,7 +32,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 10, - offset: 10, order: 'asc', schema: PayloadDivinerQuerySchema, }, @@ -40,14 +39,12 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { const expected: PayloadDivinerQueryPayload[] = [ { limit: 1, - offset: 0, order: 'desc', schema: 'network.xyo.diviner.payload.query', schemas: [TemporalIndexingDivinerResultIndexSchema], } as unknown as PayloadDivinerQueryPayload, { limit: 10, - offset: 10, order: 'asc', schema: 'network.xyo.diviner.payload.query', schemas: [TemporalIndexingDivinerResultIndexSchema], @@ -85,7 +82,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 10, - offset: 10, order: 'asc', schema: divinerQuerySchema, status: 200, @@ -98,7 +94,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { url, }, { - offset: 10, schema: divinerQuerySchema, url, }, @@ -126,7 +121,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { const expected = [ { limit: 1, - offset: 0, order: 'desc', schema: indexQuerySchema, schemas: [indexSchema], @@ -134,7 +128,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 10, - offset: 10, order: 'asc', schema: indexQuerySchema, schemas: [indexSchema], @@ -144,7 +137,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 10, - offset: 0, order: 'desc', schema: indexQuerySchema, schemas: [indexSchema], @@ -152,7 +144,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 1, - offset: 10, order: 'desc', schema: indexQuerySchema, schemas: [indexSchema], @@ -160,7 +151,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 1, - offset: 0, order: 'asc', schema: indexQuerySchema, schemas: [indexSchema], @@ -168,7 +158,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 1, - offset: 0, order: 'desc', schema: indexQuerySchema, schemas: [indexSchema], @@ -177,7 +166,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 1, - offset: 0, order: 'desc', schema: indexQuerySchema, schemas: [indexSchema], @@ -186,7 +174,6 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { }, { limit: 1, - offset: 0, order: 'desc', schema: indexQuerySchema, schemas: [indexSchema], @@ -207,9 +194,9 @@ describe('TemporalIndexingDivinerDivinerQueryToIndexQueryDiviner', () => { sourcePathExpression: '$.limit', }, { - defaultValue: 0, - destinationField: 'offset', - sourcePathExpression: '$.offset', + // defaultValue: 0, + destinationField: 'cursor', + sourcePathExpression: '$.cursor', }, { defaultValue: 'desc', diff --git a/packages/modules/packages/diviner/packages/payload/packages/generic/package.json b/packages/modules/packages/diviner/packages/payload/packages/generic/package.json index 6979ee4eed..5496396559 100644 --- a/packages/modules/packages/diviner/packages/payload/packages/generic/package.json +++ b/packages/modules/packages/diviner/packages/payload/packages/generic/package.json @@ -29,6 +29,7 @@ "module": "dist/neutral/index.mjs", "types": "dist/neutral/index.d.ts", "dependencies": { + "@xylabs/array": "^4.4.21", "@xylabs/assert": "^4.4.19", "@xylabs/forget": "^4.4.19", "@xylabs/hex": "^4.4.19", diff --git a/packages/modules/packages/diviner/packages/payload/packages/generic/src/Diviner.ts b/packages/modules/packages/diviner/packages/payload/packages/generic/src/Diviner.ts index 2724cc0512..9de3f36acc 100644 --- a/packages/modules/packages/diviner/packages/payload/packages/generic/src/Diviner.ts +++ b/packages/modules/packages/diviner/packages/payload/packages/generic/src/Diviner.ts @@ -1,17 +1,17 @@ +import { filterAs } from '@xylabs/array' import { assertEx } from '@xylabs/assert' import { forget } from '@xylabs/forget' -import { type Hash, type Hex } from '@xylabs/hex' -import { type EmptyObject } from '@xylabs/object' +import { type Hex } from '@xylabs/hex' import type { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist-model' import type { DivinerInstance, DivinerModuleEventData } from '@xyo-network/diviner-model' import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract' -import type { - Order, - PayloadDivinerConfig, - PayloadDivinerParams, - PayloadDivinerQueryPayload, +import { + asPayloadDivinerQueryPayload, + type Order, + type PayloadDivinerConfig, + type PayloadDivinerParams, + type PayloadDivinerQueryPayload, } from '@xyo-network/diviner-payload-model' -import { isPayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model' import type { EventListener } from '@xyo-network/module-events' import { PayloadBuilder } from '@xyo-network/payload-builder' import { @@ -35,7 +35,7 @@ export type GenericPayloadDivinerConfig = PayloadDivinerConfig< export class GenericPayloadDiviner< TParams extends PayloadDivinerParams = PayloadDivinerParams, - TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, + TIn extends PayloadDivinerQueryPayload = PayloadDivinerQueryPayload, TOut extends WithStorageMeta = WithStorageMeta, TEventData extends DivinerModuleEventData, TIn, TOut> = DivinerModuleEventData< DivinerInstance, @@ -97,11 +97,9 @@ export class GenericPayloadDiviner< } protected override async divineHandler(payloads?: TIn[]): Promise { - const filters = payloads?.filter(isPayloadDivinerQueryPayload) ?? [] + const filters = filterAs(payloads ?? [], asPayloadDivinerQueryPayload) assertEx(filters.length < 2, () => 'Multiple PayloadDivinerQuery payloads may not be specified') - const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as unknown as - PayloadDivinerQueryPayload - + const filter = assertEx(filters.shift(), () => 'No PayloadDivinerQuery specified') as PayloadDivinerQueryPayload await this.updateIndex() const { diff --git a/packages/modules/packages/diviner/packages/payload/packages/generic/src/spec/Diviner.indexdb.spec.ts b/packages/modules/packages/diviner/packages/payload/packages/generic/src/spec/Diviner.indexdb.spec.ts index ea5a19a04d..b46186a254 100644 --- a/packages/modules/packages/diviner/packages/payload/packages/generic/src/spec/Diviner.indexdb.spec.ts +++ b/packages/modules/packages/diviner/packages/payload/packages/generic/src/spec/Diviner.indexdb.spec.ts @@ -1,8 +1,6 @@ import '@xylabs/vitest-extended' import { delay } from '@xylabs/delay' -import type { Hash } from '@xylabs/hex' -import type { EmptyObject } from '@xylabs/object' import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb' import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model' import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model' @@ -117,7 +115,7 @@ describe('GenericPayloadDiviner', () => { describe('single', () => { it.each(['network.xyo.test', 'network.xyo.debug'])('only returns payloads of that schema', async (schema) => { const schemas = [schema] - const query = await new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ schemas }) .build() const results = await sut.divine([query]) @@ -126,7 +124,7 @@ describe('GenericPayloadDiviner', () => { }) it('only return single payload of that schema', async () => { const schemas = ['network.xyo.debug'] - const query = await new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 1, schemas }) .build() const results = await sut.divine([query]) @@ -136,7 +134,7 @@ describe('GenericPayloadDiviner', () => { }) it('only return single payload of that schema (desc)', async () => { const schemas = ['network.xyo.debug'] - const query = await new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 1, order: 'desc', schemas, }) @@ -148,7 +146,7 @@ describe('GenericPayloadDiviner', () => { }) it('only return single payload of that schema (asc)', async () => { const schemas = ['network.xyo.debug'] - const query = await new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 1, order: 'asc', schemas, }) @@ -162,7 +160,7 @@ describe('GenericPayloadDiviner', () => { describe('multiple', () => { it('only returns payloads of that schema', async () => { const schemas = ['network.xyo.test', 'network.xyo.debug'] - const query = await new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ schemas }) .build() const results = await sut.divine([query]) @@ -173,7 +171,7 @@ describe('GenericPayloadDiviner', () => { describe('paging', () => { it('test paging with multiple calls (asc)', async () => { const schemas = ['network.xyo.test', 'network.xyo.debug'] - const query = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 2, order: 'asc', schemas, }) @@ -186,7 +184,7 @@ describe('GenericPayloadDiviner', () => { expect(resultSequences[1]).toBe(insertedPayloads[1]._sequence) const cursor = resultSequences[1] - const query2 = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query2 = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 2, cursor, order: 'asc', schemas, }) @@ -198,7 +196,7 @@ describe('GenericPayloadDiviner', () => { expect(resultSequences2[1]).toBe(insertedPayloads[3]._sequence) const cursor2 = resultSequences2[1] - const query3 = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query3 = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 2, cursor: cursor2, order: 'asc', schemas, }) @@ -208,7 +206,7 @@ describe('GenericPayloadDiviner', () => { }) it('test paging with multiple calls (desc)', async () => { const schemas = ['network.xyo.test', 'network.xyo.debug'] - const query = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 2, order: 'desc', schemas, }) @@ -220,7 +218,7 @@ describe('GenericPayloadDiviner', () => { expect(resultSequences[1]).toBe(insertedPayloads[2]._sequence) const cursor = resultSequences[1] - const query2 = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query2 = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 2, cursor, order: 'desc', schemas, }) @@ -232,7 +230,7 @@ describe('GenericPayloadDiviner', () => { expect(resultSequences2[1]).toBe(insertedPayloads[0]._sequence) const cursor2 = resultSequences2[1] - const query3 = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) + const query3 = new PayloadBuilder({ schema: PayloadDivinerQuerySchema }) .fields({ limit: 2, cursor: cursor2, order: 'desc', schemas, }) @@ -247,7 +245,7 @@ describe('GenericPayloadDiviner', () => { it('only returns payloads with that property', async () => { type WithUrl = { url?: string } const url = payloadA.url - const query = await new PayloadBuilder & WithUrl>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) .fields({ url }) .build() const results = await sut.divine([query]) @@ -259,7 +257,7 @@ describe('GenericPayloadDiviner', () => { const cases: string[][] = [['bar'], ['baz'], ['bar', 'baz']] it.each(cases)('only returns payloads that have an array containing all the values supplied', async (...foo) => { type WithFoo = { foo?: string[] } - const query = await new PayloadBuilder & WithFoo>({ schema: PayloadDivinerQuerySchema }) + const query = new PayloadBuilder>({ schema: PayloadDivinerQuerySchema }) .fields({ foo }) .build() const results = await sut.divine([query]) diff --git a/packages/modules/packages/diviner/packages/payload/packages/model/src/Predicate.ts b/packages/modules/packages/diviner/packages/payload/packages/model/src/Predicate.ts index c2f506c568..40bc2e5dd1 100644 --- a/packages/modules/packages/diviner/packages/payload/packages/model/src/Predicate.ts +++ b/packages/modules/packages/diviner/packages/payload/packages/model/src/Predicate.ts @@ -1,22 +1,11 @@ -import type { - Address, Hash, Hex, -} from '@xylabs/hex' import type { EmptyObject } from '@xylabs/object' -import type { Schema } from '@xyo-network/payload-model' +import type { Schema, Sequence } from '@xyo-network/payload-model' import type { Order } from './Order.ts' -/* Note: Added Omit to PayloadFindDiviner for offset until we support hash based offsets */ - -export type PayloadDivinerPredicate = Partial< +export type PayloadDivinerPredicate = Partial< { - /** - * @deprecated Use BW Diviner to find signed Payloads matching desired - * criteria, then get Payloads by hash directly from Archivist - */ - address: Address | Address[] - cursor: TCursor - hash: Hash + cursor: Sequence limit: number order: Order schemas: Schema[] diff --git a/packages/modules/packages/diviner/packages/payload/packages/model/src/Query.ts b/packages/modules/packages/diviner/packages/payload/packages/model/src/Query.ts index e1bf315b54..3738b77852 100644 --- a/packages/modules/packages/diviner/packages/payload/packages/model/src/Query.ts +++ b/packages/modules/packages/diviner/packages/payload/packages/model/src/Query.ts @@ -1,5 +1,4 @@ -import type { Hex } from '@xylabs/hex' -import type { EmptyObject } from '@xylabs/object' +import { AsObjectFactory, type EmptyObject } from '@xylabs/object' import { isPayloadOfSchemaType, type Query } from '@xyo-network/payload-model' import type { PayloadDivinerPredicate } from './Predicate.ts' @@ -8,7 +7,10 @@ import { PayloadDivinerSchema } from './Schema.ts' export type PayloadDivinerQuerySchema = `${PayloadDivinerSchema}.query` export const PayloadDivinerQuerySchema: PayloadDivinerQuerySchema = `${PayloadDivinerSchema}.query` -export type PayloadDivinerQueryPayload = Query< - { schema: PayloadDivinerQuerySchema } & PayloadDivinerPredicate +export type PayloadDivinerQueryPayload = Query< + { schema: PayloadDivinerQuerySchema } & PayloadDivinerPredicate > + export const isPayloadDivinerQueryPayload = isPayloadOfSchemaType(PayloadDivinerQuerySchema) + +export const asPayloadDivinerQueryPayload = AsObjectFactory.create(isPayloadDivinerQueryPayload) diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/combineRules.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/combineRules.ts index 75a88904b2..ebad3f653f 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/combineRules.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/combineRules.ts @@ -9,7 +9,7 @@ import type { import { isPayloadAddressRule, isPayloadSchemaRule, - isPayloadTimestampOrderRule, + isPayloadSequenceOrderRule, } from '@xyo-network/diviner-payload-pointer-model' // TODO: Could make it so that composability is such that we: @@ -29,16 +29,16 @@ export const combineRules = (rules: PayloadRule[][]): PayloadSearchCriteria => { .filter(exists) assertEx(schemas.length, () => 'At least one schema must be supplied') - const directionTimestamp = rules.flat().filter(isPayloadTimestampOrderRule).filter(exists) - assertEx(directionTimestamp.length < 2, () => 'Must not supply more than 1 direction/timestamp rule') + const sequenceOrderRule = rules.flat().filter(isPayloadSequenceOrderRule).filter(exists) + assertEx(sequenceOrderRule.length < 2, () => 'Must not supply more than 1 direction/timestamp rule') - const order: Order = directionTimestamp[0]?.order || 'desc' - const timestamp: number = directionTimestamp.length > 0 ? directionTimestamp[0]?.timestamp : Date.now() + const order: Order = sequenceOrderRule[0]?.order || 'desc' + const sequence = sequenceOrderRule[0]?.sequence return { addresses, order, schemas, - timestamp, + sequence, } } diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/findPayload.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/findPayload.ts index 61cf2659c9..7c498ad645 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/findPayload.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/findPayload.ts @@ -6,7 +6,9 @@ import { BoundWitnessDivinerQuerySchema } from '@xyo-network/diviner-boundwitnes import type { PayloadDiviner } from '@xyo-network/diviner-payload-abstract' import type { PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model' import { PayloadDivinerQuerySchema } from '@xyo-network/diviner-payload-model' -import type { PayloadSearchCriteria, PointerPayload } from '@xyo-network/diviner-payload-pointer-model' +import type { + PayloadRule, PayloadSearchCriteria, PointerPayload, +} from '@xyo-network/diviner-payload-pointer-model' import { isBoundWitnessPointer } from '@xyo-network/diviner-payload-pointer-model' import type { Payload, Schema } from '@xyo-network/payload-model' import { PayloadWrapper } from '@xyo-network/payload-wrapper' @@ -17,7 +19,7 @@ const limit = 1 const createBoundWitnessFilterFromSearchCriteria = (searchCriteria: PayloadSearchCriteria): BoundWitnessDivinerQueryPayload[] => { const { - addresses, order = 'desc', schemas, timestamp, + addresses, order = 'desc', schemas, } = searchCriteria const query: BoundWitnessDivinerQueryPayload = { addresses, @@ -25,17 +27,14 @@ const createBoundWitnessFilterFromSearchCriteria = (searchCriteria: PayloadSearc order, payload_schemas: schemas, schema: BoundWitnessDivinerQuerySchema, - timestamp, } return [query] } const createPayloadFilterFromSearchCriteria = (searchCriteria: PayloadSearchCriteria): Payload[] => { - const { - order = 'desc', schemas, timestamp, - } = searchCriteria + const { order = 'desc', schemas } = searchCriteria const query: PayloadDivinerQueryPayload = { - limit, order, schema: PayloadDivinerQuerySchema, schemas, timestamp, + limit, order, schema: PayloadDivinerQuerySchema, schemas, } return [query] } @@ -46,7 +45,8 @@ export const findPayload = async ( payloadDiviner: PayloadDiviner, pointer: PointerPayload, ): Promise => { - const searchCriteria = combineRules(pointer.reference) + const reference = pointer.reference as PayloadRule[][] + const searchCriteria = combineRules(reference) const { addresses } = searchCriteria const findWitnessedPayload = addresses?.length const returnBoundWitness = isBoundWitnessPointer(pointer) diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.address.spec.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.address.spec.ts index 4e5f598582..9aed192ac4 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.address.spec.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.address.spec.ts @@ -62,7 +62,7 @@ describe('PayloadPointerDiviner', () => { [accountB, () => payloads[1]], ])('returns Payload signed by address', async (account, getData) => { const expected = getData() - const pointer = await createPointer([[(await account).address]], [[expected.schema]]) + const pointer = createPointer([[(await account).address]], [[expected.schema]]) const result = await sut.divine([pointer]) expect(PayloadBuilder.omitStorageMeta(result)).toEqual([expected]) }) @@ -71,7 +71,7 @@ describe('PayloadPointerDiviner', () => { describe('combined serially', () => { it('returns Payload signed by both addresses', async () => { const expected = payloads[4] - const pointer = await createPointer([[(await accountC).address], [(await accountD).address]], [[expected.schema]]) + const pointer = createPointer([[(await accountC).address], [(await accountD).address]], [[expected.schema]]) const result = await sut.divine([pointer]) expect(PayloadBuilder.omitStorageMeta(result)).toEqual([expected]) }) @@ -79,14 +79,14 @@ describe('PayloadPointerDiviner', () => { describe('combined in parallel', () => { it('returns Payload signed by both address', async () => { const expected = payloads[4] - const pointer = await createPointer([[(await accountC).address, (await accountD).address]], [[expected.schema]]) + const pointer = createPointer([[(await accountC).address, (await accountD).address]], [[expected.schema]]) const result = await sut.divine([pointer]) expect(PayloadBuilder.omitStorageMeta(result)).toEqual([expected]) }) }) }) it('no matching address', async () => { - const pointer = await createPointer([[(await Account.random()).address]], [[payloads[0].schema]]) + const pointer = createPointer([[(await Account.random()).address]], [[payloads[0].schema]]) const result = await sut.divine([pointer]) expect(result).toEqual([]) }) diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.schema.spec.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.schema.spec.ts index 1c6ef3127d..c5aaebe921 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.schema.spec.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.schema.spec.ts @@ -7,10 +7,8 @@ import { BoundWitnessBuilder } from '@xyo-network/boundwitness-builder' import type { NodeInstance } from '@xyo-network/node-model' import { PayloadBuilder } from '@xyo-network/payload-builder' import type { Payload } from '@xyo-network/payload-model' -import { PayloadWrapper } from '@xyo-network/payload-wrapper' import { - beforeAll, - describe, expect, it, + beforeAll, describe, expect, it, } from 'vitest' import type { PayloadPointerDiviner } from '../Diviner.ts' @@ -29,18 +27,13 @@ describe('PayloadPointerDiviner', () => { const account = Account.random() const schemaA = getTestSchemaName() const schemaB = getTestSchemaName() - const payloadBaseA = (async () => { - return { - ...(await getNewPayload()), schema: schemaA, timestamp: Date.now(), - } - })() - const payloadA: Promise = (async () => PayloadWrapper.wrap(await payloadBaseA))() - const payloadBaseB = (async () => { - return { - ...(await getNewPayload()), schema: schemaB, timestamp: Date.now(), - } - })() - const payloadB: Promise = (async () => PayloadWrapper.wrap(await payloadBaseB))() + + const payloadA = { + ...getNewPayload(), schema: schemaA, salt: 1, + } + const payloadB = { + ...getNewPayload(), schema: schemaB, salt: 2, + } const schemas = [schemaA, schemaB] let node: NodeInstance let archivist: ArchivistInstance @@ -50,10 +43,10 @@ describe('PayloadPointerDiviner', () => { archivist = await getArchivist(node) sut = await getPayloadPointerDiviner(node) const [bw] = await new BoundWitnessBuilder() - .payloads([(await payloadA).payload, (await payloadB).payload]) + .payloads([payloadA, payloadB]) .witness(await account) .build() - const payloads: Payload[] = [bw, (await payloadA).payload, (await payloadB).payload] + const payloads: Payload[] = [bw, payloadA, payloadB] for (const payload of payloads) { await delay(2) const payloadResponse = await insertPayload(archivist, payload) @@ -65,9 +58,11 @@ describe('PayloadPointerDiviner', () => { [schemaA, payloadA], [schemaB, payloadB], ])('returns Payload of schema type', async (schema, expected) => { - const pointer = await createPointer([[]], [[schema]]) + const pointer = createPointer([[]], [[schema]]) const result = await sut.divine([pointer]) - expect(result).toEqual([(await expected).payload]) + expect(result).toBeArrayOfSize(1) + const [actual] = result + expect(PayloadBuilder.omitMeta(actual)).toEqual(expected) }) }) describe('single schema [w/address]', () => { @@ -75,15 +70,17 @@ describe('PayloadPointerDiviner', () => { [schemaA, payloadA], [schemaB, payloadB], ])('returns Payload of schema type', async (schema, expected) => { - const pointer = await createPointer([[(await account).address]], [[schema]]) + const pointer = createPointer([[(await account).address]], [[schema]]) const result = await sut.divine([pointer]) - expect(PayloadBuilder.omitStorageMeta(result)).toEqual([(await expected).payload]) + expect(result).toBeArrayOfSize(1) + const [actual] = result + expect(PayloadBuilder.omitMeta(actual)).toEqual(expected) }) }) describe('multiple schema rules', () => { describe('combined serially', () => { it('returns Payload of either schema', async () => { - const pointer = await createPointer([[]], [[(await payloadA).schema(), (await payloadB).schema()]]) + const pointer = createPointer([[]], [[payloadA.schema, payloadB.schema]]) const results = await sut.divine([pointer]) expect(results).toBeDefined() expect(results).toBeArrayOfSize(1) @@ -94,7 +91,7 @@ describe('PayloadPointerDiviner', () => { }) describe('combined serially [w/address]', () => { it('returns Payload of either schema', async () => { - const pointer = await createPointer([[(await account).address]], [[(await payloadA).schema(), (await payloadB).schema()]]) + const pointer = createPointer([[(await account).address]], [[payloadA.schema, payloadB.schema]]) const results = await sut.divine([pointer]) expect(results).toBeDefined() expect(results).toBeArrayOfSize(1) @@ -105,7 +102,7 @@ describe('PayloadPointerDiviner', () => { }) describe('combined in parallel', () => { it('returns Payload of either schema', async () => { - const pointer = await createPointer([[]], [[(await payloadA).schema()], [(await payloadB).schema()]]) + const pointer = createPointer([[]], [[payloadA.schema], [payloadB.schema]]) const results = await sut.divine([pointer]) expect(results).toBeDefined() expect(results).toBeArrayOfSize(1) @@ -116,7 +113,7 @@ describe('PayloadPointerDiviner', () => { }) describe('combined in parallel [w/address]', () => { it('returns Payload of either schema', async () => { - const pointer = await createPointer([[(await account).address]], [[(await payloadA).schema()], [(await payloadB).schema()]]) + const pointer = createPointer([[(await account).address]], [[payloadA.schema], [payloadB.schema]]) const results = await sut.divine([pointer]) expect(results).toBeDefined() expect(results).toBeArrayOfSize(1) @@ -127,7 +124,7 @@ describe('PayloadPointerDiviner', () => { }) }) it('no matching schema', async () => { - const pointer = await createPointer([[(await account).address]], [['network.xyo.test']]) + const pointer = createPointer([[(await account).address]], [['network.xyo.test']]) const results = await sut.divine([pointer]) expect(results).toBeDefined() expect(results).toBeEmpty() diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.timestamp.spec.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.sequence.spec.ts similarity index 84% rename from packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.timestamp.spec.ts rename to packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.sequence.spec.ts index 447589efe9..5783bdd5d8 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.timestamp.spec.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/Diviner.payloadPointer.sequence.spec.ts @@ -7,7 +7,7 @@ import { Account } from '@xyo-network/account' import type { ArchivistInstance } from '@xyo-network/archivist-model' import type { NodeInstance } from '@xyo-network/node-model' import { PayloadBuilder } from '@xyo-network/payload-builder' -import type { Payload } from '@xyo-network/payload-model' +import { type Payload, SequenceConstants } from '@xyo-network/payload-model' import { beforeAll, describe, expect, it, @@ -25,7 +25,7 @@ import { } from './testUtil/index.ts' describe('PayloadPointerDiviner', () => { - describe('with rules for [timestamp]', () => { + describe('with rules for [sequence]', () => { let account: AccountInstance let payloads: Payload[] let expectedSchema: string @@ -59,18 +59,18 @@ describe('PayloadPointerDiviner', () => { }) it('ascending', async () => { const expected = assertEx(payloads.at(0)) - const pointer = await createPointer([[account.address]], [[expectedSchema]], 0, 'asc') + const pointer = createPointer([[account.address]], [[expectedSchema]], 'asc') const result = await sut.divine([pointer]) expect(PayloadBuilder.omitStorageMeta(result)).toEqual([expected]) }) it('descending', async () => { const expected = assertEx(payloads.at(-1)) - const pointer = await createPointer([[account.address]], [[expectedSchema]], Date.now(), 'desc') + const pointer = createPointer([[account.address]], [[expectedSchema]], 'desc') const result = await sut.divine([pointer]) expect(PayloadBuilder.omitStorageMeta(result)).toEqual([expected]) }) - it('no matching timestamp', async () => { - const pointer = await createPointer([[account.address]], [[expectedSchema]], Date.now(), 'asc') + it('no matching sequence', async () => { + const pointer = createPointer([[account.address]], [[expectedSchema]], 'asc', SequenceConstants.maxLocalSequence) const result = await sut.divine([pointer]) expect(result).toEqual([]) }) diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/combineRules.spec.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/combineRules.spec.ts index 4b31a004c3..77b1108772 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/combineRules.spec.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/combineRules.spec.ts @@ -1,20 +1,20 @@ import '@xylabs/vitest-extended' import type { - PayloadAddressRule, PayloadRule, PayloadSchemaRule, PayloadTimestampOrderRule, + PayloadAddressRule, PayloadRule, PayloadSchemaRule, PayloadSequenceOrderRule, } from '@xyo-network/diviner-payload-pointer-model' +import { SequenceConstants } from '@xyo-network/payload-model' import { - describe, expect, it, vi, + describe, expect, it, } from 'vitest' import { combineRules } from '../combineRules.ts' -// Mock Date.now -const now = Date.now() -vi.spyOn(Date, 'now').mockReturnValue(now) - const validRules = (): PayloadRule[][] => { - return [[{ schema: 'network.xyo.debug' }], [{ order: 'desc', timestamp: Date.now() }]] + return [ + [{ schema: 'network.xyo.debug' }], + [{ order: 'desc', sequence: SequenceConstants.maxLocalSequence }], + ] } describe('combineRules', () => { @@ -35,14 +35,14 @@ describe('combineRules', () => { combineRules(rules) }).toThrow() }) - describe('for timestamp defaults to', () => { - it('timestamp set to current time', () => { - const rules = validRules().filter(rule => !(rule?.[0] as PayloadTimestampOrderRule)?.timestamp) + describe('for sequence defaults to', () => { + it('sequence set to current time', () => { + const rules = validRules().filter(rule => !(rule?.[0] as PayloadSequenceOrderRule)?.sequence) const actual = combineRules(rules) - expect(actual.timestamp).toBe(+now) + expect(actual.sequence).toBe(undefined) }) it('direction defaults to desc', () => { - const rules = validRules().filter(rule => !(rule?.[0] as PayloadTimestampOrderRule)?.timestamp) + const rules = validRules().filter(rule => !(rule?.[0] as PayloadSequenceOrderRule)?.sequence) const actual = combineRules(rules) expect(actual.order).toBe('desc') }) @@ -59,18 +59,21 @@ describe('combineRules', () => { }) describe('with PayloadSchemaRule rules', () => { it('combines multiple rules', () => { - const rules: PayloadRule[][] = [[{ order: 'desc', timestamp: Date.now() }], [{ schema: 'network.xyo.test' }, { schema: 'network.xyo.debug' }]] + const rules: PayloadRule[][] = [ + [{ order: 'desc', sequence: SequenceConstants.maxLocalSequence }], + [{ schema: 'network.xyo.test' }, { schema: 'network.xyo.debug' }], + ] const actual = combineRules(rules) expect(actual.schemas.sort()).toEqual(['network.xyo.debug', 'network.xyo.test']) }) }) - describe('with PayloadTimestampDirectionRule rules', () => { + describe('with multiple PayloadSequenceDirectionRule rules', () => { it('should only allow one rule', () => { const rules: PayloadRule[][] = [ [{ schema: 'network.xyo.debug' }], [ - { order: 'desc', timestamp: Date.now() }, - { order: 'asc', timestamp: Date.now() }, + { order: 'desc', sequence: SequenceConstants.maxLocalSequence }, + { order: 'asc', sequence: SequenceConstants.minLocalSequence }, ], ] expect(() => { diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Payload/getNewPayload.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Payload/getNewPayload.ts index 0c748389c9..4d4f5d8c46 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Payload/getNewPayload.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Payload/getNewPayload.ts @@ -4,11 +4,11 @@ import { v4 as uuid } from 'uuid' import { schema } from './schema.ts' -export const getNewPayload = async (): Promise => { +export const getNewPayload = (): Payload => { const fields = { schema: 'network.xyo.id', salt: uuid() } - return await new PayloadBuilder({ schema }).fields(fields).build() + return new PayloadBuilder({ schema }).fields(fields).build() } -export const getNewPayloads = async (numPayloads: number) => { - return await Promise.all(Array.from({ length: numPayloads }).fill(0).map(getNewPayload)) +export const getNewPayloads = (numPayloads: number) => { + return Array.from({ length: numPayloads }).fill(0).map(getNewPayload) } diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Pointer/createPointer.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Pointer/createPointer.ts index 30621df5de..07478732cb 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Pointer/createPointer.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/memory/src/spec/testUtil/Pointer/createPointer.ts @@ -4,17 +4,19 @@ import type { PayloadPointerPayload, PayloadRule, PayloadSchemaRule, - PayloadTimestampOrderRule, + PayloadSequenceOrderRule, } from '@xyo-network/diviner-payload-pointer-model' import { PayloadPointerSchema } from '@xyo-network/diviner-payload-pointer-model' import { PayloadBuilder } from '@xyo-network/payload-builder' +import type { Sequence } from '@xyo-network/payload-model' +import { SequenceConstants } from '@xyo-network/payload-model' -export const createPointer = async ( +export const createPointer = ( addresses: string[][] = [], schemas: string[][] = [], - timestamp = Date.now(), order: Order = 'desc', -): Promise => { + sequence?: Sequence, +): PayloadPointerPayload => { const reference: PayloadRule[][] = [] const schemaRules: PayloadSchemaRule[][] = schemas.map((rules) => { @@ -31,8 +33,9 @@ export const createPointer = async ( }) if (addressRules.length > 0) reference.push(...addressRules) - const timestampRule: PayloadTimestampOrderRule = { order, timestamp } - reference.push([timestampRule]) + const sequenceOrderRule: PayloadSequenceOrderRule = { order } + if (sequence != SequenceConstants.minLocalSequence) sequenceOrderRule.sequence = sequence + reference.push([sequenceOrderRule]) - return await new PayloadBuilder({ schema: PayloadPointerSchema }).fields({ reference }).build() + return new PayloadBuilder({ schema: PayloadPointerSchema }).fields({ reference }).build() } diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/package.json b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/package.json index 6ff672b711..43e9e7982c 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/package.json +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/package.json @@ -29,6 +29,7 @@ "module": "dist/neutral/index.mjs", "types": "dist/neutral/index.d.ts", "dependencies": { + "@xylabs/exists": "^4.4.19", "@xylabs/hex": "^4.4.19", "@xyo-network/boundwitness-model": "workspace:^", "@xyo-network/diviner-model": "workspace:^", diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadRule.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadRule.ts index 0f18b7f8d5..9a73d2309d 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadRule.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadRule.ts @@ -1,5 +1,5 @@ import type { - PayloadAddressRule, PayloadSchemaRule, PayloadTimestampOrderRule, + PayloadAddressRule, PayloadSchemaRule, PayloadSequenceOrderRule, } from './Rules/index.ts' -export type PayloadRule = PayloadAddressRule | PayloadTimestampOrderRule | PayloadSchemaRule +export type PayloadRule = PayloadAddressRule | PayloadSequenceOrderRule | PayloadSchemaRule diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadSearchCriteria.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadSearchCriteria.ts index 2918a72eab..69bdf565d6 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadSearchCriteria.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/PayloadSearchCriteria.ts @@ -1,10 +1,10 @@ import type { Address } from '@xylabs/hex' import type { Order } from '@xyo-network/diviner-payload-model' -import type { Schema } from '@xyo-network/payload-model' +import type { Schema, Sequence } from '@xyo-network/payload-model' export interface PayloadSearchCriteria { addresses: Address[] order: Order schemas: Schema[] - timestamp: number + sequence?: Sequence } diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/PayloadSequenceOrderRule.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/PayloadSequenceOrderRule.ts new file mode 100644 index 0000000000..9862896bd0 --- /dev/null +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/PayloadSequenceOrderRule.ts @@ -0,0 +1,8 @@ +import type { Order } from '@xyo-network/diviner-payload-model' +import type { Sequence } from '@xyo-network/payload-model' + +export interface PayloadSequenceOrderRule { + order?: Order + // timestamp: number + sequence?: Sequence +} diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/PayloadTimestampDirectionRule.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/PayloadTimestampDirectionRule.ts deleted file mode 100644 index bfba5f4ea2..0000000000 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/PayloadTimestampDirectionRule.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { Order } from '@xyo-network/diviner-payload-model' - -export interface PayloadTimestampOrderRule { - order?: Order - timestamp: number -} diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/index.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/index.ts index 7e76c9ec74..0bfc418e81 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/index.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/Rules/index.ts @@ -1,3 +1,3 @@ export * from './PayloadAddressRule.ts' export * from './PayloadSchemaRule.ts' -export * from './PayloadTimestampDirectionRule.ts' +export * from './PayloadSequenceOrderRule.ts' diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/index.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/index.ts index fcb1105cfc..8eb3480ec0 100644 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/index.ts +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/index.ts @@ -1,3 +1,3 @@ export * from './isPayloadAddressRule.ts' export * from './isPayloadSchemaRule.ts' -export * from './isPayloadTimestampDirectionRule.ts' +export * from './isPayloadSequenceOrderRule.ts' diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/isPayloadSequenceOrderRule.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/isPayloadSequenceOrderRule.ts new file mode 100644 index 0000000000..def2d5fd01 --- /dev/null +++ b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/isPayloadSequenceOrderRule.ts @@ -0,0 +1,17 @@ +import { exists } from '@xylabs/exists' +import { isSequence } from '@xyo-network/payload-model' + +import type { PayloadRule } from '../PayloadRule.ts' +import type { PayloadSequenceOrderRule } from '../Rules/index.ts' + +export const isPayloadSequenceOrderRule = (rule: PayloadRule): rule is PayloadSequenceOrderRule => { + const { order, sequence } = rule as Partial + // If sequence is defined, but not a sequence, it's not a PayloadSequenceOrderRule + if (exists(sequence) && !isSequence(sequence)) return false + // If neither order or sequence is defined, it's not a PayloadSequenceOrderRule + if (!exists(order) && !exists(sequence)) return false + // If order is defined, but not a valid order, it's not a PayloadSequenceOrderRule + if (order && order !== 'asc' && order !== 'desc') return false + // It's a PayloadSequenceOrderRule + return true +} diff --git a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/isPayloadTimestampDirectionRule.ts b/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/isPayloadTimestampDirectionRule.ts deleted file mode 100644 index c082834e1e..0000000000 --- a/packages/modules/packages/diviner/packages/payloadpointer/packages/model/src/Pointer/PayloadRules/TypePredicates/isPayloadTimestampDirectionRule.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { PayloadRule } from '../PayloadRule.ts' -import type { PayloadTimestampOrderRule } from '../Rules/index.ts' - -export const isPayloadTimestampOrderRule = (rule: PayloadRule): rule is PayloadTimestampOrderRule => { - return typeof (rule as PayloadTimestampOrderRule)?.timestamp === 'number' -} diff --git a/packages/protocol/packages/boundwitness/packages/model/src/isBoundWitness.ts b/packages/protocol/packages/boundwitness/packages/model/src/isBoundWitness.ts index 6330abbc77..9aceabdf42 100644 --- a/packages/protocol/packages/boundwitness/packages/model/src/isBoundWitness.ts +++ b/packages/protocol/packages/boundwitness/packages/model/src/isBoundWitness.ts @@ -1,3 +1,4 @@ +import { AsObjectFactory } from '@xylabs/object' import type { WithStorageMeta } from '@xyo-network/payload-model' import { isPayloadOfSchemaType, isStorageMeta, notPayloadOfSchemaType, @@ -9,13 +10,12 @@ import { BoundWitnessSchema } from './BoundWitness/index.ts' export const isBoundWitness = (value: unknown): value is BoundWitness => isPayloadOfSchemaType(BoundWitnessSchema)(value) export const isBoundWitnessWithStorageMeta = (value: unknown): value is WithStorageMeta => isPayloadOfSchemaType(BoundWitnessSchema)(value) && isStorageMeta(value) +export const asBoundWitnessWithStorageMeta = AsObjectFactory.create>(isBoundWitnessWithStorageMeta) export const isUnsignedBoundWitness = (value: unknown): value is UnsignedBoundWitness => isPayloadOfSchemaType(BoundWitnessSchema)(value) export const notBoundWitness = notPayloadOfSchemaType(BoundWitnessSchema) +// TODO: Use AsObjectFactory here to match standard patter export const asBoundWitness = = BoundWitness>(payload?: unknown) => isBoundWitness(payload) ? (payload as T) : undefined - -/** @deprecated use isBoundWitness instead */ -export const isBoundWitnessPayload = isBoundWitness diff --git a/yarn.lock b/yarn.lock index b8a7095640..4f6a8ff2f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5340,6 +5340,15 @@ __metadata: languageName: node linkType: hard +"@xylabs/array@npm:^4.4.21": + version: 4.4.21 + resolution: "@xylabs/array@npm:4.4.21" + dependencies: + "@xylabs/exists": "npm:^4.4.21" + checksum: 10/5a4e6f65d8c2a4d1337c438136bf02355ee0fb0fefad2e1d91122dbc35ddcdbc501fecc51fd9af2987e66afecab342633782c5f7bc8616b88cac2cb530191864 + languageName: node + linkType: hard + "@xylabs/arraybuffer@npm:^4.4.19": version: 4.4.19 resolution: "@xylabs/arraybuffer@npm:4.4.19" @@ -5357,6 +5366,13 @@ __metadata: languageName: node linkType: hard +"@xylabs/assert@npm:^4.4.21": + version: 4.4.21 + resolution: "@xylabs/assert@npm:4.4.21" + checksum: 10/2d73ab45b5b81c96e44736ca7d1bf24922a897ed9d0d7be414a7d516ac4e0fed02715cebb8f81f749e249c4ad28631411236cb16561eb87b93a72fb899053c7b + languageName: node + linkType: hard + "@xylabs/axios@npm:^4.4.19": version: 4.4.19 resolution: "@xylabs/axios@npm:4.4.19" @@ -5413,6 +5429,13 @@ __metadata: languageName: node linkType: hard +"@xylabs/exists@npm:^4.4.21": + version: 4.4.21 + resolution: "@xylabs/exists@npm:4.4.21" + checksum: 10/52c23dc3da5155d5b2dad200d354c3148fcaddef547726527c527ff95e1287e5c7053e6922d8ba674650deab9ad7ccb02b28c7a4e1148e61b700e520e652c6e9 + languageName: node + linkType: hard + "@xylabs/forget@npm:^4.4.19": version: 4.4.19 resolution: "@xylabs/forget@npm:4.4.19" @@ -6708,6 +6731,7 @@ __metadata: resolution: "@xyo-network/diviner-boundwitness-indexeddb@workspace:packages/modules/packages/diviner/packages/indexeddb/packages/boundwitness" dependencies: "@xylabs/array": "npm:^4.4.19" + "@xylabs/assert": "npm:^4.4.21" "@xylabs/exists": "npm:^4.4.19" "@xylabs/ts-scripts-yarn3": "npm:^4.2.4" "@xylabs/tsconfig": "npm:^4.2.4" @@ -7319,6 +7343,7 @@ __metadata: version: 0.0.0-use.local resolution: "@xyo-network/diviner-payload-generic@workspace:packages/modules/packages/diviner/packages/payload/packages/generic" dependencies: + "@xylabs/array": "npm:^4.4.21" "@xylabs/assert": "npm:^4.4.19" "@xylabs/delay": "npm:^4.4.19" "@xylabs/forget": "npm:^4.4.19" @@ -7452,6 +7477,7 @@ __metadata: version: 0.0.0-use.local resolution: "@xyo-network/diviner-payload-pointer-model@workspace:packages/modules/packages/diviner/packages/payloadpointer/packages/model" dependencies: + "@xylabs/exists": "npm:^4.4.19" "@xylabs/hex": "npm:^4.4.19" "@xylabs/ts-scripts-yarn3": "npm:^4.2.4" "@xylabs/tsconfig": "npm:^4.2.4"