From 40040aa23ed81dbdd80eece7b183ba8e6d634dc3 Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Wed, 11 Oct 2023 18:55:58 -0700 Subject: [PATCH 1/6] Updated Sentinel Code Complete but not tested --- .../Jobs/CryptoMarket/reportCryptoPrices.ts | 2 +- .../Jobs/CryptoMarket/reportDivinerResult.ts | 2 +- .../Jobs/EthereumGas/reportDivinerResult.ts | 2 +- .../src/Jobs/EthereumGas/reportGasPrices.ts | 2 +- .../module/packages/model/src/Config.ts | 4 +- .../sentinel/packages/model/src/Config.ts | 5 +- .../sentinel/packages/model/src/Instance.ts | 8 +- .../sentinel/packages/model/src/Job.ts | 12 +++ .../sentinel/packages/model/src/Task.ts | 8 ++ .../sentinel/packages/model/src/index.ts | 2 + .../packages/sentinel/src/AbstractSentinel.ts | 58 +++++++------- .../packages/sentinel/src/Automation.ts | 4 +- .../packages/sentinel/src/MemorySentinel.ts | 79 ++++++++++--------- .../src/spec/Sentinel.resolve.spec.ts | 5 +- .../sentinel/src/spec/SentinelRunner.spec.ts | 4 +- 15 files changed, 110 insertions(+), 87 deletions(-) create mode 100644 packages/modules/packages/sentinel/packages/model/src/Job.ts create mode 100644 packages/modules/packages/sentinel/packages/model/src/Task.ts diff --git a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts index 186cb072441..c82053fccf6 100644 --- a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts +++ b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts @@ -24,7 +24,7 @@ export const reportCryptoPrices = async (provider = getProvider()): Promise mod.address), }, schema: SentinelConfigSchema, - witnesses: witnesses.map((mod) => mod.address), + tasks: witnesses.map((mod) => ({ module: mod.address })), } const account = await getAccount(WalletPaths.CryptoMarket.Sentinel.Market) const sentinel = await MemorySentinel.create({ account, config }) diff --git a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts index 119ede2a8e9..757d213b6f6 100644 --- a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts +++ b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts @@ -24,7 +24,7 @@ export const reportDivinerResult = async (payload: Payload): Promise archivists: archivists.map((mod) => mod.address), }, schema: SentinelConfigSchema, - witnesses: witnesses.map((mod) => mod.address), + tasks: witnesses.map((mod) => ({ module: mod.address })), } const sentinelAccount = await getAccount(WalletPaths.CryptoMarket.Sentinel.AssetDivinerResult) const sentinel = await MemorySentinel.create({ account: sentinelAccount, config }) diff --git a/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts b/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts index 1330b5c8d4a..4b62f521b42 100644 --- a/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts +++ b/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts @@ -24,7 +24,7 @@ export const reportDivinerResult = async (payload: Payload): Promise archivists: archivists.map((mod) => mod.address), }, schema: SentinelConfigSchema, - witnesses: witnesses.map((mod) => mod.address), + tasks: witnesses.map((mod) => ({ module: mod.address })), } const sentinelAccount = await getAccount(WalletPaths.EthereumGas.Sentinel.PriceDivinerResult) const sentinel = await MemorySentinel.create({ account: sentinelAccount, config }) diff --git a/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts b/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts index 111e3f3e6e0..84ceba82d08 100644 --- a/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts +++ b/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts @@ -24,7 +24,7 @@ export const reportGasPrices = async (provider = getProvider()): Promise mod.address), }, schema: SentinelConfigSchema, - witnesses: witnesses.map((mod) => mod.address), + tasks: witnesses.map((mod) => ({ module: mod.address })), } const account = await getAccount(WalletPaths.EthereumGas.Sentinel.Gas) const sentinel = await MemorySentinel.create({ account, config }) diff --git a/packages/modules/packages/module/packages/model/src/Config.ts b/packages/modules/packages/module/packages/model/src/Config.ts index c2370dee911..b5f1567c97e 100644 --- a/packages/modules/packages/module/packages/model/src/Config.ts +++ b/packages/modules/packages/module/packages/model/src/Config.ts @@ -8,8 +8,8 @@ export const ModuleConfigSchema: ModuleConfigSchema = 'network.xyo.module.config export type CosigningAddressSet = string[] export type SchemaString = string - -export type NameOrAddress = string +export type ModuleName = string +export type NameOrAddress = Address | ModuleName /** @deprecated */ export interface IndividualArchivistConfig { diff --git a/packages/modules/packages/sentinel/packages/model/src/Config.ts b/packages/modules/packages/sentinel/packages/model/src/Config.ts index 30c3733159d..ce7064f740b 100644 --- a/packages/modules/packages/sentinel/packages/model/src/Config.ts +++ b/packages/modules/packages/sentinel/packages/model/src/Config.ts @@ -1,14 +1,15 @@ import { ModuleConfig } from '@xyo-network/module-model' import { Payload } from '@xyo-network/payload-model' +import { SentinelTask } from './Task' + export type SentinelConfigSchema = 'network.xyo.sentinel.config' export const SentinelConfigSchema: SentinelConfigSchema = 'network.xyo.sentinel.config' export type SentinelConfig = ModuleConfig< TConfig, { - passthrough?: boolean - witnesses?: string[] + tasks: SentinelTask[] }, TConfig extends Payload ? TConfig['schema'] : SentinelConfigSchema > diff --git a/packages/modules/packages/sentinel/packages/model/src/Instance.ts b/packages/modules/packages/sentinel/packages/model/src/Instance.ts index 6b07286760f..9d35e21e1f4 100644 --- a/packages/modules/packages/sentinel/packages/model/src/Instance.ts +++ b/packages/modules/packages/sentinel/packages/model/src/Instance.ts @@ -1,17 +1,11 @@ import { ModuleInstance } from '@xyo-network/module-model' -import { Promisable } from '@xyo-network/promise' -import { WitnessInstance } from '@xyo-network/witness-model' import { SentinelModuleEventData } from './EventData' import { CustomSentinelModule, SentinelModule } from './Module' import { SentinelParams } from './Params' import { Sentinel } from './Sentinel' -export type SentinelInstance = SentinelModule & - Sentinel & - ModuleInstance & { - witnesses: () => Promisable - } +export type SentinelInstance = SentinelModule & Sentinel & ModuleInstance export type CustomSentinelInstance< TParams extends SentinelParams = SentinelParams, diff --git a/packages/modules/packages/sentinel/packages/model/src/Job.ts b/packages/modules/packages/sentinel/packages/model/src/Job.ts new file mode 100644 index 00000000000..e5fa761c3b2 --- /dev/null +++ b/packages/modules/packages/sentinel/packages/model/src/Job.ts @@ -0,0 +1,12 @@ +import { ModuleInstance } from '@xyo-network/module-model' + +import { SentinelTask } from './Task' + +export type ResolvedSentinelTask = Omit & { + /** @field the modules that performs the task */ + module: ModuleInstance +} + +export interface SentinelJob { + tasks: ResolvedSentinelTask[][] +} diff --git a/packages/modules/packages/sentinel/packages/model/src/Task.ts b/packages/modules/packages/sentinel/packages/model/src/Task.ts new file mode 100644 index 00000000000..29f94fed9e0 --- /dev/null +++ b/packages/modules/packages/sentinel/packages/model/src/Task.ts @@ -0,0 +1,8 @@ +import { NameOrAddress } from '@xyo-network/module-model' + +export interface SentinelTask { + /** @field determines what inputs are sent to each witness */ + input?: boolean | string + /** @field the modules that performs the task */ + module: NameOrAddress +} diff --git a/packages/modules/packages/sentinel/packages/model/src/index.ts b/packages/modules/packages/sentinel/packages/model/src/index.ts index f0976bbfbd0..35866070e88 100644 --- a/packages/modules/packages/sentinel/packages/model/src/index.ts +++ b/packages/modules/packages/sentinel/packages/model/src/index.ts @@ -2,8 +2,10 @@ export * from './Config' export * from './EventData' export * from './EventsModels' export * from './Instance' +export * from './Job' export * from './Module' export * from './Params' export * from './Queries' export * from './Sentinel' +export * from './Task' export * from './typeChecks' diff --git a/packages/modules/packages/sentinel/src/AbstractSentinel.ts b/packages/modules/packages/sentinel/src/AbstractSentinel.ts index 2125a9957e6..717c38487a8 100644 --- a/packages/modules/packages/sentinel/src/AbstractSentinel.ts +++ b/packages/modules/packages/sentinel/src/AbstractSentinel.ts @@ -1,5 +1,4 @@ import { assertEx } from '@xylabs/assert' -import { uniq } from '@xylabs/lodash' import { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder' import { BoundWitness, isBoundWitness, notBoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model' import { AbstractModuleInstance } from '@xyo-network/module-abstract' @@ -7,13 +6,14 @@ import { ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module-mode import { Payload } from '@xyo-network/payload-model' import { CustomSentinelInstance, + ResolvedSentinelTask, SentinelInstance, + SentinelJob, SentinelModuleEventData, SentinelParams, SentinelQueryBase, SentinelReportQuerySchema, } from '@xyo-network/sentinel-model' -import { isWitnessInstance, WitnessInstance } from '@xyo-network/witness-model' export abstract class AbstractSentinel< TParams extends SentinelParams = SentinelParams, @@ -23,6 +23,12 @@ export abstract class AbstractSentinel< implements CustomSentinelInstance { history: BoundWitness[] = [] + private _jobPromise?: Promise + + get jobPromise() { + this._jobPromise = this._jobPromise ?? this.generateJob() + return this._jobPromise + } override get queries(): string[] { return [SentinelReportQuerySchema, ...super.queries] @@ -34,10 +40,6 @@ export abstract class AbstractSentinel< } } - addWitness(address: string[]) { - this.config.witnesses = uniq([...address, ...(this.config.witnesses ?? [])]) - } - async report(inPayloads?: Payload[]): Promise { this._noOverride('report') await this.emit('reportStart', { inPayloads, module: this }) @@ -50,29 +52,29 @@ export abstract class AbstractSentinel< return payloads } - async witnesses() { - this.logger?.debug(`witnesses:config:witnesses: ${this.config?.witnesses?.length}`) - const namesOrAddresses = this.config?.witnesses - ? Array.isArray(this.config.witnesses) - ? this.config?.witnesses - : [this.config.witnesses] - : undefined - this.logger?.debug(`witnesses:namesOrAddresses: ${namesOrAddresses?.length}`) - const result = namesOrAddresses - ? [ - ...(await this.resolve({ address: namesOrAddresses }, { identity: isWitnessInstance })), - ...(await this.resolve({ name: namesOrAddresses }, { identity: isWitnessInstance })), - ] - : await this.resolve(undefined, { identity: isWitnessInstance }) - - if (namesOrAddresses && namesOrAddresses.length !== result.length) { - this.logger?.warn(`Not all witnesses found [Requested: ${namesOrAddresses.length}, Found: ${result.length}]`) + protected async generateJob() { + const job: SentinelJob = { tasks: [] } + let tasks: ResolvedSentinelTask[] = await Promise.all( + this.config.tasks.map(async (task) => ({ + input: task.input ?? false, + module: assertEx(await this.resolve(task.module), `Unable to resolve task module [${task.module}]`), + })), + ) + while (tasks.length) { + const previousTasks = job.tasks.length ? job.tasks[job.tasks.length - 1] : [] + const newList = + //add all tasks that either require no previous input or have the previous input module already added + tasks.filter( + (task) => + typeof task.input === 'boolean' || + previousTasks.find((prevTask) => prevTask.module.address === task.input || prevTask.module.config.name === task.input), + ) + assertEx(newList.length > 0, `Unable to generateJob [${tasks.length}]`) + job.tasks.push(newList) + //remove the tasks we just added + tasks = tasks.filter((task) => !newList.includes(task)) } - result.map((item) => { - this.logger?.debug(`witnesses:result: ${item.config.schema}`) - }) - - return result + return job } protected override async queryHandler( diff --git a/packages/modules/packages/sentinel/src/Automation.ts b/packages/modules/packages/sentinel/src/Automation.ts index ac48e3dc346..7e5c92330e5 100644 --- a/packages/modules/packages/sentinel/src/Automation.ts +++ b/packages/modules/packages/sentinel/src/Automation.ts @@ -1,5 +1,6 @@ import { AnyObject } from '@xyo-network/core' import { Payload } from '@xyo-network/payload-model' +import { SentinelTask } from '@xyo-network/sentinel-model' export type SentinelAutomationSchema = 'network.xyo.automation' export const SentinelAutomationSchema: SentinelAutomationSchema = 'network.xyo.automation' @@ -7,9 +8,8 @@ export const SentinelAutomationSchema: SentinelAutomationSchema = 'network.xyo.a export type SentinelBaseAutomationPayload = Payload< T & { schema: SentinelAutomationSchema + tasks?: SentinelTask[] type?: 'interval' | 'change' - /** @field The list of witnesses to invoke [all if undefined] */ - witnesses?: string[] } > diff --git a/packages/modules/packages/sentinel/src/MemorySentinel.ts b/packages/modules/packages/sentinel/src/MemorySentinel.ts index 9216125bd7b..4101fc91084 100644 --- a/packages/modules/packages/sentinel/src/MemorySentinel.ts +++ b/packages/modules/packages/sentinel/src/MemorySentinel.ts @@ -1,17 +1,17 @@ -import { assertEx } from '@xylabs/assert' import { fulfilled, rejected } from '@xylabs/promise' -import { handleError } from '@xyo-network/error' +import { Address } from '@xyo-network/core' +import { asDivinerInstance } from '@xyo-network/diviner' import { AnyConfigSchema } from '@xyo-network/module-model' import { Payload } from '@xyo-network/payload-model' import { + ResolvedSentinelTask, SentinelConfig, SentinelConfigSchema, SentinelInstance, SentinelModuleEventData, SentinelParams, - SentinelReportQuerySchema, } from '@xyo-network/sentinel-model' -import { WitnessInstance } from '@xyo-network/witness-model' +import { asWitnessInstance } from '@xyo-network/witness-model' import { AbstractSentinel } from './AbstractSentinel' @@ -23,42 +23,49 @@ export class MemorySentinel< > extends AbstractSentinel { static override configSchemas = [SentinelConfigSchema] - async reportHandler(payloads: Payload[] = []): Promise { + async reportHandler(inPayloads: Payload[] = []): Promise { await this.started('throw') - const errors: Error[] = [] - const allWitnesses = [...(await this.witnesses())] - const resultPayloads: Payload[] = [] + const job = await this.jobPromise - try { - const [generatedPayloads, generatedErrors] = await this.generateResults(allWitnesses, payloads) - resultPayloads.push(...generatedPayloads) - errors.push(...generatedErrors) - if (this.config.passthrough) { - resultPayloads.push(...payloads) - } - } catch (ex) { - handleError(ex, (error) => { - errors.push(error) - }) + let index = 0 + let previousResults: Record = {} + while (index < job.tasks.length) { + const generatedPayloads = await this.generateResults(job.tasks[index], previousResults, inPayloads) + previousResults = generatedPayloads + index++ } - - const [boundWitness] = await this.bindQueryResult({ schema: SentinelReportQuerySchema }, resultPayloads) - this.history.push(assertEx(boundWitness)) - return [boundWitness, ...resultPayloads] + return Object.values(previousResults).flat() } - private async generateResults(witnesses: WitnessInstance[], inPayloads?: Payload[]): Promise<[Payload[], Error[]]> { - const results: PromiseSettledResult[] = await Promise.allSettled(witnesses?.map((witness) => witness.observe(inPayloads))) - const payloads = results - .filter(fulfilled) - .map((result) => result.value) - .flat() - const errors = results - .filter(rejected) - .map((result) => result.reason) - .flat() - // console.log(`payloads: ${JSON.stringify(payloads, null, 2)}`) - // console.log(`errors: ${JSON.stringify(errors, null, 2)}`) - return [payloads, errors] + private async generateResults( + tasks: ResolvedSentinelTask[], + previousResults: Record, + inPayloads?: Payload[], + ): Promise> { + const results: PromiseSettledResult<[Address, Payload[]]>[] = await Promise.allSettled( + tasks?.map(async (task) => { + const witness = asWitnessInstance(task.module) + const input = task.input ?? false + if (witness) { + return [witness.address, await witness.observe(input === true ? inPayloads : input === false ? [] : previousResults[input])] + } + const diviner = asDivinerInstance(task.module) + if (diviner) { + return [diviner.address, await diviner.divine(input === true ? inPayloads : input === false ? [] : previousResults[input])] + } + throw Error('Unsupported module type') + }), + ) + const finalResult: Record = {} + results.filter(fulfilled).forEach((result) => { + const [address, payloads] = result.value + finalResult[address] = finalResult[address] ?? [] + finalResult[address].push(...payloads) + }) + const errors = results.filter(rejected).map((result) => result.reason) + if (errors.length > 0) { + throw Error('At least one module failed') + } + return finalResult } } diff --git a/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts b/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts index c9df9c2a027..6d1ba20ebf7 100644 --- a/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts @@ -49,8 +49,8 @@ describe('Sentinel', () => { archiving: { archivists: [archivistA.address, archivistB.address], }, - passthrough: true, schema: SentinelConfigSchema, + tasks: [{ module: witnessA.address }, { module: witnessB.address }], }, } const sentinel = await MemorySentinel.create(params) @@ -61,9 +61,6 @@ describe('Sentinel', () => { }) await node.register(sentinel) await node.attach(sentinel.address) - const witnesses = await sentinel.witnesses() - witnesses.forEach((witness) => console.log(`Witness: ${witness.address}`)) - expect(witnesses).toBeArrayOfSize(2) const result = await sentinel.report() result.forEach((payload) => console.log(`Result: ${payload.schema}`)) expect(result?.length).toBe(3) diff --git a/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts b/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts index 7764e3a1368..b7cf8ed4a70 100644 --- a/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts @@ -34,7 +34,7 @@ describe('SentinelRunner', () => { config = { schema: SentinelConfigSchema, - witnesses, + tasks: witnesses.map((module) => ({ module })), } sentinel = (await MemorySentinel.create({ account: await HDWallet.random(), config })) as MemorySentinel @@ -51,8 +51,8 @@ describe('SentinelRunner', () => { remaining: 1, schema: SentinelAutomationSchema, start: Date.now() - 1, + tasks: config.tasks, type: 'interval', - witnesses: config.witnesses, } const onTriggerResult: OnSentinelRunnerTriggerResult = (results) => { triggered = true From 0c560520f64485398766d08d364820d82ce3c8da Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Thu, 12 Oct 2023 11:22:38 -0700 Subject: [PATCH 2/6] Some tests fixed --- .../src/spec/Sentinel.resolve.spec.ts | 2 +- .../sentinel/src/spec/Sentinel.spec.ts | 131 ++---------------- .../sentinel/src/spec/SentinelRunner.spec.ts | 5 +- 3 files changed, 14 insertions(+), 124 deletions(-) diff --git a/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts b/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts index 6d1ba20ebf7..48bf798b3cc 100644 --- a/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts @@ -63,6 +63,6 @@ describe('Sentinel', () => { await node.attach(sentinel.address) const result = await sentinel.report() result.forEach((payload) => console.log(`Result: ${payload.schema}`)) - expect(result?.length).toBe(3) + expect(result?.length).toBe(2) }) }) diff --git a/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts b/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts index c8c9268b721..bf13a92fc7c 100644 --- a/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts @@ -1,12 +1,8 @@ -/* eslint-disable import/no-deprecated */ - import { AbstractWitness } from '@xyo-network/abstract-witness' import { HDWallet } from '@xyo-network/account' import { Archivist, ArchivistInstance, MemoryArchivist } from '@xyo-network/archivist' -import { BoundWitness, BoundWitnessSchema } from '@xyo-network/boundwitness-model' -import { BoundWitnessValidator } from '@xyo-network/boundwitness-validator' -import { BoundWitnessWrapper } from '@xyo-network/boundwitness-wrapper' import { PayloadHasher } from '@xyo-network/core' +import { IdSchema } from '@xyo-network/id-payload-plugin' import { IdWitness, IdWitnessConfigSchema } from '@xyo-network/id-plugin' import { MemoryNode } from '@xyo-network/node-memory' import { NodeSystemInfoWitness, NodeSystemInfoWitnessConfigSchema } from '@xyo-network/node-system-info-plugin' @@ -21,7 +17,7 @@ import { MemorySentinel, MemorySentinelParams } from '../MemorySentinel' */ describe('Sentinel', () => { - test('all [simple panel send]', async () => { + test('all [simple sentinel report]', async () => { const node = await MemoryNode.create({ account: await HDWallet.random() }) const archivist = await MemoryArchivist.create({ account: await HDWallet.random() }) await node.register(archivist) @@ -52,47 +48,16 @@ describe('Sentinel', () => { archivists: [archivist.address], }, schema: SentinelConfigSchema, - witnesses: witnesses.map((witness) => witness.address), + tasks: witnesses.map((witness) => ({ module: witness.address })), } const sentinel = (await MemorySentinel.create({ account: await HDWallet.random(), config })) as MemorySentinel await node.register(sentinel) await node.attach(sentinel.address) - //expect(await sentinel.archivists()).toBeArrayOfSize(1) - expect(await sentinel.witnesses()).toBeArrayOfSize(2) - const adhocWitness = (await AdhocWitness.create({ - account: await HDWallet.random(), - config: { - payload: { - schema: 'network.xyo.test.array', - testArray: [1, 2, 3], - testBoolean: true, - testNull: null, - testNullObject: { t: null, x: undefined }, - testNumber: 5, - testObject: { t: 1 }, - testSomeNullObject: { s: 1, t: null, x: undefined }, - testString: 'hi', - testUndefined: undefined, - }, - schema: AdhocWitnessConfigSchema, - }, - })) as AdhocWitness - - const adhocObserved = await adhocWitness.observe(adhocWitness.config.payload ? [adhocWitness.config.payload] : []) - const report1Result = await sentinel.report(adhocObserved) - const report1 = BoundWitnessWrapper.parse(report1Result[0]) - expect(report1.schema()).toBe(BoundWitnessSchema) - expect(report1.payloadHashes).toBeArrayOfSize(3) - const report2 = BoundWitnessWrapper.parse((await sentinel.report())[0]) - expect(report2.schema()).toBeDefined() - expect(report2.payloadHashes).toBeArrayOfSize(2) - expect(report2.hashSync() !== report1.hashSync()).toBe(true) - expect(report2.prev(sentinel.address)).toBeDefined() - //expect(report2.prev(panel.address)).toBe(report1.hash) - expect(await report1.getValid()).toBe(true) - expect(await report2.getValid()).toBe(true) + const report1Result = await sentinel.report() + const report1 = report1Result[0] + expect(report1.schema).toBe(IdSchema) }) describe('report', () => { describe('reports witnesses when supplied in', () => { @@ -100,11 +65,8 @@ describe('Sentinel', () => { let archivistB: ArchivistInstance let witnessA: AbstractWitness let witnessB: AbstractWitness - const assertPanelReport = async (panelReport: Payload[]) => { - expect(panelReport).toBeArrayOfSize(3) - const [bw, ...payloads] = panelReport - expect(await new BoundWitnessValidator(bw as BoundWitness).validate()).toBeArrayOfSize(0) - expect(payloads).toBeArrayOfSize(2) + const assertPanelReport = (panelReport: Payload[]) => { + expect(panelReport).toBeArrayOfSize(2) } const assertArchivistStateMatchesPanelReport = async (payloads: Payload[], archivists: Archivist[]) => { for (const archivist of archivists) { @@ -153,7 +115,7 @@ describe('Sentinel', () => { archivists: [archivistA.address, archivistB.address], }, schema: SentinelConfigSchema, - witnesses: [witnessA.address, witnessB.address], + tasks: [{ module: witnessA.address }, { module: witnessB.address }], }, } const sentinel = await MemorySentinel.create(params) @@ -165,78 +127,7 @@ describe('Sentinel', () => { await node.register(sentinel) await node.attach(sentinel.address) const result = await sentinel.report() - await assertPanelReport(result) - await assertArchivistStateMatchesPanelReport(result, [archivistA, archivistB]) - }) - it('config & inline', async () => { - const node = await MemoryNode.create({ account: await HDWallet.random() }) - await Promise.all( - [witnessA, archivistA, archivistB].map(async (module) => { - await node.register(module) - await node.attach(module.address) - }), - ) - const params: MemorySentinelParams = { - account: await HDWallet.random(), - config: { - archiving: { - archivists: [archivistA.address, archivistB.address], - }, - schema: SentinelConfigSchema, - witnesses: [witnessA.address], - }, - } - const sentinel = await MemorySentinel.create(params) - sentinel.on('reportEnd', (args) => { - const { outPayloads } = args as ReportEndEventArgs - console.log('reportEnd') - expect(outPayloads?.length).toBeGreaterThan(0) - }) - await node.register(sentinel) - await node.attach(sentinel.address) - const observed = await witnessB.observe() - expect(observed).toBeArrayOfSize(1) - const result = await sentinel.report(observed) - await assertPanelReport(result) - await assertArchivistStateMatchesPanelReport(result, [archivistA, archivistB]) - }) - it('inline', async () => { - const node = await MemoryNode.create({ account: await HDWallet.random() }) - await Promise.all( - [archivistA, archivistB].map(async (module) => { - await node.register(module) - await node.attach(module.address) - }), - ) - const params: MemorySentinelParams = { - account: await HDWallet.random(), - config: { - archiving: { - archivists: [archivistA.address, archivistB.address], - }, - passthrough: true, - schema: SentinelConfigSchema, - witnesses: [], - }, - } - const sentinel = await MemorySentinel.create(params) - sentinel.on('reportEnd', (args) => { - const { outPayloads } = args as ReportEndEventArgs - console.log('reportEnd') - expect(outPayloads?.length).toBeGreaterThan(0) - }) - await node.register(sentinel) - await node.attach(sentinel.address) - const observedA = await witnessA.observe() - expect(observedA).toBeArrayOfSize(1) - const observedB = await witnessB.observe() - expect(observedB).toBeArrayOfSize(1) - const result = await sentinel.report([...observedA, ...observedB]) - await assertPanelReport(result) - expect((await archivistA.get([PayloadHasher.hashSync(observedA[0])])).length).toBe(1) - expect((await archivistA.get([PayloadHasher.hashSync(observedB[0])])).length).toBe(1) - expect((await archivistB.get([PayloadHasher.hashSync(observedA[0])])).length).toBe(1) - expect((await archivistB.get([PayloadHasher.hashSync(observedB[0])])).length).toBe(1) + assertPanelReport(result) await assertArchivistStateMatchesPanelReport(result, [archivistA, archivistB]) }) it('reports errors', async () => { @@ -269,7 +160,7 @@ describe('Sentinel', () => { archivists: [archivistA.address, archivistB.address], }, schema: SentinelConfigSchema, - witnesses: [witnessA.address, witnessB.address], + tasks: [{ module: witnessA.address }, { module: witnessB.address }], }, } diff --git a/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts b/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts index b7cf8ed4a70..6cfcb556e5b 100644 --- a/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/SentinelRunner.spec.ts @@ -56,9 +56,8 @@ describe('SentinelRunner', () => { } const onTriggerResult: OnSentinelRunnerTriggerResult = (results) => { triggered = true - expect(results.length).toBe(2) - expect(results[0]?.schema).toBe(BoundWitnessSchema) - expect(results[1]?.schema).toBe(IdSchema) + expect(results.length).toBe(1) + expect(results[0]?.schema).toBe(IdSchema) } const runner = new SentinelRunner(sentinel, [intervalAutomation], onTriggerResult) From 66d815392d4e52633acbf3aa29deb723b2f77100 Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Thu, 12 Oct 2023 11:51:21 -0700 Subject: [PATCH 3/6] tests fixed --- .../sentinel/src/spec/Sentinel.spec.ts | 54 +++---------------- .../src/Witness/spec/Witness.sentinel.spec.ts | 13 +++-- .../witnesses/src/Adhoc/spec/Witness.spec.ts | 4 +- 3 files changed, 18 insertions(+), 53 deletions(-) diff --git a/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts b/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts index bf13a92fc7c..4cb4f452261 100644 --- a/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts @@ -10,6 +10,7 @@ import { Payload, PayloadSchema } from '@xyo-network/payload-model' import { ReportEndEventArgs, SentinelConfig, SentinelConfigSchema } from '@xyo-network/sentinel-model' import { AdhocWitness, AdhocWitnessConfigSchema } from '@xyo-network/witnesses' +import { SentinelWrapper } from '../../dist/node' import { MemorySentinel, MemorySentinelParams } from '../MemorySentinel' /** @@ -71,7 +72,7 @@ describe('Sentinel', () => { const assertArchivistStateMatchesPanelReport = async (payloads: Payload[], archivists: Archivist[]) => { for (const archivist of archivists) { const archivistPayloads = await archivist.all?.() - expect(archivistPayloads).toBeArrayOfSize(payloads.length) + expect(archivistPayloads).toBeArrayOfSize(payloads.length + 1) const panelPayloads = payloads.map((payload) => { return PayloadHasher.hashFields(payload) }) @@ -105,7 +106,7 @@ describe('Sentinel', () => { await Promise.all( [witnessA, witnessB, archivistA, archivistB].map(async (module) => { await node.register(module) - await node.attach(module.address) + await node.attach(module.address, true) }), ) const params: MemorySentinelParams = { @@ -126,55 +127,12 @@ describe('Sentinel', () => { }) await node.register(sentinel) await node.attach(sentinel.address) - const result = await sentinel.report() + //using a wrapper to trigger archiving + const wrapper = SentinelWrapper.wrap(sentinel, await HDWallet.random()) + const result = await wrapper.report() assertPanelReport(result) await assertArchivistStateMatchesPanelReport(result, [archivistA, archivistB]) }) - it('reports errors', async () => { - const paramsA = { - account: await HDWallet.random(), - config: { - payload: { nonce: Date.now() * 7, schema: 'network.xyo.test' }, - schema: AdhocWitnessConfigSchema, - }, - } - class FailingWitness extends AdhocWitness { - protected override async observeHandler(): Promise { - await Promise.reject(Error('observation failed')) - return [{ schema: 'fake.result' }] - } - } - const witnessA = await FailingWitness.create(paramsA) - - const node = await MemoryNode.create({ account: await HDWallet.random() }) - await Promise.all( - [witnessA, witnessB, archivistA, archivistB].map(async (module) => { - await node.register(module) - await node.attach(module.address) - }), - ) - const params: MemorySentinelParams = { - account: await HDWallet.random(), - config: { - archiving: { - archivists: [archivistA.address, archivistB.address], - }, - schema: SentinelConfigSchema, - tasks: [{ module: witnessA.address }, { module: witnessB.address }], - }, - } - - const sentinel = await MemorySentinel.create(params) - sentinel.on('reportEnd', (args) => { - const { outPayloads } = args as ReportEndEventArgs - console.log('reportEnd') - expect(outPayloads?.length).toBeGreaterThan(0) - }) - await node.register(sentinel) - await node.attach(sentinel.address) - await sentinel.report() - return - }) }) }) }) diff --git a/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts b/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts index f5a7daf0762..6dbdd739d16 100644 --- a/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts +++ b/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts @@ -3,7 +3,7 @@ import { isImageThumbnail } from '@xyo-network/image-thumbnail-payload-plugin' import { MemoryArchivist } from '@xyo-network/memory-archivist' import { MemoryNode } from '@xyo-network/node-memory' import { PayloadBuilder } from '@xyo-network/payload-builder' -import { MemorySentinel } from '@xyo-network/sentinel' +import { MemorySentinel, SentinelWrapper } from '@xyo-network/sentinel' import { UrlSchema } from '@xyo-network/url-payload-plugin' import { isTimestamp, TimestampWitness } from '@xyo-network/witness-timestamp' import { mock } from 'jest-mock-extended' @@ -12,6 +12,7 @@ import { ImageThumbnailWitness } from '../Witness' /** * @group thumbnail + * @group sentinel */ describe('Witness', () => { @@ -40,7 +41,11 @@ describe('Witness', () => { wallet: await HDWallet.random(), }) sentinel = await MemorySentinel.create({ - config: { archiving: { archivists: [archivistName] }, schema: MemorySentinel.configSchema }, + config: { + archiving: { archivists: [archivistName] }, + schema: MemorySentinel.configSchema, + tasks: [{ input: true, module: thumbnailWitness.address }, { module: timestampWitness.address }], + }, logger, wallet: await HDWallet.random(), }) @@ -64,7 +69,9 @@ describe('Witness', () => { // "data:image/svg+xml;utf8," const url = 'https://placekitten.com/200/300' const query = new PayloadBuilder({ schema: UrlSchema }).fields({ url }).build() - const values = await sentinel.report([query]) + const sentinelWrapper = SentinelWrapper.wrap(sentinel, await HDWallet.random()) + //using wrapper for archiving + const values = await sentinelWrapper.report([query]) const timestamps = values.filter(isTimestamp) expect(timestamps.length).toBe(1) const thumbnails = values.filter(isImageThumbnail) diff --git a/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts b/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts index f91c17f94dd..8b98cf5d516 100644 --- a/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts +++ b/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts @@ -13,13 +13,13 @@ describe('AdhocWitness', () => { it('uses payload schema', async () => { const witness = await AdhocWitness.create({ account: await HDWallet.random(), config }) const observation = await witness.observe([observed]) - expect(observation).toBeArrayOfSize(1) + expect(observation).toBeArrayOfSize(2) expect(observation?.[0]?.schema).toBe(observed.schema) }) it('uses payload schema with WitnessWrapper', async () => { const witness = await AdhocWitness.create({ account: await HDWallet.random(), config }) const observation = await witness.observe([observed]) - expect(observation).toBeArrayOfSize(1) + expect(observation).toBeArrayOfSize(2) expect(observation?.[0]?.schema).toBe(observed.schema) }) it('manifest [direct]', async () => { From 87292c5f9bc082aa6d45e7655b0d93934cf3a2b2 Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Thu, 12 Oct 2023 11:58:49 -0700 Subject: [PATCH 4/6] 2 more tests fixed --- .../packages/witnesses/src/Adhoc/spec/Witness.spec.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts b/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts index 8b98cf5d516..972edc0667d 100644 --- a/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts +++ b/packages/sdk/packages/witnesses/src/Adhoc/spec/Witness.spec.ts @@ -14,13 +14,8 @@ describe('AdhocWitness', () => { const witness = await AdhocWitness.create({ account: await HDWallet.random(), config }) const observation = await witness.observe([observed]) expect(observation).toBeArrayOfSize(2) - expect(observation?.[0]?.schema).toBe(observed.schema) - }) - it('uses payload schema with WitnessWrapper', async () => { - const witness = await AdhocWitness.create({ account: await HDWallet.random(), config }) - const observation = await witness.observe([observed]) - expect(observation).toBeArrayOfSize(2) - expect(observation?.[0]?.schema).toBe(observed.schema) + expect(observation?.[0]?.schema).toBe(payload.schema) + expect(observation?.[1]?.schema).toBe(observed.schema) }) it('manifest [direct]', async () => { const witness = await AdhocWitness.create({ account: await HDWallet.random(), config }) From eb4a5300c15330166f6c6b85bf3ffb072c56f465 Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Thu, 12 Oct 2023 12:05:26 -0700 Subject: [PATCH 5/6] fixed app outputs --- package.json | 2 +- packages/node/packages/core/packages/app/typedoc.json | 1 - packages/node/packages/core/packages/app/xy.config.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 16fd2c9c62c..6d39d894259 100644 --- a/package.json +++ b/package.json @@ -127,4 +127,4 @@ "yarn": "1.22.19" }, "type": "module" -} +} \ No newline at end of file diff --git a/packages/node/packages/core/packages/app/typedoc.json b/packages/node/packages/core/packages/app/typedoc.json index 8eda4f2661e..1497c9b33dd 100644 --- a/packages/node/packages/core/packages/app/typedoc.json +++ b/packages/node/packages/core/packages/app/typedoc.json @@ -1,5 +1,4 @@ { "$schema": "https://typedoc.org/schema.json", - "entryPoints": ["src/index.ts"], "tsconfig": "./tsconfig.typedoc.json" } \ No newline at end of file diff --git a/packages/node/packages/core/packages/app/xy.config.ts b/packages/node/packages/core/packages/app/xy.config.ts index ff5fd24c6f8..6ca497c84fb 100644 --- a/packages/node/packages/core/packages/app/xy.config.ts +++ b/packages/node/packages/core/packages/app/xy.config.ts @@ -3,7 +3,7 @@ const config: XyTsupConfig = { compile: { browser: {}, node: { - src: true, + src: { entry: ['src/index.ts', 'src/app.ts'] }, }, }, } From f7796dd6e163e732c7cc5e08488821975d6fd907 Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Thu, 12 Oct 2023 12:07:15 -0700 Subject: [PATCH 6/6] added app export to make build process check if they exist --- .../node/packages/core/packages/app/package.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/node/packages/core/packages/app/package.json b/packages/node/packages/core/packages/app/package.json index 276b8b92c46..3fd84521c21 100644 --- a/packages/node/packages/core/packages/app/package.json +++ b/packages/node/packages/core/packages/app/package.json @@ -23,6 +23,18 @@ } } }, + "./app": { + "node": { + "require": { + "types": "./dist/node/app.d.ts", + "default": "./dist/node/app.js" + }, + "import": { + "types": "./dist/node/app.d.mts", + "default": "./dist/node/app.mjs" + } + } + }, "./docs": { "default": "./dist/docs.json" }, @@ -60,4 +72,4 @@ "author": "Arie Trouw", "license": "LGPL-3.0", "types": "dist/node/index.d.ts" -} +} \ No newline at end of file