Merge branch 'feature/sentinel-upgrade' into main

package.json

@@ -127,4 +127,4 @@
     "yarn": "1.22.19"
   },
   "type": "module"
-}
+}

packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts

@@ -24,7 +24,7 @@ export const reportCryptoPrices = async (provider = getProvider()): Promise<Payl
       archivists: archivists.map((mod) => mod.address),
     },
     schema: SentinelConfigSchema,
-    witnesses: witnesses.map((mod) => mod.address),
+    tasks: witnesses.map((mod) => ({ module: mod.address })),
   }
   const account = await getAccount(WalletPaths.CryptoMarket.Sentinel.Market)
   const sentinel = await MemorySentinel.create({ account, config })

packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts

@@ -24,7 +24,7 @@ export const reportDivinerResult = async (payload: Payload): Promise<Payload[]>
       archivists: archivists.map((mod) => mod.address),
     },
     schema: SentinelConfigSchema,
-    witnesses: witnesses.map((mod) => mod.address),
+    tasks: witnesses.map((mod) => ({ module: mod.address })),
   }
   const sentinelAccount = await getAccount(WalletPaths.CryptoMarket.Sentinel.AssetDivinerResult)
   const sentinel = await MemorySentinel.create({ account: sentinelAccount, config })

packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts

@@ -24,7 +24,7 @@ export const reportDivinerResult = async (payload: Payload): Promise<Payload[]>
       archivists: archivists.map((mod) => mod.address),
     },
     schema: SentinelConfigSchema,
-    witnesses: witnesses.map((mod) => mod.address),
+    tasks: witnesses.map((mod) => ({ module: mod.address })),
   }
   const sentinelAccount = await getAccount(WalletPaths.EthereumGas.Sentinel.PriceDivinerResult)
   const sentinel = await MemorySentinel.create({ account: sentinelAccount, config })

packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts

@@ -24,7 +24,7 @@ export const reportGasPrices = async (provider = getProvider()): Promise<Payload
       archivists: archivists.map((mod) => mod.address),
     },
     schema: SentinelConfigSchema,
-    witnesses: witnesses.map((mod) => mod.address),
+    tasks: witnesses.map((mod) => ({ module: mod.address })),
   }
   const account = await getAccount(WalletPaths.EthereumGas.Sentinel.Gas)
   const sentinel = await MemorySentinel.create({ account, config })

packages/modules/packages/module/packages/model/src/Config.ts

@@ -8,8 +8,8 @@ export const ModuleConfigSchema: ModuleConfigSchema = 'network.xyo.module.config
 
 export type CosigningAddressSet = string[]
 export type SchemaString = string
-
-export type NameOrAddress = string
+export type ModuleName = string
+export type NameOrAddress = Address | ModuleName
 
 /** @deprecated */
 export interface IndividualArchivistConfig {

packages/modules/packages/sentinel/packages/model/src/Config.ts

@@ -1,14 +1,15 @@
 import { ModuleConfig } from '@xyo-network/module-model'
 import { Payload } from '@xyo-network/payload-model'
 
+import { SentinelTask } from './Task'
+
 export type SentinelConfigSchema = 'network.xyo.sentinel.config'
 export const SentinelConfigSchema: SentinelConfigSchema = 'network.xyo.sentinel.config'
 
 export type SentinelConfig<TConfig extends Payload | void = void> = ModuleConfig<
   TConfig,
   {
-    passthrough?: boolean
-    witnesses?: string[]
+    tasks: SentinelTask[]
   },
   TConfig extends Payload ? TConfig['schema'] : SentinelConfigSchema
 >

packages/modules/packages/sentinel/packages/model/src/Instance.ts

@@ -1,17 +1,11 @@
 import { ModuleInstance } from '@xyo-network/module-model'
-import { Promisable } from '@xyo-network/promise'
-import { WitnessInstance } from '@xyo-network/witness-model'
 
 import { SentinelModuleEventData } from './EventData'
 import { CustomSentinelModule, SentinelModule } from './Module'
 import { SentinelParams } from './Params'
 import { Sentinel } from './Sentinel'
 
-export type SentinelInstance<TParams extends SentinelParams = SentinelParams> = SentinelModule<TParams> &
-  Sentinel &
-  ModuleInstance<TParams> & {
-    witnesses: () => Promisable<WitnessInstance[]>
-  }
+export type SentinelInstance<TParams extends SentinelParams = SentinelParams> = SentinelModule<TParams> & Sentinel & ModuleInstance<TParams>
 
 export type CustomSentinelInstance<
   TParams extends SentinelParams = SentinelParams,

packages/modules/packages/sentinel/packages/model/src/Job.ts

@@ -0,0 +1,12 @@
+import { ModuleInstance } from '@xyo-network/module-model'
+
+import { SentinelTask } from './Task'
+
+export type ResolvedSentinelTask = Omit<SentinelTask, 'module'> & {
+  /** @field the modules that performs the task */
+  module: ModuleInstance
+}
+
+export interface SentinelJob {
+  tasks: ResolvedSentinelTask[][]
+}

packages/modules/packages/sentinel/packages/model/src/Task.ts

@@ -0,0 +1,8 @@
+import { NameOrAddress } from '@xyo-network/module-model'
+
+export interface SentinelTask {
+  /** @field determines what inputs are sent to each witness */
+  input?: boolean | string
+  /** @field the modules that performs the task */
+  module: NameOrAddress
+}

packages/modules/packages/sentinel/packages/model/src/index.ts

@@ -2,8 +2,10 @@ export * from './Config'
 export * from './EventData'
 export * from './EventsModels'
 export * from './Instance'
+export * from './Job'
 export * from './Module'
 export * from './Params'
 export * from './Queries'
 export * from './Sentinel'
+export * from './Task'
 export * from './typeChecks'

packages/modules/packages/sentinel/src/AbstractSentinel.ts

@@ -1,19 +1,19 @@
 import { assertEx } from '@xylabs/assert'
-import { uniq } from '@xylabs/lodash'
 import { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder'
 import { BoundWitness, isBoundWitness, notBoundWitness, QueryBoundWitness } from '@xyo-network/boundwitness-model'
 import { AbstractModuleInstance } from '@xyo-network/module-abstract'
 import { ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module-model'
 import { Payload } from '@xyo-network/payload-model'
 import {
   CustomSentinelInstance,
+  ResolvedSentinelTask,
   SentinelInstance,
+  SentinelJob,
   SentinelModuleEventData,
   SentinelParams,
   SentinelQueryBase,
   SentinelReportQuerySchema,
 } from '@xyo-network/sentinel-model'
-import { isWitnessInstance, WitnessInstance } from '@xyo-network/witness-model'
 
 export abstract class AbstractSentinel<
     TParams extends SentinelParams = SentinelParams,
@@ -23,6 +23,12 @@ export abstract class AbstractSentinel<
   implements CustomSentinelInstance<TParams, TEventData>
 {
   history: BoundWitness[] = []
+  private _jobPromise?: Promise<SentinelJob>
+
+  get jobPromise() {
+    this._jobPromise = this._jobPromise ?? this.generateJob()
+    return this._jobPromise
+  }
 
   override get queries(): string[] {
     return [SentinelReportQuerySchema, ...super.queries]
@@ -34,10 +40,6 @@ export abstract class AbstractSentinel<
     }
   }
 
-  addWitness(address: string[]) {
-    this.config.witnesses = uniq([...address, ...(this.config.witnesses ?? [])])
-  }
-
   async report(inPayloads?: Payload[]): Promise<Payload[]> {
     this._noOverride('report')
     await this.emit('reportStart', { inPayloads, module: this })
@@ -50,29 +52,29 @@ export abstract class AbstractSentinel<
     return payloads
   }
 
-  async witnesses() {
-    this.logger?.debug(`witnesses:config:witnesses: ${this.config?.witnesses?.length}`)
-    const namesOrAddresses = this.config?.witnesses
-      ? Array.isArray(this.config.witnesses)
-        ? this.config?.witnesses
-        : [this.config.witnesses]
-      : undefined
-    this.logger?.debug(`witnesses:namesOrAddresses: ${namesOrAddresses?.length}`)
-    const result = namesOrAddresses
-      ? [
-          ...(await this.resolve<WitnessInstance>({ address: namesOrAddresses }, { identity: isWitnessInstance })),
-          ...(await this.resolve<WitnessInstance>({ name: namesOrAddresses }, { identity: isWitnessInstance })),
-        ]
-      : await this.resolve<WitnessInstance>(undefined, { identity: isWitnessInstance })
-
-    if (namesOrAddresses && namesOrAddresses.length !== result.length) {
-      this.logger?.warn(`Not all witnesses found [Requested: ${namesOrAddresses.length}, Found: ${result.length}]`)
+  protected async generateJob() {
+    const job: SentinelJob = { tasks: [] }
+    let tasks: ResolvedSentinelTask[] = await Promise.all(
+      this.config.tasks.map(async (task) => ({
+        input: task.input ?? false,
+        module: assertEx(await this.resolve(task.module), `Unable to resolve task module [${task.module}]`),
+      })),
+    )
+    while (tasks.length) {
+      const previousTasks = job.tasks.length ? job.tasks[job.tasks.length - 1] : []
+      const newList =
+        //add all tasks that either require no previous input or have the previous input module already added
+        tasks.filter(
+          (task) =>
+            typeof task.input === 'boolean' ||
+            previousTasks.find((prevTask) => prevTask.module.address === task.input || prevTask.module.config.name === task.input),
+        )
+      assertEx(newList.length > 0, `Unable to generateJob [${tasks.length}]`)
+      job.tasks.push(newList)
+      //remove the tasks we just added
+      tasks = tasks.filter((task) => !newList.includes(task))
     }
-    result.map((item) => {
-      this.logger?.debug(`witnesses:result: ${item.config.schema}`)
-    })
-
-    return result
+    return job
   }
 
   protected override async queryHandler<T extends QueryBoundWitness = QueryBoundWitness, TConfig extends ModuleConfig = ModuleConfig>(

packages/modules/packages/sentinel/src/Automation.ts

@@ -1,15 +1,15 @@
 import { AnyObject } from '@xyo-network/core'
 import { Payload } from '@xyo-network/payload-model'
+import { SentinelTask } from '@xyo-network/sentinel-model'
 
 export type SentinelAutomationSchema = 'network.xyo.automation'
 export const SentinelAutomationSchema: SentinelAutomationSchema = 'network.xyo.automation'
 
 export type SentinelBaseAutomationPayload<T extends AnyObject = AnyObject> = Payload<
   T & {
     schema: SentinelAutomationSchema
+    tasks?: SentinelTask[]
     type?: 'interval' | 'change'
-    /** @field The list of witnesses to invoke [all if undefined] */
-    witnesses?: string[]
   }
 >
 

packages/modules/packages/sentinel/src/MemorySentinel.ts

@@ -1,17 +1,17 @@
-import { assertEx } from '@xylabs/assert'
 import { fulfilled, rejected } from '@xylabs/promise'
-import { handleError } from '@xyo-network/error'
+import { Address } from '@xyo-network/core'
+import { asDivinerInstance } from '@xyo-network/diviner'
 import { AnyConfigSchema } from '@xyo-network/module-model'
 import { Payload } from '@xyo-network/payload-model'
 import {
+  ResolvedSentinelTask,
   SentinelConfig,
   SentinelConfigSchema,
   SentinelInstance,
   SentinelModuleEventData,
   SentinelParams,
-  SentinelReportQuerySchema,
 } from '@xyo-network/sentinel-model'
-import { WitnessInstance } from '@xyo-network/witness-model'
+import { asWitnessInstance } from '@xyo-network/witness-model'
 
 import { AbstractSentinel } from './AbstractSentinel'
 
@@ -23,42 +23,49 @@ export class MemorySentinel<
 > extends AbstractSentinel<TParams, TEventData> {
   static override configSchemas = [SentinelConfigSchema]
 
-  async reportHandler(payloads: Payload[] = []): Promise<Payload[]> {
+  async reportHandler(inPayloads: Payload[] = []): Promise<Payload[]> {
     await this.started('throw')
-    const errors: Error[] = []
-    const allWitnesses = [...(await this.witnesses())]
-    const resultPayloads: Payload[] = []
+    const job = await this.jobPromise
 
-    try {
-      const [generatedPayloads, generatedErrors] = await this.generateResults(allWitnesses, payloads)
-      resultPayloads.push(...generatedPayloads)
-      errors.push(...generatedErrors)
-      if (this.config.passthrough) {
-        resultPayloads.push(...payloads)
-      }
-    } catch (ex) {
-      handleError(ex, (error) => {
-        errors.push(error)
-      })
+    let index = 0
+    let previousResults: Record<Address, Payload[]> = {}
+    while (index < job.tasks.length) {
+      const generatedPayloads = await this.generateResults(job.tasks[index], previousResults, inPayloads)
+      previousResults = generatedPayloads
+      index++
     }
-
-    const [boundWitness] = await this.bindQueryResult({ schema: SentinelReportQuerySchema }, resultPayloads)
-    this.history.push(assertEx(boundWitness))
-    return [boundWitness, ...resultPayloads]
+    return Object.values(previousResults).flat()
   }
 
-  private async generateResults(witnesses: WitnessInstance[], inPayloads?: Payload[]): Promise<[Payload[], Error[]]> {
-    const results: PromiseSettledResult<Payload[]>[] = await Promise.allSettled(witnesses?.map((witness) => witness.observe(inPayloads)))
-    const payloads = results
-      .filter(fulfilled)
-      .map((result) => result.value)
-      .flat()
-    const errors = results
-      .filter(rejected)
-      .map((result) => result.reason)
-      .flat()
-    // console.log(`payloads: ${JSON.stringify(payloads, null, 2)}`)
-    // console.log(`errors: ${JSON.stringify(errors, null, 2)}`)
-    return [payloads, errors]
+  private async generateResults(
+    tasks: ResolvedSentinelTask[],
+    previousResults: Record<Address, Payload[]>,
+    inPayloads?: Payload[],
+  ): Promise<Record<Address, Payload[]>> {
+    const results: PromiseSettledResult<[Address, Payload[]]>[] = await Promise.allSettled(
+      tasks?.map(async (task) => {
+        const witness = asWitnessInstance(task.module)
+        const input = task.input ?? false
+        if (witness) {
+          return [witness.address, await witness.observe(input === true ? inPayloads : input === false ? [] : previousResults[input])]
+        }
+        const diviner = asDivinerInstance(task.module)
+        if (diviner) {
+          return [diviner.address, await diviner.divine(input === true ? inPayloads : input === false ? [] : previousResults[input])]
+        }
+        throw Error('Unsupported module type')
+      }),
+    )
+    const finalResult: Record<Address, Payload[]> = {}
+    results.filter(fulfilled).forEach((result) => {
+      const [address, payloads] = result.value
+      finalResult[address] = finalResult[address] ?? []
+      finalResult[address].push(...payloads)
+    })
+    const errors = results.filter(rejected).map((result) => result.reason)
+    if (errors.length > 0) {
+      throw Error('At least one module failed')
+    }
+    return finalResult
   }
 }

packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts

@@ -49,8 +49,8 @@ describe('Sentinel', () => {
         archiving: {
           archivists: [archivistA.address, archivistB.address],
         },
-        passthrough: true,
         schema: SentinelConfigSchema,
+        tasks: [{ module: witnessA.address }, { module: witnessB.address }],
       },
     }
     const sentinel = await MemorySentinel.create(params)
@@ -61,11 +61,8 @@ describe('Sentinel', () => {
     })
     await node.register(sentinel)
     await node.attach(sentinel.address)
-    const witnesses = await sentinel.witnesses()
-    witnesses.forEach((witness) => console.log(`Witness: ${witness.address}`))
-    expect(witnesses).toBeArrayOfSize(2)
     const result = await sentinel.report()
     result.forEach((payload) => console.log(`Result: ${payload.schema}`))
-    expect(result?.length).toBe(3)
+    expect(result?.length).toBe(2)
   })
 })