archiving module functionality moved into abstract module

packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts

@@ -20,7 +20,9 @@ export const reportCryptoPrices = async (provider = getProvider()): Promise<Payl
     }),
   )
   const config: SentinelConfig = {
-    archivists: archivists.map((mod) => mod.address),
+    archiving: {
+      archivists: archivists.map((mod) => mod.address),
+    },
     schema: SentinelConfigSchema,
     witnesses: witnesses.map((mod) => mod.address),
   }

packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts

@@ -20,7 +20,9 @@ export const reportDivinerResult = async (payload: Payload): Promise<Payload[]>
     }),
   )
   const config: SentinelConfig = {
-    archivists: archivists.map((mod) => mod.address),
+    archiving: {
+      archivists: archivists.map((mod) => mod.address),
+    },
     schema: SentinelConfigSchema,
     witnesses: witnesses.map((mod) => mod.address),
   }

packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts

@@ -20,7 +20,9 @@ export const reportDivinerResult = async (payload: Payload): Promise<Payload[]>
     }),
   )
   const config: SentinelConfig = {
-    archivists: archivists.map((mod) => mod.address),
+    archiving: {
+      archivists: archivists.map((mod) => mod.address),
+    },
     schema: SentinelConfigSchema,
     witnesses: witnesses.map((mod) => mod.address),
   }

packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts

@@ -20,7 +20,9 @@ export const reportGasPrices = async (provider = getProvider()): Promise<Payload
     }),
   )
   const config: SentinelConfig = {
-    archivists: archivists.map((mod) => mod.address),
+    archiving: {
+      archivists: archivists.map((mod) => mod.address),
+    },
     schema: SentinelConfigSchema,
     witnesses: witnesses.map((mod) => mod.address),
   }

packages/modules/packages/archivist/packages/model/src/Config.ts

@@ -1,10 +1,11 @@
-import { AddressString, ModuleConfig } from '@xyo-network/module-model'
+import { Address } from '@xyo-network/core'
+import { ModuleConfig } from '@xyo-network/module-model'
 import { Payload } from '@xyo-network/payload-model'
 
 export interface ArchivistParents {
-  commit?: AddressString[]
-  read?: AddressString[]
-  write?: AddressString[]
+  commit?: Address[]
+  read?: Address[]
+  write?: Address[]
 }
 
 export type ArchivistConfigSchema = 'network.xyo.archivist.config'

packages/modules/packages/archivist/src/AbstractArchivingModule.ts

@@ -1,49 +1,17 @@
-import { compact } from '@xylabs/lodash'
-import { AccountInstance } from '@xyo-network/account-model'
-import { ArchivistInstance, asArchivistInstance } from '@xyo-network/archivist-model'
-import { AbstractModuleInstance, AnyConfigSchema, Module, ModuleConfig, ModuleEventData, ModuleParams, ModuleQueryResult } from '@xyo-network/module'
-import { ModuleError, Payload, Query } from '@xyo-network/payload-model'
+import { AbstractModuleInstance, AnyConfigSchema, Module, ModuleConfig, ModuleEventData, ModuleParams } from '@xyo-network/module'
 
-export type ArchivingModuleConfig<T extends ModuleConfig = ModuleConfig> = ModuleConfig<
+/** @deprecated use AbstractModule which now has this functionality included */
+export type ArchivingModuleBaseConfig<T extends ModuleConfig = ModuleConfig> = ModuleConfig<
   {
     archivists?: string[]
     schema: string
   } & T
 >
-// @creatableModule()
+
+/** @deprecated use AbstractModule which now has this functionality included */
 export abstract class AbstractArchivingModule<
-    TParams extends ModuleParams<AnyConfigSchema<ArchivingModuleConfig>> = ModuleParams<AnyConfigSchema<ArchivingModuleConfig>>,
+    TParams extends ModuleParams<AnyConfigSchema<ArchivingModuleBaseConfig>> = ModuleParams<AnyConfigSchema<ArchivingModuleBaseConfig>>,
     TEventData extends ModuleEventData = ModuleEventData,
   >
   extends AbstractModuleInstance<TParams, TEventData>
-  implements Module<TParams, TEventData>
-{
-  protected override async bindQueryResult<T extends Query>(
-    query: T,
-    payloads: Payload[],
-    additionalWitnesses: AccountInstance[] = [],
-    errorPayloads: ModuleError[] = [],
-  ): Promise<ModuleQueryResult> {
-    const result = await super.bindQueryResult(query, payloads, additionalWitnesses, errorPayloads)
-    await this.storeToArchivists(result.flat())
-    return result
-  }
-
-  protected async resolveArchivists(): Promise<ArchivistInstance[]> {
-    const archivists = this.config.archivists
-    if (!archivists) return []
-    const resolved = await Promise.all(archivists.map((archivist) => this.resolve(archivist)))
-    return compact(resolved.map((mod) => asArchivistInstance(mod)))
-  }
-
-  protected async storeToArchivists(payloads: Payload[]): Promise<Payload[]> {
-    const archivists = await this.resolveArchivists()
-    return (
-      await Promise.all(
-        archivists.map((archivist) => {
-          return archivist.insert?.(payloads)
-        }),
-      )
-    ).map(([bw]) => bw)
-  }
-}
+  implements Module<TParams, TEventData> {}

packages/modules/packages/module/packages/abstract/src/AbstractModule.ts

@@ -470,6 +470,9 @@ export abstract class AbstractModule<TParams extends ModuleParams = ModuleParams
     const witnesses = [this.account, queryWitnessAccount, ...additionalWitnesses].filter(exists)
     builder.witnesses(witnesses)
     const result: ModuleQueryResult = [(await builder.build())[0], payloads, errors ?? []]
+    if (this.config.archiving) {
+      await this.storeToArchivists(result.flat())
+    }
     return result
   }
 
@@ -608,6 +611,13 @@ export abstract class AbstractModule<TParams extends ModuleParams = ModuleParams
 
   protected readArchivist = () => this.getArchivist('read')
 
+  protected async resolveArchivingArchivists(): Promise<ArchivistInstance[]> {
+    const archivists = this.config.archiving?.archivists
+    if (!archivists) return []
+    const resolved = await Promise.all(archivists.map((archivist) => this.resolve(archivist)))
+    return compact(resolved.map((mod) => asArchivistInstance(mod)))
+  }
+
   protected async startHandler(): Promise<boolean> {
     this.validateConfig()
     await this.initializeQueryAccounts()
@@ -620,6 +630,17 @@ export abstract class AbstractModule<TParams extends ModuleParams = ModuleParams
     return true
   }
 
+  protected async storeToArchivists(payloads: Payload[]): Promise<Payload[]> {
+    const archivists = await this.resolveArchivingArchivists()
+    return (
+      await Promise.all(
+        archivists.map((archivist) => {
+          return archivist.insert?.(payloads)
+        }),
+      )
+    ).map(([bw]) => bw)
+  }
+
   protected subscribeHandler() {
     return
   }

packages/modules/packages/module/packages/abstract/src/QueryValidator/ModuleConfigQueryValidator.ts

@@ -1,5 +1,6 @@
 import { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder'
-import { AddressString, AnyConfigSchema, CosigningAddressSet, ModuleConfig, ModuleQuery, SchemaString } from '@xyo-network/module-model'
+import { Address } from '@xyo-network/core'
+import { AnyConfigSchema, CosigningAddressSet, ModuleConfig, ModuleQuery, SchemaString } from '@xyo-network/module-model'
 
 import { Queryable, QueryValidator } from './QueryValidator'
 
@@ -9,7 +10,7 @@ const delimiter = ''
 
 export class ModuleConfigQueryValidator<TConfig extends AnyConfigSchema<ModuleConfig>> implements QueryValidator {
   protected allowed: Record<SchemaString, SortedPipedAddressesString[]> = {}
-  protected disallowed: Record<SchemaString, AddressString[]> = {}
+  protected disallowed: Record<SchemaString, Address[]> = {}
   protected readonly hasAllowedRules: boolean
   protected readonly hasDisallowedRules: boolean
   protected readonly hasRules: boolean

packages/modules/packages/module/packages/abstract/src/QueryValidator/spec/ModuleConfigQueryValidator.spec.ts

@@ -1,13 +1,7 @@
 import { Account } from '@xyo-network/account'
 import { QueryBoundWitnessBuilder } from '@xyo-network/boundwitness-builder'
-import {
-  AddressString,
-  CosigningAddressSet,
-  ModuleConfig,
-  ModuleConfigSchema,
-  ModuleDiscoverQuerySchema,
-  SchemaString,
-} from '@xyo-network/module-model'
+import { Address } from '@xyo-network/core'
+import { CosigningAddressSet, ModuleConfig, ModuleConfigSchema, ModuleDiscoverQuerySchema, SchemaString } from '@xyo-network/module-model'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 
 import { ModuleConfigQueryValidator } from '../ModuleConfigQueryValidator'
@@ -23,9 +17,9 @@ describe('ModuleConfigQueryValidator', () => {
   const disallowed2 = Account.randomSync()
   const other = Account.randomSync()
   const queryPayload = new PayloadBuilder({ schema }).build()
-  const allowed: Record<SchemaString, (AddressString | CosigningAddressSet)[]> = {}
+  const allowed: Record<SchemaString, (Address | CosigningAddressSet)[]> = {}
   allowed[ModuleDiscoverQuerySchema] = [allowed1.address.toUpperCase(), allowed2.address, [allowedCosigner1.address, allowedCosigner2.address]]
-  const disallowed: Record<SchemaString, AddressString[]> = {}
+  const disallowed: Record<SchemaString, Address[]> = {}
   disallowed[ModuleDiscoverQuerySchema] = [disallowed1.address.toUpperCase(), disallowed2.address]
   describe('queryable', () => {
     describe('allowed', () => {

packages/modules/packages/module/packages/model/src/Config.ts

@@ -1,25 +1,32 @@
-import { AnyObject, WithAdditional } from '@xyo-network/core'
+import { Address, AnyObject, WithAdditional } from '@xyo-network/core'
 import { Payload, Schema } from '@xyo-network/payload-model'
 
 import { Labels } from './Labels'
 
 export type ModuleConfigSchema = 'network.xyo.module.config'
 export const ModuleConfigSchema: ModuleConfigSchema = 'network.xyo.module.config'
 
-export type AddressString = string
 export type CosigningAddressSet = string[]
 export type SchemaString = string
 
 export type NameOrAddress = string
 
+/** @deprecated */
 export interface IndividualArchivistConfig {
   readonly commit?: NameOrAddress
   readonly read?: NameOrAddress
   readonly write?: NameOrAddress
 }
 
+/** @deprecated */
 export type ArchivistModuleConfig = NameOrAddress | IndividualArchivistConfig
 
+export interface ArchivingModuleConfig {
+  readonly archiving?: {
+    readonly archivists?: Address[]
+  }
+}
+
 export type ModuleConfig<
   TConfig extends Payload | void = void,
   TAdditionalParams extends AnyObject | void = void,
@@ -53,10 +60,10 @@ export type ModuleConfig<
           readonly allowAnonymous?: boolean
 
           /** @field If schema in record, then only these address sets can access query */
-          readonly allowed?: Record<SchemaString, (AddressString | CosigningAddressSet)[]>
+          readonly allowed?: Record<SchemaString, (Address | CosigningAddressSet)[]>
 
           /** @field If schema in record, then anyone except these addresses can access query */
-          readonly disallowed?: Record<SchemaString, AddressString[]>
+          readonly disallowed?: Record<SchemaString, Address[]>
         }
 
         /** @field sign every query */
@@ -67,7 +74,7 @@ export type ModuleConfig<
 
         /** @field add a timestamp payload to every query  */
         readonly timestamp?: boolean
-      },
+      } & ArchivingModuleConfig,
       TConfig
     >,
     TAdditionalParams

packages/modules/packages/sentinel/packages/model/src/Config.ts

@@ -1,4 +1,3 @@
-import { ArchivingModuleConfig } from '@xyo-network/archivist'
 import { ModuleConfig } from '@xyo-network/module-model'
 import { Payload } from '@xyo-network/payload-model'
 
@@ -7,7 +6,7 @@ export const SentinelConfigSchema: SentinelConfigSchema = 'network.xyo.sentinel.
 
 export type SentinelConfig<TConfig extends Payload | void = void> = ModuleConfig<
   TConfig,
-  ArchivingModuleConfig & {
+  {
     witnesses?: string[]
   },
   TConfig extends Payload ? TConfig['schema'] : SentinelConfigSchema

packages/modules/packages/sentinel/packages/model/src/Instance.ts

@@ -1,4 +1,3 @@
-import { ArchivistInstance } from '@xyo-network/archivist'
 import { ModuleInstance } from '@xyo-network/module-model'
 import { Promisable } from '@xyo-network/promise'
 import { WitnessInstance } from '@xyo-network/witness-model'
@@ -11,7 +10,6 @@ import { Sentinel } from './Sentinel'
 export type SentinelInstance<TParams extends SentinelParams = SentinelParams> = SentinelModule<TParams> &
   Sentinel &
   ModuleInstance<TParams> & {
-    archivists: () => Promisable<ArchivistInstance[]>
     witnesses: () => Promisable<WitnessInstance[]>
   }
 

packages/modules/packages/sentinel/src/AbstractSentinel.ts

@@ -1,9 +1,8 @@
 import { assertEx } from '@xylabs/assert'
 import { uniq } from '@xylabs/lodash'
-import { AbstractArchivingModule, asArchivistInstance } from '@xyo-network/archivist'
 import { QueryBoundWitness, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder'
 import { BoundWitness, isBoundWitness, notBoundWitness } from '@xyo-network/boundwitness-model'
-import { ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module'
+import { AbstractModuleInstance, ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module'
 import { Payload } from '@xyo-network/payload-model'
 import {
   CustomSentinelInstance,
@@ -19,7 +18,7 @@ export abstract class AbstractSentinel<
     TParams extends SentinelParams = SentinelParams,
     TEventData extends SentinelModuleEventData<SentinelInstance<TParams>> = SentinelModuleEventData<SentinelInstance<TParams>>,
   >
-  extends AbstractArchivingModule<TParams, TEventData>
+  extends AbstractModuleInstance<TParams, TEventData>
   implements CustomSentinelInstance<TParams, TEventData>
 {
   history: BoundWitness[] = []
@@ -34,44 +33,10 @@ export abstract class AbstractSentinel<
     }
   }
 
-  addArchivist(address: string[]) {
-    this.config.archivists = uniq([...address, ...(this.config.archivists ?? [])])
-  }
-
   addWitness(address: string[]) {
     this.config.witnesses = uniq([...address, ...(this.config.witnesses ?? [])])
   }
 
-  async archivists() {
-    this.logger?.debug(`archivists:config:archivist: ${this.config?.archivists?.length}`)
-    const namesOrAddresses = this.config?.archivists
-      ? Array.isArray(this.config.archivists)
-        ? this.config?.archivists
-        : [this.config.archivists]
-      : undefined
-    this.logger?.debug(`archivist:namesOrAddresses: ${namesOrAddresses?.length}`)
-    const result = [
-      ...(await this.resolve(namesOrAddresses ? { address: namesOrAddresses } : undefined)),
-      ...(await this.resolve(namesOrAddresses ? { name: namesOrAddresses } : undefined)),
-    ].map((module) => assertEx(asArchivistInstance(module), 'Tried to resolve a non-archivist as an archivist'))
-
-    if (namesOrAddresses && namesOrAddresses.length !== result.length) {
-      this.logger?.warn(`Not all archivists found [Requested: ${namesOrAddresses.length}, Found: ${result.length}]`)
-    }
-
-    this.logger?.debug(`archivists:result: ${result?.length}`)
-
-    return result
-  }
-
-  removeArchivist(address: string[]) {
-    this.config.archivists = (this.config.archivists ?? []).filter((archivist) => !address.includes(archivist))
-  }
-
-  removeWitness(address: string[]) {
-    this.config.witnesses = (this.config.witnesses ?? []).filter((witness) => !address.includes(witness))
-  }
-
   async report(inPayloads?: Payload[]): Promise<Payload[]> {
     this._noOverride('report')
     await this.emit('reportStart', { inPayloads, module: this })

packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts

@@ -46,7 +46,9 @@ describe('Sentinel', () => {
     const params: MemorySentinelParams<SentinelConfig> = {
       account: await HDWallet.random(),
       config: {
-        archivists: [archivistA.address, archivistB.address],
+        archiving: {
+          archivists: [archivistA.address, archivistB.address],
+        },
         schema: SentinelConfigSchema,
       },
     }