From ddccd121682a20aeb9618fab3cc130fbbbf62f65 Mon Sep 17 00:00:00 2001 From: Arie Trouw Date: Fri, 6 Oct 2023 16:58:13 -0700 Subject: [PATCH] archiving module functionality moved into abstract module --- .../Jobs/CryptoMarket/reportCryptoPrices.ts | 4 +- .../Jobs/CryptoMarket/reportDivinerResult.ts | 4 +- .../Jobs/EthereumGas/reportDivinerResult.ts | 4 +- .../src/Jobs/EthereumGas/reportGasPrices.ts | 4 +- .../archivist/packages/model/src/Config.ts | 9 ++-- .../archivist/src/AbstractArchivingModule.ts | 46 +++---------------- .../packages/abstract/src/AbstractModule.ts | 21 +++++++++ .../ModuleConfigQueryValidator.ts | 5 +- .../spec/ModuleConfigQueryValidator.spec.ts | 14 ++---- .../module/packages/model/src/Config.ts | 17 +++++-- .../sentinel/packages/model/src/Config.ts | 3 +- .../sentinel/packages/model/src/Instance.ts | 2 - .../packages/sentinel/src/AbstractSentinel.ts | 39 +--------------- .../src/spec/Sentinel.resolve.spec.ts | 4 +- .../sentinel/src/spec/Sentinel.spec.ts | 26 +++++++---- .../src/Witness/spec/Witness.sentinel.spec.ts | 2 +- 16 files changed, 87 insertions(+), 117 deletions(-) diff --git a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts index 0dc6fe23224..186cb072441 100644 --- a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts +++ b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportCryptoPrices.ts @@ -20,7 +20,9 @@ export const reportCryptoPrices = async (provider = getProvider()): Promise mod.address), + archiving: { + archivists: archivists.map((mod) => mod.address), + }, schema: SentinelConfigSchema, witnesses: witnesses.map((mod) => mod.address), } diff --git a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts index 2a314c13284..119ede2a8e9 100644 --- a/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts +++ b/packages/automation-witness-server/src/Jobs/CryptoMarket/reportDivinerResult.ts @@ -20,7 +20,9 @@ export const reportDivinerResult = async (payload: Payload): Promise }), ) const config: SentinelConfig = { - archivists: archivists.map((mod) => mod.address), + archiving: { + archivists: archivists.map((mod) => mod.address), + }, schema: SentinelConfigSchema, witnesses: witnesses.map((mod) => mod.address), } diff --git a/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts b/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts index 9e3f2592a76..1330b5c8d4a 100644 --- a/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts +++ b/packages/automation-witness-server/src/Jobs/EthereumGas/reportDivinerResult.ts @@ -20,7 +20,9 @@ export const reportDivinerResult = async (payload: Payload): Promise }), ) const config: SentinelConfig = { - archivists: archivists.map((mod) => mod.address), + archiving: { + archivists: archivists.map((mod) => mod.address), + }, schema: SentinelConfigSchema, witnesses: witnesses.map((mod) => mod.address), } diff --git a/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts b/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts index f5f706e0809..111e3f3e6e0 100644 --- a/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts +++ b/packages/automation-witness-server/src/Jobs/EthereumGas/reportGasPrices.ts @@ -20,7 +20,9 @@ export const reportGasPrices = async (provider = getProvider()): Promise mod.address), + archiving: { + archivists: archivists.map((mod) => mod.address), + }, schema: SentinelConfigSchema, witnesses: witnesses.map((mod) => mod.address), } diff --git a/packages/modules/packages/archivist/packages/model/src/Config.ts b/packages/modules/packages/archivist/packages/model/src/Config.ts index 795da287b24..ab0619bd381 100644 --- a/packages/modules/packages/archivist/packages/model/src/Config.ts +++ b/packages/modules/packages/archivist/packages/model/src/Config.ts @@ -1,10 +1,11 @@ -import { AddressString, ModuleConfig } from '@xyo-network/module-model' +import { Address } from '@xyo-network/core' +import { ModuleConfig } from '@xyo-network/module-model' import { Payload } from '@xyo-network/payload-model' export interface ArchivistParents { - commit?: AddressString[] - read?: AddressString[] - write?: AddressString[] + commit?: Address[] + read?: Address[] + write?: Address[] } export type ArchivistConfigSchema = 'network.xyo.archivist.config' diff --git a/packages/modules/packages/archivist/src/AbstractArchivingModule.ts b/packages/modules/packages/archivist/src/AbstractArchivingModule.ts index 81b183288ff..0e91b218b58 100644 --- a/packages/modules/packages/archivist/src/AbstractArchivingModule.ts +++ b/packages/modules/packages/archivist/src/AbstractArchivingModule.ts @@ -1,49 +1,17 @@ -import { compact } from '@xylabs/lodash' -import { AccountInstance } from '@xyo-network/account-model' -import { ArchivistInstance, asArchivistInstance } from '@xyo-network/archivist-model' -import { AbstractModuleInstance, AnyConfigSchema, Module, ModuleConfig, ModuleEventData, ModuleParams, ModuleQueryResult } from '@xyo-network/module' -import { ModuleError, Payload, Query } from '@xyo-network/payload-model' +import { AbstractModuleInstance, AnyConfigSchema, Module, ModuleConfig, ModuleEventData, ModuleParams } from '@xyo-network/module' -export type ArchivingModuleConfig = ModuleConfig< +/** @deprecated use AbstractModule which now has this functionality included */ +export type ArchivingModuleBaseConfig = ModuleConfig< { archivists?: string[] schema: string } & T > -// @creatableModule() + +/** @deprecated use AbstractModule which now has this functionality included */ export abstract class AbstractArchivingModule< - TParams extends ModuleParams> = ModuleParams>, + TParams extends ModuleParams> = ModuleParams>, TEventData extends ModuleEventData = ModuleEventData, > extends AbstractModuleInstance - implements Module -{ - protected override async bindQueryResult( - query: T, - payloads: Payload[], - additionalWitnesses: AccountInstance[] = [], - errorPayloads: ModuleError[] = [], - ): Promise { - const result = await super.bindQueryResult(query, payloads, additionalWitnesses, errorPayloads) - await this.storeToArchivists(result.flat()) - return result - } - - protected async resolveArchivists(): Promise { - const archivists = this.config.archivists - if (!archivists) return [] - const resolved = await Promise.all(archivists.map((archivist) => this.resolve(archivist))) - return compact(resolved.map((mod) => asArchivistInstance(mod))) - } - - protected async storeToArchivists(payloads: Payload[]): Promise { - const archivists = await this.resolveArchivists() - return ( - await Promise.all( - archivists.map((archivist) => { - return archivist.insert?.(payloads) - }), - ) - ).map(([bw]) => bw) - } -} + implements Module {} diff --git a/packages/modules/packages/module/packages/abstract/src/AbstractModule.ts b/packages/modules/packages/module/packages/abstract/src/AbstractModule.ts index 6ce540e7ee6..e74e8c49075 100644 --- a/packages/modules/packages/module/packages/abstract/src/AbstractModule.ts +++ b/packages/modules/packages/module/packages/abstract/src/AbstractModule.ts @@ -470,6 +470,9 @@ export abstract class AbstractModule this.getArchivist('read') + protected async resolveArchivingArchivists(): Promise { + const archivists = this.config.archiving?.archivists + if (!archivists) return [] + const resolved = await Promise.all(archivists.map((archivist) => this.resolve(archivist))) + return compact(resolved.map((mod) => asArchivistInstance(mod))) + } + protected async startHandler(): Promise { this.validateConfig() await this.initializeQueryAccounts() @@ -620,6 +630,17 @@ export abstract class AbstractModule { + const archivists = await this.resolveArchivingArchivists() + return ( + await Promise.all( + archivists.map((archivist) => { + return archivist.insert?.(payloads) + }), + ) + ).map(([bw]) => bw) + } + protected subscribeHandler() { return } diff --git a/packages/modules/packages/module/packages/abstract/src/QueryValidator/ModuleConfigQueryValidator.ts b/packages/modules/packages/module/packages/abstract/src/QueryValidator/ModuleConfigQueryValidator.ts index 74d2ce3d3b7..d2c7d1cc5cd 100644 --- a/packages/modules/packages/module/packages/abstract/src/QueryValidator/ModuleConfigQueryValidator.ts +++ b/packages/modules/packages/module/packages/abstract/src/QueryValidator/ModuleConfigQueryValidator.ts @@ -1,5 +1,6 @@ import { QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder' -import { AddressString, AnyConfigSchema, CosigningAddressSet, ModuleConfig, ModuleQuery, SchemaString } from '@xyo-network/module-model' +import { Address } from '@xyo-network/core' +import { AnyConfigSchema, CosigningAddressSet, ModuleConfig, ModuleQuery, SchemaString } from '@xyo-network/module-model' import { Queryable, QueryValidator } from './QueryValidator' @@ -9,7 +10,7 @@ const delimiter = '' export class ModuleConfigQueryValidator> implements QueryValidator { protected allowed: Record = {} - protected disallowed: Record = {} + protected disallowed: Record = {} protected readonly hasAllowedRules: boolean protected readonly hasDisallowedRules: boolean protected readonly hasRules: boolean diff --git a/packages/modules/packages/module/packages/abstract/src/QueryValidator/spec/ModuleConfigQueryValidator.spec.ts b/packages/modules/packages/module/packages/abstract/src/QueryValidator/spec/ModuleConfigQueryValidator.spec.ts index 57ca96a1be9..a2cf1050e7d 100644 --- a/packages/modules/packages/module/packages/abstract/src/QueryValidator/spec/ModuleConfigQueryValidator.spec.ts +++ b/packages/modules/packages/module/packages/abstract/src/QueryValidator/spec/ModuleConfigQueryValidator.spec.ts @@ -1,13 +1,7 @@ import { Account } from '@xyo-network/account' import { QueryBoundWitnessBuilder } from '@xyo-network/boundwitness-builder' -import { - AddressString, - CosigningAddressSet, - ModuleConfig, - ModuleConfigSchema, - ModuleDiscoverQuerySchema, - SchemaString, -} from '@xyo-network/module-model' +import { Address } from '@xyo-network/core' +import { CosigningAddressSet, ModuleConfig, ModuleConfigSchema, ModuleDiscoverQuerySchema, SchemaString } from '@xyo-network/module-model' import { PayloadBuilder } from '@xyo-network/payload-builder' import { ModuleConfigQueryValidator } from '../ModuleConfigQueryValidator' @@ -23,9 +17,9 @@ describe('ModuleConfigQueryValidator', () => { const disallowed2 = Account.randomSync() const other = Account.randomSync() const queryPayload = new PayloadBuilder({ schema }).build() - const allowed: Record = {} + const allowed: Record = {} allowed[ModuleDiscoverQuerySchema] = [allowed1.address.toUpperCase(), allowed2.address, [allowedCosigner1.address, allowedCosigner2.address]] - const disallowed: Record = {} + const disallowed: Record = {} disallowed[ModuleDiscoverQuerySchema] = [disallowed1.address.toUpperCase(), disallowed2.address] describe('queryable', () => { describe('allowed', () => { diff --git a/packages/modules/packages/module/packages/model/src/Config.ts b/packages/modules/packages/module/packages/model/src/Config.ts index 2ee64f834b9..c2370dee911 100644 --- a/packages/modules/packages/module/packages/model/src/Config.ts +++ b/packages/modules/packages/module/packages/model/src/Config.ts @@ -1,4 +1,4 @@ -import { AnyObject, WithAdditional } from '@xyo-network/core' +import { Address, AnyObject, WithAdditional } from '@xyo-network/core' import { Payload, Schema } from '@xyo-network/payload-model' import { Labels } from './Labels' @@ -6,20 +6,27 @@ import { Labels } from './Labels' export type ModuleConfigSchema = 'network.xyo.module.config' export const ModuleConfigSchema: ModuleConfigSchema = 'network.xyo.module.config' -export type AddressString = string export type CosigningAddressSet = string[] export type SchemaString = string export type NameOrAddress = string +/** @deprecated */ export interface IndividualArchivistConfig { readonly commit?: NameOrAddress readonly read?: NameOrAddress readonly write?: NameOrAddress } +/** @deprecated */ export type ArchivistModuleConfig = NameOrAddress | IndividualArchivistConfig +export interface ArchivingModuleConfig { + readonly archiving?: { + readonly archivists?: Address[] + } +} + export type ModuleConfig< TConfig extends Payload | void = void, TAdditionalParams extends AnyObject | void = void, @@ -53,10 +60,10 @@ export type ModuleConfig< readonly allowAnonymous?: boolean /** @field If schema in record, then only these address sets can access query */ - readonly allowed?: Record + readonly allowed?: Record /** @field If schema in record, then anyone except these addresses can access query */ - readonly disallowed?: Record + readonly disallowed?: Record } /** @field sign every query */ @@ -67,7 +74,7 @@ export type ModuleConfig< /** @field add a timestamp payload to every query */ readonly timestamp?: boolean - }, + } & ArchivingModuleConfig, TConfig >, TAdditionalParams diff --git a/packages/modules/packages/sentinel/packages/model/src/Config.ts b/packages/modules/packages/sentinel/packages/model/src/Config.ts index f05c3737dad..4f53211324e 100644 --- a/packages/modules/packages/sentinel/packages/model/src/Config.ts +++ b/packages/modules/packages/sentinel/packages/model/src/Config.ts @@ -1,4 +1,3 @@ -import { ArchivingModuleConfig } from '@xyo-network/archivist' import { ModuleConfig } from '@xyo-network/module-model' import { Payload } from '@xyo-network/payload-model' @@ -7,7 +6,7 @@ export const SentinelConfigSchema: SentinelConfigSchema = 'network.xyo.sentinel. export type SentinelConfig = ModuleConfig< TConfig, - ArchivingModuleConfig & { + { witnesses?: string[] }, TConfig extends Payload ? TConfig['schema'] : SentinelConfigSchema diff --git a/packages/modules/packages/sentinel/packages/model/src/Instance.ts b/packages/modules/packages/sentinel/packages/model/src/Instance.ts index 937589f5071..6b07286760f 100644 --- a/packages/modules/packages/sentinel/packages/model/src/Instance.ts +++ b/packages/modules/packages/sentinel/packages/model/src/Instance.ts @@ -1,4 +1,3 @@ -import { ArchivistInstance } from '@xyo-network/archivist' import { ModuleInstance } from '@xyo-network/module-model' import { Promisable } from '@xyo-network/promise' import { WitnessInstance } from '@xyo-network/witness-model' @@ -11,7 +10,6 @@ import { Sentinel } from './Sentinel' export type SentinelInstance = SentinelModule & Sentinel & ModuleInstance & { - archivists: () => Promisable witnesses: () => Promisable } diff --git a/packages/modules/packages/sentinel/src/AbstractSentinel.ts b/packages/modules/packages/sentinel/src/AbstractSentinel.ts index 84b2a226054..6a09c1ee7fe 100644 --- a/packages/modules/packages/sentinel/src/AbstractSentinel.ts +++ b/packages/modules/packages/sentinel/src/AbstractSentinel.ts @@ -1,9 +1,8 @@ import { assertEx } from '@xylabs/assert' import { uniq } from '@xylabs/lodash' -import { AbstractArchivingModule, asArchivistInstance } from '@xyo-network/archivist' import { QueryBoundWitness, QueryBoundWitnessWrapper } from '@xyo-network/boundwitness-builder' import { BoundWitness, isBoundWitness, notBoundWitness } from '@xyo-network/boundwitness-model' -import { ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module' +import { AbstractModuleInstance, ModuleConfig, ModuleQueryHandlerResult } from '@xyo-network/module' import { Payload } from '@xyo-network/payload-model' import { CustomSentinelInstance, @@ -19,7 +18,7 @@ export abstract class AbstractSentinel< TParams extends SentinelParams = SentinelParams, TEventData extends SentinelModuleEventData> = SentinelModuleEventData>, > - extends AbstractArchivingModule + extends AbstractModuleInstance implements CustomSentinelInstance { history: BoundWitness[] = [] @@ -34,44 +33,10 @@ export abstract class AbstractSentinel< } } - addArchivist(address: string[]) { - this.config.archivists = uniq([...address, ...(this.config.archivists ?? [])]) - } - addWitness(address: string[]) { this.config.witnesses = uniq([...address, ...(this.config.witnesses ?? [])]) } - async archivists() { - this.logger?.debug(`archivists:config:archivist: ${this.config?.archivists?.length}`) - const namesOrAddresses = this.config?.archivists - ? Array.isArray(this.config.archivists) - ? this.config?.archivists - : [this.config.archivists] - : undefined - this.logger?.debug(`archivist:namesOrAddresses: ${namesOrAddresses?.length}`) - const result = [ - ...(await this.resolve(namesOrAddresses ? { address: namesOrAddresses } : undefined)), - ...(await this.resolve(namesOrAddresses ? { name: namesOrAddresses } : undefined)), - ].map((module) => assertEx(asArchivistInstance(module), 'Tried to resolve a non-archivist as an archivist')) - - if (namesOrAddresses && namesOrAddresses.length !== result.length) { - this.logger?.warn(`Not all archivists found [Requested: ${namesOrAddresses.length}, Found: ${result.length}]`) - } - - this.logger?.debug(`archivists:result: ${result?.length}`) - - return result - } - - removeArchivist(address: string[]) { - this.config.archivists = (this.config.archivists ?? []).filter((archivist) => !address.includes(archivist)) - } - - removeWitness(address: string[]) { - this.config.witnesses = (this.config.witnesses ?? []).filter((witness) => !address.includes(witness)) - } - async report(inPayloads?: Payload[]): Promise { this._noOverride('report') await this.emit('reportStart', { inPayloads, module: this }) diff --git a/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts b/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts index 955e3663130..616cb563eef 100644 --- a/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/Sentinel.resolve.spec.ts @@ -46,7 +46,9 @@ describe('Sentinel', () => { const params: MemorySentinelParams = { account: await HDWallet.random(), config: { - archivists: [archivistA.address, archivistB.address], + archiving: { + archivists: [archivistA.address, archivistB.address], + }, schema: SentinelConfigSchema, }, } diff --git a/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts b/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts index 21bdc8573d5..02dab9b87c4 100644 --- a/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts +++ b/packages/modules/packages/sentinel/src/spec/Sentinel.spec.ts @@ -48,7 +48,9 @@ describe('Sentinel', () => { ) const config: SentinelConfig = { - archivists: [archivist.address], + archiving: { + archivists: [archivist.address], + }, schema: SentinelConfigSchema, witnesses: witnesses.map((witness) => witness.address), } @@ -56,7 +58,7 @@ describe('Sentinel', () => { const sentinel = (await MemorySentinel.create({ account: await HDWallet.random(), config })) as MemorySentinel await node.register(sentinel) await node.attach(sentinel.address) - expect(await sentinel.archivists()).toBeArrayOfSize(1) + //expect(await sentinel.archivists()).toBeArrayOfSize(1) expect(await sentinel.witnesses()).toBeArrayOfSize(2) const adhocWitness = (await AdhocWitness.create({ account: await HDWallet.random(), @@ -147,8 +149,9 @@ describe('Sentinel', () => { const params: MemorySentinelParams = { account: await HDWallet.random(), config: { - archivists: [archivistA.address, archivistB.address], - + archiving: { + archivists: [archivistA.address, archivistB.address], + }, schema: SentinelConfigSchema, witnesses: [witnessA.address, witnessB.address], }, @@ -176,8 +179,9 @@ describe('Sentinel', () => { const params: MemorySentinelParams = { account: await HDWallet.random(), config: { - archivists: [archivistA.address, archivistB.address], - + archiving: { + archivists: [archivistA.address, archivistB.address], + }, schema: SentinelConfigSchema, witnesses: [witnessA.address], }, @@ -207,8 +211,9 @@ describe('Sentinel', () => { const params: MemorySentinelParams = { account: await HDWallet.random(), config: { - archivists: [archivistA.address, archivistB.address], - + archiving: { + archivists: [archivistA.address, archivistB.address], + }, schema: SentinelConfigSchema, witnesses: [], }, @@ -259,8 +264,9 @@ describe('Sentinel', () => { const params: MemorySentinelParams = { account: await HDWallet.random(), config: { - archivists: [archivistA.address, archivistB.address], - + archiving: { + archivists: [archivistA.address, archivistB.address], + }, schema: SentinelConfigSchema, witnesses: [witnessA.address, witnessB.address], }, diff --git a/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts b/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts index 5f59e0ece7c..995dbc1a4a3 100644 --- a/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts +++ b/packages/plugins/packages/payloadset/packages/image-thumbnail/src/Witness/spec/Witness.sentinel.spec.ts @@ -36,7 +36,7 @@ describe('Witness', () => { wallet: await HDWallet.random(), }) sentinel = await MemorySentinel.create({ - config: { archivists: [archivistName], schema: MemorySentinel.configSchema }, + config: { archiving: { archivists: [archivistName] }, schema: MemorySentinel.configSchema }, logger, wallet: await HDWallet.random(), })