From 3530cc6ccbfab61403f7d3e07f976995dbb34da5 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 07:34:11 -0500 Subject: [PATCH 01/11] Index description placeholder --- .../packages/module/packages/abstract/src/Module.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts index 07c72450650..d0b740bd02b 100644 --- a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts +++ b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts @@ -9,6 +9,12 @@ import { BaseMongoSdk, BaseMongoSdkConfig } from '@xyo-network/sdk-xyo-mongo-js' import { COLLECTIONS } from './Collections' import { getBaseMongoSdkPrivateConfig } from './config' +// name?: string; +// key: { +// [key: string]: IndexDirection; +// } | Map; +// unique?: boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyAbstractModule = abstract new (...args: any[]) => Module From f654a81af985499ae2cdc40c533f78f9d1918ec5 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 07:41:45 -0500 Subject: [PATCH 02/11] Codify Index Description --- .../packages/abstract/src/IndexDescription.ts | 26 +++++++++++++++++++ .../module/packages/abstract/src/Module.ts | 6 ----- .../module/packages/abstract/src/index.ts | 1 + 3 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 packages/modules-mongo/packages/module/packages/abstract/src/IndexDescription.ts diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/IndexDescription.ts b/packages/modules-mongo/packages/module/packages/abstract/src/IndexDescription.ts new file mode 100644 index 00000000000..74fe91a9119 --- /dev/null +++ b/packages/modules-mongo/packages/module/packages/abstract/src/IndexDescription.ts @@ -0,0 +1,26 @@ +/** + * The index direction (1 for ascending, -1 for descending) + */ +export type IndexDirection = -1 | 1 + +/** + * Description of index(es) to be created on a store + */ +export type IndexDescription = { + /** + * The key(s) to index + */ + key: + | { + [key: string]: IndexDirection + } + | Map + /** + * The name of the index + */ + name?: string + /** + * If true, the index must enforce uniqueness on the key + */ + unique?: boolean +} diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts index d0b740bd02b..07c72450650 100644 --- a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts +++ b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts @@ -9,12 +9,6 @@ import { BaseMongoSdk, BaseMongoSdkConfig } from '@xyo-network/sdk-xyo-mongo-js' import { COLLECTIONS } from './Collections' import { getBaseMongoSdkPrivateConfig } from './config' -// name?: string; -// key: { -// [key: string]: IndexDirection; -// } | Map; -// unique?: boolean; - // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyAbstractModule = abstract new (...args: any[]) => Module diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/index.ts b/packages/modules-mongo/packages/module/packages/abstract/src/index.ts index 4f4becc2571..bf378289158 100644 --- a/packages/modules-mongo/packages/module/packages/abstract/src/index.ts +++ b/packages/modules-mongo/packages/module/packages/abstract/src/index.ts @@ -2,6 +2,7 @@ export * from './Collections' export * from './config' export * from './Databases' export * from './Defaults' +export * from './IndexDescription' export * from './Indexes' export * from './Module' export * from './util' From 53f3fec649b88082470bd2467fef4aeb50a8b075 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 07:46:38 -0500 Subject: [PATCH 03/11] Add archivist indexes to config --- .../express/packages/dependencies/src/node.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/node/packages/express/packages/dependencies/src/node.json b/packages/node/packages/express/packages/dependencies/src/node.json index 86dfdfa520e..0f831f16542 100644 --- a/packages/node/packages/express/packages/dependencies/src/node.json +++ b/packages/node/packages/express/packages/dependencies/src/node.json @@ -28,7 +28,19 @@ "language": "javascript", "name": "Archivist", "payloadSdkConfig": { "collection": "payloads" }, - "schema": "network.xyo.archivist.config" + "schema": "network.xyo.archivist.config", + "storage": { + "indexes": [ + { + "key": { "_hash": 1 }, + "name": "bound_witnesses.IX__hash" + }, + { + "key": { "_hash": 1 }, + "name": "payloads.IX__hash" + } + ] + } } }, { From 3c35acaa6369c2b9058ce67ff82cb809c5b6aad8 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 08:00:46 -0500 Subject: [PATCH 04/11] Ensure indexes helper method --- .../module/packages/abstract/src/Module.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts index 07c72450650..2dc4d23785b 100644 --- a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts +++ b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts @@ -8,6 +8,7 @@ import { BaseMongoSdk, BaseMongoSdkConfig } from '@xyo-network/sdk-xyo-mongo-js' import { COLLECTIONS } from './Collections' import { getBaseMongoSdkPrivateConfig } from './config' +import { IndexDescription } from './IndexDescription' // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyAbstractModule = abstract new (...args: any[]) => Module @@ -51,6 +52,25 @@ export const MongoDBModuleMixin = < this._payloadSdk = this._payloadSdk ?? new BaseMongoSdk(this.payloadSdkConfig) return assertEx(this._payloadSdk) } + + /** + * Ensures any indexes specified within the config are created. This method should be idempotent + * allowing for multiple calls without causing errors while ensuring the desired state. + */ + async ensureIndexes(): Promise { + const configIndexes = (this.config as { storage?: { indexes?: IndexDescription[] } })?.storage?.indexes ?? [] + await this.boundWitnesses.useCollection(async (collection) => { + const collectionName = collection.collectionName.toLowerCase() + const indexes = configIndexes.filter((ix) => ix?.name?.toLowerCase().startsWith(collectionName)) + await collection.createIndexes(indexes) + }) + await this.payloads.useCollection(async (collection) => { + const collectionName = collection.collectionName.toLowerCase() + const indexes = configIndexes.filter((ix) => ix?.name?.toLowerCase().startsWith(collectionName)) + await collection.createIndexes(indexes) + }) + await Promise.resolve() + } } return MongoModuleBase } From 37166e1f28fb03b61df72774ff68749ecb2af05d Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 08:04:10 -0500 Subject: [PATCH 05/11] Create indexes on start --- .../packages/archivist/src/Archivist.ts | 34 ++----------------- .../module/packages/abstract/src/Module.ts | 1 - 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/packages/modules-mongo/packages/archivist/src/Archivist.ts b/packages/modules-mongo/packages/archivist/src/Archivist.ts index f1da79ff80c..0a51725af97 100644 --- a/packages/modules-mongo/packages/archivist/src/Archivist.ts +++ b/packages/modules-mongo/packages/archivist/src/Archivist.ts @@ -3,34 +3,13 @@ import { fulfilledValues } from '@xylabs/promise' import { AbstractArchivist } from '@xyo-network/archivist-abstract' import { ArchivistConfigSchema, ArchivistInsertQuerySchema } from '@xyo-network/archivist-model' import { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb' -import { CollectionIndexFunction, MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb' +import { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb' import { PayloadWithPartialMeta } from '@xyo-network/node-core-model' import { Payload } from '@xyo-network/payload-model' import { PayloadWrapper } from '@xyo-network/payload-wrapper' -import { IndexDescription } from 'mongodb' import { toBoundWitnessWithMeta, toPayloadWithMeta, toReturnValue, validByType } from './lib' -const getBoundWitnessesIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { _hash: 1 }, - name: `${collectionName}.IX__hash`, - }, - ] -} - -const getPayloadsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { _hash: 1 }, - name: `${collectionName}.IX__hash`, - }, - ] -} - const MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist) export class MongoDBArchivist extends MongoDBArchivistBase { @@ -70,16 +49,7 @@ export class MongoDBArchivist extends MongoDBArchivistBase { protected override async startHandler() { await super.startHandler() - await this.boundWitnesses.useCollection(async (collection) => { - const { collectionName } = collection - const indexes = getBoundWitnessesIndexes(collectionName) - await collection.createIndexes(indexes) - }) - await this.payloads.useCollection(async (collection) => { - const { collectionName } = collection - const indexes = getPayloadsIndexes(collectionName) - await collection.createIndexes(indexes) - }) + await this.ensureIndexes() return true } } diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts index 2dc4d23785b..ad61a9381a2 100644 --- a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts +++ b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts @@ -69,7 +69,6 @@ export const MongoDBModuleMixin = < const indexes = configIndexes.filter((ix) => ix?.name?.toLowerCase().startsWith(collectionName)) await collection.createIndexes(indexes) }) - await Promise.resolve() } } return MongoModuleBase From d9582eff6706cd85546cb2677635a09ccb51cd1f Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 08:09:05 -0500 Subject: [PATCH 06/11] Bail early if no indexes specified --- .../packages/module/packages/abstract/src/Module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts index ad61a9381a2..f01be3e6261 100644 --- a/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts +++ b/packages/modules-mongo/packages/module/packages/abstract/src/Module.ts @@ -62,11 +62,13 @@ export const MongoDBModuleMixin = < await this.boundWitnesses.useCollection(async (collection) => { const collectionName = collection.collectionName.toLowerCase() const indexes = configIndexes.filter((ix) => ix?.name?.toLowerCase().startsWith(collectionName)) + if (indexes.length === 0) return await collection.createIndexes(indexes) }) await this.payloads.useCollection(async (collection) => { const collectionName = collection.collectionName.toLowerCase() const indexes = configIndexes.filter((ix) => ix?.name?.toLowerCase().startsWith(collectionName)) + if (indexes.length === 0) return await collection.createIndexes(indexes) }) } From 113cdc7c44f2275581c9b0f36233c9616d175920 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 08:29:52 -0500 Subject: [PATCH 07/11] Migrate BoundWitness Diviner Indexes to config --- .../src/MongoDBAddressSpaceBatchDiviner.ts | 4 +- .../diviner/src/MongoDBAddressSpaceDiviner.ts | 6 +++ .../diviner/src/MongoDBBoundWitnessDiviner.ts | 42 ++---------------- .../packages/dependencies/src/node.json | 44 ++++++++++++++++++- 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceBatchDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceBatchDiviner.ts index 0dd4404482d..af8c744027e 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceBatchDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceBatchDiviner.ts @@ -98,7 +98,9 @@ export class MongoDBAddressSpaceBatchDiviner extends MongoDBDivinerBase { } protected override async startHandler() { + await super.startHandler() + await this.ensureIndexes() void this.backgroundDivine() - return await super.startHandler() + return true } } diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceDiviner.ts index 18b7811faf7..fbaf4a61538 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceDiviner.ts @@ -21,4 +21,10 @@ export class MongoDBAddressSpaceDiviner extends MongoDBDivinerBase { return { address, schema: AddressSchema } }) } + + protected override async startHandler() { + await super.startHandler() + await this.ensureIndexes() + return true + } } diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessDiviner.ts index 79463a897fc..cef67f5f14b 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessDiviner.ts @@ -7,42 +7,10 @@ import { BoundWitnessDivinerQueryPayload, isBoundWitnessDivinerQueryPayload, } from '@xyo-network/diviner-boundwitness-model' -import { - CollectionIndexFunction, - DefaultLimit, - DefaultMaxTimeMS, - DefaultOrder, - MongoDBModuleMixin, - removeId, -} from '@xyo-network/module-abstract-mongodb' +import { DefaultLimit, DefaultMaxTimeMS, DefaultOrder, MongoDBModuleMixin, removeId } from '@xyo-network/module-abstract-mongodb' import { BoundWitnessWithMeta } from '@xyo-network/node-core-model' import { Payload } from '@xyo-network/payload-model' -import { Filter, IndexDescription, SortDirection } from 'mongodb' - -const getBoundWitnessesIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { _timestamp: -1, addresses: 1 }, - name: `${collectionName}.IX__timestamp_addresses`, - }, - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { addresses: 1, _timestamp: -1 }, - name: `${collectionName}.IX_addresses__timestamp`, - }, - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { addresses: 1 }, - name: `${collectionName}.IX_addresses`, - }, - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { payload_hashes: 1 }, - name: `${collectionName}.IX_payload_hashes`, - }, - ] -} +import { Filter, SortDirection } from 'mongodb' const MongoDBDivinerBase = MongoDBModuleMixin(BoundWitnessDiviner) @@ -78,11 +46,7 @@ export class MongoDBBoundWitnessDiviner extends MongoDBDivinerBase { protected override async startHandler() { await super.startHandler() - await this.boundWitnesses.useCollection(async (collection) => { - const { collectionName } = collection - const indexes = getBoundWitnessesIndexes(collectionName) - await collection.createIndexes(indexes) - }) + await this.ensureIndexes() return true } } diff --git a/packages/node/packages/express/packages/dependencies/src/node.json b/packages/node/packages/express/packages/dependencies/src/node.json index 0f831f16542..2a06a47476b 100644 --- a/packages/node/packages/express/packages/dependencies/src/node.json +++ b/packages/node/packages/express/packages/dependencies/src/node.json @@ -92,7 +92,27 @@ }, "language": "javascript", "name": "BoundWitnessDiviner", - "schema": "network.xyo.diviner.boundwitness.config" + "schema": "network.xyo.diviner.boundwitness.config", + "storage": { + "indexes": [ + { + "key": { "_timestamp": -1, "addresses": 1 }, + "name": "bound_witnesses.IX__timestamp_addresses" + }, + { + "key": { "addresses": 1, "_timestamp": -1 }, + "name": "bound_witnesses.IX_addresses__timestamp" + }, + { + "key": { "addresses": 1 }, + "name": "bound_witnesses.IX_addresses" + }, + { + "key": { "payload_hashes": 1 }, + "name": "bound_witnesses.IX_payload_hashes" + } + ] + } } }, { @@ -250,7 +270,27 @@ }, "language": "javascript", "name": "ThumbnailBoundWitnessDiviner", - "schema": "network.xyo.diviner.boundwitness.config" + "schema": "network.xyo.diviner.boundwitness.config", + "storage": { + "indexes": [ + { + "key": { "_timestamp": -1, "addresses": 1 }, + "name": "thumbnail_bound_witnesses.IX__timestamp_addresses" + }, + { + "key": { "addresses": 1, "_timestamp": -1 }, + "name": "thumbnail_bound_witnesses.IX_addresses__timestamp" + }, + { + "key": { "addresses": 1 }, + "name": "thumbnail_bound_witnesses.IX_addresses" + }, + { + "key": { "payload_hashes": 1 }, + "name": "thumbnail_bound_witnesses.IX_payload_hashes" + } + ] + } } }, { From 2c108d431722795c9ce0627c7ca498a9db96ca2e Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 08:32:23 -0500 Subject: [PATCH 08/11] Add start handler --- .../packages/diviner/src/MongoDBAddressHistoryDiviner.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBAddressHistoryDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBAddressHistoryDiviner.ts index 856d39005c6..3cd8bab945c 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBAddressHistoryDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBAddressHistoryDiviner.ts @@ -35,6 +35,12 @@ export class MongoDBAddressHistoryDiviner extends MongoDBDivinerBase { return blocks.map(removeId) } + protected override async startHandler() { + await super.startHandler() + await this.ensureIndexes() + return true + } + private getBlocks = async (hash: string, address: string, limit: number): Promise => { let nextHash = hash const blocks: BoundWitnessWithMeta[] = [] From 8e65f4b00a6cbb48935afd68c11cffdf452d75b0 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 08:51:18 -0500 Subject: [PATCH 09/11] Stats diviners indexes in config --- .../src/MongoDBBoundWitnessStatsDiviner.ts | 18 +--------- .../diviner/src/MongoDBPayloadStatsDiviner.ts | 22 ++----------- .../diviner/src/MongoDBSchemaStatsDiviner.ts | 29 ++-------------- .../packages/dependencies/src/node.json | 33 +++++++++++++++++-- 4 files changed, 37 insertions(+), 65 deletions(-) diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessStatsDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessStatsDiviner.ts index cfb0b6a93da..55d9474e912 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessStatsDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessStatsDiviner.ts @@ -32,17 +32,6 @@ interface Stats { } } -const getArchivistStatsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { address: 1 }, - name: `${collectionName}.UX_address`, - unique: true, - }, - ] -} - const MongoDBDivinerBase = MongoDBModuleMixin(BoundWitnessStatsDiviner) const moduleName = 'MongoDBBoundWitnessStatsDiviner' @@ -98,12 +87,7 @@ export class MongoDBBoundWitnessStatsDiviner extends MongoDBDivinerBase implemen protected override async startHandler() { await super.startHandler() - await this.boundWitnesses.useMongo(async (mongo) => { - const collection = mongo.db(DATABASES.Archivist).collection(COLLECTIONS.ArchivistStats) - const { collectionName } = collection - const indexes = getArchivistStatsIndexes(collectionName) - await collection.createIndexes(indexes) - }) + await this.ensureIndexes() await this.registerWithChangeStream() defineJobs(this.jobQueue, this.jobs) this.jobQueue.once('ready', async () => await scheduleJobs(this.jobQueue, this.jobs)) diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBPayloadStatsDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBPayloadStatsDiviner.ts index 11e34d2ffcc..39f6aef58aa 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBPayloadStatsDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBPayloadStatsDiviner.ts @@ -11,14 +11,14 @@ import { PayloadStatsPayload, PayloadStatsQueryPayload, } from '@xyo-network/diviner-payload-stats-model' -import { CollectionIndexFunction, COLLECTIONS, DATABASES, MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb' +import { COLLECTIONS, DATABASES, MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb' import { BoundWitnessWithMeta } from '@xyo-network/node-core-model' import { TYPES } from '@xyo-network/node-core-types' import { PayloadBuilder } from '@xyo-network/payload-builder' import { Payload } from '@xyo-network/payload-model' import { MongoClientWrapper } from '@xyo-network/sdk-xyo-mongo-js' import { Job, JobProvider } from '@xyo-network/shared' -import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, IndexDescription, ResumeToken, UpdateOptions } from 'mongodb' +import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, ResumeToken, UpdateOptions } from 'mongodb' import { defineJobs, scheduleJobs } from './JobQueue' import { SetIterator } from './SetIterator' @@ -32,17 +32,6 @@ interface Stats { } } -const getArchivistStatsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { address: 1 }, - name: `${collectionName}.UX_address`, - unique: true, - }, - ] -} - const MongoDBDivinerBase = MongoDBModuleMixin(PayloadStatsDiviner) const moduleName = 'MongoDBPayloadStatsDiviner' @@ -114,12 +103,7 @@ export class MongoDBPayloadStatsDiviner extends MongoDBDivinerBase implements Pa protected override async startHandler() { await super.startHandler() - await this.boundWitnesses.useMongo(async (mongo) => { - const collection = mongo.db(DATABASES.Archivist).collection(COLLECTIONS.ArchivistStats) - const { collectionName } = collection - const indexes = getArchivistStatsIndexes(collectionName) - await collection.createIndexes(indexes) - }) + await this.ensureIndexes() await this.registerWithChangeStream() defineJobs(this.jobQueue, this.jobs) this.jobQueue.once('ready', async () => await scheduleJobs(this.jobQueue, this.jobs)) diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBSchemaStatsDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBSchemaStatsDiviner.ts index d9d6e577186..1985d707e4d 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBSchemaStatsDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBSchemaStatsDiviner.ts @@ -11,21 +11,14 @@ import { SchemaStatsPayload, SchemaStatsQueryPayload, } from '@xyo-network/diviner-schema-stats-model' -import { - CollectionIndexFunction, - COLLECTIONS, - DATABASES, - fromDbProperty, - MongoDBModuleMixin, - toDbProperty, -} from '@xyo-network/module-abstract-mongodb' +import { COLLECTIONS, DATABASES, fromDbProperty, MongoDBModuleMixin, toDbProperty } from '@xyo-network/module-abstract-mongodb' import { BoundWitnessWithMeta } from '@xyo-network/node-core-model' import { TYPES } from '@xyo-network/node-core-types' import { PayloadBuilder } from '@xyo-network/payload-builder' import { Payload } from '@xyo-network/payload-model' import { MongoClientWrapper } from '@xyo-network/sdk-xyo-mongo-js' import { Job, JobProvider } from '@xyo-network/shared' -import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, IndexDescription, ResumeToken, UpdateOptions } from 'mongodb' +import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, ResumeToken, UpdateOptions } from 'mongodb' import { defineJobs, scheduleJobs } from './JobQueue' import { SetIterator } from './SetIterator' @@ -44,17 +37,6 @@ interface Stats { } } -const getArchivistStatsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { address: 1 }, - name: `${collectionName}.UX_address`, - unique: true, - }, - ] -} - const MongoDBDivinerBase = MongoDBModuleMixin(SchemaStatsDiviner) const moduleName = 'MongoDBSchemaStatsDiviner' @@ -136,12 +118,7 @@ export class MongoDBSchemaStatsDiviner extends MongoDBDivinerBase implements Job protected override async startHandler() { await super.startHandler() - await this.boundWitnesses.useMongo(async (mongo) => { - const collection = mongo.db(DATABASES.Archivist).collection(COLLECTIONS.ArchivistStats) - const { collectionName } = collection - const indexes = getArchivistStatsIndexes(collectionName) - await collection.createIndexes(indexes) - }) + await this.ensureIndexes() await this.registerWithChangeStream() defineJobs(this.jobQueue, this.jobs) this.jobQueue.once('ready', async () => await scheduleJobs(this.jobQueue, this.jobs)) diff --git a/packages/node/packages/express/packages/dependencies/src/node.json b/packages/node/packages/express/packages/dependencies/src/node.json index 2a06a47476b..c23560014bf 100644 --- a/packages/node/packages/express/packages/dependencies/src/node.json +++ b/packages/node/packages/express/packages/dependencies/src/node.json @@ -124,7 +124,16 @@ }, "language": "javascript", "name": "BoundWitnessStatsDiviner", - "schema": "network.xyo.diviner.boundwitness.stats.config" + "schema": "network.xyo.diviner.boundwitness.stats.config", + "storage": { + "indexes": [ + { + "key": { "address": 1 }, + "name": "archivist_stats.UX_address", + "unique": true + } + ] + } } }, { @@ -166,7 +175,16 @@ }, "language": "javascript", "name": "PayloadStatsDiviner", - "schema": "network.xyo.diviner.payload.stats.config" + "schema": "network.xyo.diviner.payload.stats.config", + "storage": { + "indexes": [ + { + "key": { "address": 1 }, + "name": "archivist_stats.UX_address", + "unique": true + } + ] + } } }, { @@ -190,7 +208,16 @@ }, "language": "javascript", "name": "SchemaStatsDiviner", - "schema": "network.xyo.diviner.schema.stats.config" + "schema": "network.xyo.diviner.schema.stats.config", + "storage": { + "indexes": [ + { + "key": { "address": 1 }, + "name": "archivist_stats.UX_address", + "unique": true + } + ] + } } }, From 98f0b59243db0a71c828fdc270a2b5542d1bde91 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 09:02:18 -0500 Subject: [PATCH 10/11] payload diviner indexes in config --- .../diviner/src/MongoDBPayloadDiviner.ts | 32 ++----------------- .../diviner/src/MongoDBSchemaListDiviner.ts | 6 ++++ .../packages/dependencies/src/node.json | 15 ++++++++- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBPayloadDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBPayloadDiviner.ts index 865728c5804..b53b23c62dd 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBPayloadDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBPayloadDiviner.ts @@ -1,31 +1,9 @@ import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract' import { isPayloadDivinerQueryPayload, PayloadDivinerConfigSchema, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model' -import { - CollectionIndexFunction, - DefaultLimit, - DefaultMaxTimeMS, - DefaultOrder, - MongoDBModuleMixin, - removeId, -} from '@xyo-network/module-abstract-mongodb' +import { DefaultLimit, DefaultMaxTimeMS, DefaultOrder, MongoDBModuleMixin, removeId } from '@xyo-network/module-abstract-mongodb' import { PayloadWithMeta } from '@xyo-network/node-core-model' import { Payload } from '@xyo-network/payload-model' -import { Filter, IndexDescription, SortDirection } from 'mongodb' - -const getPayloadsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => { - return [ - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { _timestamp: 1 }, - name: `${collectionName}.IX__timestamp`, - }, - { - // eslint-disable-next-line sort-keys-fix/sort-keys-fix - key: { schema: 1, _timestamp: -1 }, - name: `${collectionName}.IX_schema__timestamp`, - }, - ] -} +import { Filter, SortDirection } from 'mongodb' const MongoDBDivinerBase = MongoDBModuleMixin(PayloadDiviner) @@ -57,11 +35,7 @@ export class MongoDBPayloadDiviner extends MongoDBDivinerBase { protected override async startHandler() { await super.startHandler() - await this.payloads.useCollection(async (collection) => { - const { collectionName } = collection - const indexes = getPayloadsIndexes(collectionName) - await collection.createIndexes(indexes) - }) + await this.ensureIndexes() return true } } diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBSchemaListDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBSchemaListDiviner.ts index fce13220c33..df169ac2d3c 100644 --- a/packages/modules-mongo/packages/diviner/src/MongoDBSchemaListDiviner.ts +++ b/packages/modules-mongo/packages/diviner/src/MongoDBSchemaListDiviner.ts @@ -22,6 +22,12 @@ export class MongoDBSchemaListDiviner extends MongoDBDivinerBase { return counts.map((schemas) => new PayloadBuilder({ schema: SchemaListDivinerSchema }).fields({ schemas }).build()) } + protected override async startHandler() { + await super.startHandler() + await this.ensureIndexes() + return true + } + private divineAddress = async (archive: string): Promise => { const result = await this.boundWitnesses.useCollection((collection) => { return collection.distinct('payload_schemas', { addresses: { $in: [archive] } }) diff --git a/packages/node/packages/express/packages/dependencies/src/node.json b/packages/node/packages/express/packages/dependencies/src/node.json index c23560014bf..6804b2dc3df 100644 --- a/packages/node/packages/express/packages/dependencies/src/node.json +++ b/packages/node/packages/express/packages/dependencies/src/node.json @@ -163,7 +163,20 @@ }, "language": "javascript", "name": "PayloadDiviner", - "schema": "network.xyo.diviner.payload.config" + "payloadSdkConfig": { "collection": "payloads" }, + "schema": "network.xyo.diviner.payload.config", + "storage": { + "indexes": [ + { + "key": { "_timestamp": 1 }, + "name": "payloads.IX__timestamp" + }, + { + "key": { "schema": 1, "_timestamp": -1 }, + "name": "payloads.IX_schema__timestamp" + } + ] + } } }, { From 4aa09e149066b049ad4cec98a9c0e19422164719 Mon Sep 17 00:00:00 2001 From: Joel Carter Date: Mon, 2 Oct 2023 11:41:19 -0500 Subject: [PATCH 11/11] Add payload SDK collection to archivist stats diviners --- .../node/packages/express/packages/dependencies/src/node.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/node/packages/express/packages/dependencies/src/node.json b/packages/node/packages/express/packages/dependencies/src/node.json index 6804b2dc3df..8f2b430ced1 100644 --- a/packages/node/packages/express/packages/dependencies/src/node.json +++ b/packages/node/packages/express/packages/dependencies/src/node.json @@ -124,6 +124,7 @@ }, "language": "javascript", "name": "BoundWitnessStatsDiviner", + "payloadSdkConfig": { "collection": "archivist_stats" }, "schema": "network.xyo.diviner.boundwitness.stats.config", "storage": { "indexes": [ @@ -188,6 +189,7 @@ }, "language": "javascript", "name": "PayloadStatsDiviner", + "payloadSdkConfig": { "collection": "archivist_stats" }, "schema": "network.xyo.diviner.payload.stats.config", "storage": { "indexes": [ @@ -221,6 +223,7 @@ }, "language": "javascript", "name": "SchemaStatsDiviner", + "payloadSdkConfig": { "collection": "archivist_stats" }, "schema": "network.xyo.diviner.schema.stats.config", "storage": { "indexes": [