Merge pull request #1143 from XYOracleNetwork/feature/config-indexes

Module Config Indexes
XYOracleNetwork · Oct 2, 2023 · c69bfb6 · c69bfb6
2 parents 2c63fc6 + 4aa09e1
commit c69bfb6
Show file tree

Hide file tree

Showing 14 changed files with 186 additions and 170 deletions.
diff --git a/packages/modules-mongo/packages/archivist/src/Archivist.ts b/packages/modules-mongo/packages/archivist/src/Archivist.ts
@@ -3,34 +3,13 @@ import { fulfilledValues } from '@xylabs/promise'
 import { AbstractArchivist } from '@xyo-network/archivist-abstract'
 import { ArchivistConfigSchema, ArchivistInsertQuerySchema } from '@xyo-network/archivist-model'
 import { MongoDBArchivistConfigSchema } from '@xyo-network/archivist-model-mongodb'
-import { CollectionIndexFunction, MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'
+import { MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'
 import { PayloadWithPartialMeta } from '@xyo-network/node-core-model'
 import { Payload } from '@xyo-network/payload-model'
 import { PayloadWrapper } from '@xyo-network/payload-wrapper'
-import { IndexDescription } from 'mongodb'
 
 import { toBoundWitnessWithMeta, toPayloadWithMeta, toReturnValue, validByType } from './lib'
 
-const getBoundWitnessesIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { _hash: 1 },
-      name: `${collectionName}.IX__hash`,
-    },
-  ]
-}
-
-const getPayloadsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { _hash: 1 },
-      name: `${collectionName}.IX__hash`,
-    },
-  ]
-}
-
 const MongoDBArchivistBase = MongoDBModuleMixin(AbstractArchivist)
 
 export class MongoDBArchivist extends MongoDBArchivistBase {
@@ -70,16 +49,7 @@ export class MongoDBArchivist extends MongoDBArchivistBase {
 
   protected override async startHandler() {
     await super.startHandler()
-    await this.boundWitnesses.useCollection(async (collection) => {
-      const { collectionName } = collection
-      const indexes = getBoundWitnessesIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
-    await this.payloads.useCollection(async (collection) => {
-      const { collectionName } = collection
-      const indexes = getPayloadsIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
+    await this.ensureIndexes()
     return true
   }
 }
diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBAddressHistoryDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBAddressHistoryDiviner.ts
@@ -35,6 +35,12 @@ export class MongoDBAddressHistoryDiviner extends MongoDBDivinerBase {
     return blocks.map(removeId)
   }
 
+  protected override async startHandler() {
+    await super.startHandler()
+    await this.ensureIndexes()
+    return true
+  }
+
   private getBlocks = async (hash: string, address: string, limit: number): Promise<BoundWitnessWithMeta[]> => {
     let nextHash = hash
     const blocks: BoundWitnessWithMeta[] = []

diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceBatchDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceBatchDiviner.ts
@@ -98,7 +98,9 @@ export class MongoDBAddressSpaceBatchDiviner extends MongoDBDivinerBase {
   }
 
   protected override async startHandler() {
+    await super.startHandler()
+    await this.ensureIndexes()
     void this.backgroundDivine()
-    return await super.startHandler()
+    return true
   }
 }
diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBAddressSpaceDiviner.ts
@@ -21,4 +21,10 @@ export class MongoDBAddressSpaceDiviner extends MongoDBDivinerBase {
       return { address, schema: AddressSchema }
     })
   }
+
+  protected override async startHandler() {
+    await super.startHandler()
+    await this.ensureIndexes()
+    return true
+  }
 }
diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessDiviner.ts
@@ -7,42 +7,10 @@ import {
   BoundWitnessDivinerQueryPayload,
   isBoundWitnessDivinerQueryPayload,
 } from '@xyo-network/diviner-boundwitness-model'
-import {
-  CollectionIndexFunction,
-  DefaultLimit,
-  DefaultMaxTimeMS,
-  DefaultOrder,
-  MongoDBModuleMixin,
-  removeId,
-} from '@xyo-network/module-abstract-mongodb'
+import { DefaultLimit, DefaultMaxTimeMS, DefaultOrder, MongoDBModuleMixin, removeId } from '@xyo-network/module-abstract-mongodb'
 import { BoundWitnessWithMeta } from '@xyo-network/node-core-model'
 import { Payload } from '@xyo-network/payload-model'
-import { Filter, IndexDescription, SortDirection } from 'mongodb'
-
-const getBoundWitnessesIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { _timestamp: -1, addresses: 1 },
-      name: `${collectionName}.IX__timestamp_addresses`,
-    },
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { addresses: 1, _timestamp: -1 },
-      name: `${collectionName}.IX_addresses__timestamp`,
-    },
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { addresses: 1 },
-      name: `${collectionName}.IX_addresses`,
-    },
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { payload_hashes: 1 },
-      name: `${collectionName}.IX_payload_hashes`,
-    },
-  ]
-}
+import { Filter, SortDirection } from 'mongodb'
 
 const MongoDBDivinerBase = MongoDBModuleMixin(BoundWitnessDiviner)
 
@@ -78,11 +46,7 @@ export class MongoDBBoundWitnessDiviner extends MongoDBDivinerBase {
 
   protected override async startHandler() {
     await super.startHandler()
-    await this.boundWitnesses.useCollection(async (collection) => {
-      const { collectionName } = collection
-      const indexes = getBoundWitnessesIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
+    await this.ensureIndexes()
     return true
   }
 }
diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessStatsDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBBoundWitnessStatsDiviner.ts
@@ -32,17 +32,6 @@ interface Stats {
   }
 }
 
-const getArchivistStatsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { address: 1 },
-      name: `${collectionName}.UX_address`,
-      unique: true,
-    },
-  ]
-}
-
 const MongoDBDivinerBase = MongoDBModuleMixin(BoundWitnessStatsDiviner)
 
 const moduleName = 'MongoDBBoundWitnessStatsDiviner'
@@ -98,12 +87,7 @@ export class MongoDBBoundWitnessStatsDiviner extends MongoDBDivinerBase implemen
 
   protected override async startHandler() {
     await super.startHandler()
-    await this.boundWitnesses.useMongo(async (mongo) => {
-      const collection = mongo.db(DATABASES.Archivist).collection<Stats>(COLLECTIONS.ArchivistStats)
-      const { collectionName } = collection
-      const indexes = getArchivistStatsIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
+    await this.ensureIndexes()
     await this.registerWithChangeStream()
     defineJobs(this.jobQueue, this.jobs)
     this.jobQueue.once('ready', async () => await scheduleJobs(this.jobQueue, this.jobs))

diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBPayloadDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBPayloadDiviner.ts
@@ -1,31 +1,9 @@
 import { PayloadDiviner } from '@xyo-network/diviner-payload-abstract'
 import { isPayloadDivinerQueryPayload, PayloadDivinerConfigSchema, PayloadDivinerQueryPayload } from '@xyo-network/diviner-payload-model'
-import {
-  CollectionIndexFunction,
-  DefaultLimit,
-  DefaultMaxTimeMS,
-  DefaultOrder,
-  MongoDBModuleMixin,
-  removeId,
-} from '@xyo-network/module-abstract-mongodb'
+import { DefaultLimit, DefaultMaxTimeMS, DefaultOrder, MongoDBModuleMixin, removeId } from '@xyo-network/module-abstract-mongodb'
 import { PayloadWithMeta } from '@xyo-network/node-core-model'
 import { Payload } from '@xyo-network/payload-model'
-import { Filter, IndexDescription, SortDirection } from 'mongodb'
-
-const getPayloadsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { _timestamp: 1 },
-      name: `${collectionName}.IX__timestamp`,
-    },
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { schema: 1, _timestamp: -1 },
-      name: `${collectionName}.IX_schema__timestamp`,
-    },
-  ]
-}
+import { Filter, SortDirection } from 'mongodb'
 
 const MongoDBDivinerBase = MongoDBModuleMixin(PayloadDiviner)
 
@@ -57,11 +35,7 @@ export class MongoDBPayloadDiviner extends MongoDBDivinerBase {
 
   protected override async startHandler() {
     await super.startHandler()
-    await this.payloads.useCollection(async (collection) => {
-      const { collectionName } = collection
-      const indexes = getPayloadsIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
+    await this.ensureIndexes()
     return true
   }
 }
diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBPayloadStatsDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBPayloadStatsDiviner.ts
@@ -11,14 +11,14 @@ import {
   PayloadStatsPayload,
   PayloadStatsQueryPayload,
 } from '@xyo-network/diviner-payload-stats-model'
-import { CollectionIndexFunction, COLLECTIONS, DATABASES, MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'
+import { COLLECTIONS, DATABASES, MongoDBModuleMixin } from '@xyo-network/module-abstract-mongodb'
 import { BoundWitnessWithMeta } from '@xyo-network/node-core-model'
 import { TYPES } from '@xyo-network/node-core-types'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import { Payload } from '@xyo-network/payload-model'
 import { MongoClientWrapper } from '@xyo-network/sdk-xyo-mongo-js'
 import { Job, JobProvider } from '@xyo-network/shared'
-import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, IndexDescription, ResumeToken, UpdateOptions } from 'mongodb'
+import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, ResumeToken, UpdateOptions } from 'mongodb'
 
 import { defineJobs, scheduleJobs } from './JobQueue'
 import { SetIterator } from './SetIterator'
@@ -32,17 +32,6 @@ interface Stats {
   }
 }
 
-const getArchivistStatsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { address: 1 },
-      name: `${collectionName}.UX_address`,
-      unique: true,
-    },
-  ]
-}
-
 const MongoDBDivinerBase = MongoDBModuleMixin(PayloadStatsDiviner)
 
 const moduleName = 'MongoDBPayloadStatsDiviner'
@@ -114,12 +103,7 @@ export class MongoDBPayloadStatsDiviner extends MongoDBDivinerBase implements Pa
 
   protected override async startHandler() {
     await super.startHandler()
-    await this.boundWitnesses.useMongo(async (mongo) => {
-      const collection = mongo.db(DATABASES.Archivist).collection<Stats>(COLLECTIONS.ArchivistStats)
-      const { collectionName } = collection
-      const indexes = getArchivistStatsIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
+    await this.ensureIndexes()
     await this.registerWithChangeStream()
     defineJobs(this.jobQueue, this.jobs)
     this.jobQueue.once('ready', async () => await scheduleJobs(this.jobQueue, this.jobs))

diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBSchemaListDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBSchemaListDiviner.ts
@@ -22,6 +22,12 @@ export class MongoDBSchemaListDiviner extends MongoDBDivinerBase {
     return counts.map((schemas) => new PayloadBuilder<SchemaListPayload>({ schema: SchemaListDivinerSchema }).fields({ schemas }).build())
   }
 
+  protected override async startHandler() {
+    await super.startHandler()
+    await this.ensureIndexes()
+    return true
+  }
+
   private divineAddress = async (archive: string): Promise<string[]> => {
     const result = await this.boundWitnesses.useCollection((collection) => {
       return collection.distinct('payload_schemas', { addresses: { $in: [archive] } })

diff --git a/packages/modules-mongo/packages/diviner/src/MongoDBSchemaStatsDiviner.ts b/packages/modules-mongo/packages/diviner/src/MongoDBSchemaStatsDiviner.ts
@@ -11,21 +11,14 @@ import {
   SchemaStatsPayload,
   SchemaStatsQueryPayload,
 } from '@xyo-network/diviner-schema-stats-model'
-import {
-  CollectionIndexFunction,
-  COLLECTIONS,
-  DATABASES,
-  fromDbProperty,
-  MongoDBModuleMixin,
-  toDbProperty,
-} from '@xyo-network/module-abstract-mongodb'
+import { COLLECTIONS, DATABASES, fromDbProperty, MongoDBModuleMixin, toDbProperty } from '@xyo-network/module-abstract-mongodb'
 import { BoundWitnessWithMeta } from '@xyo-network/node-core-model'
 import { TYPES } from '@xyo-network/node-core-types'
 import { PayloadBuilder } from '@xyo-network/payload-builder'
 import { Payload } from '@xyo-network/payload-model'
 import { MongoClientWrapper } from '@xyo-network/sdk-xyo-mongo-js'
 import { Job, JobProvider } from '@xyo-network/shared'
-import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, IndexDescription, ResumeToken, UpdateOptions } from 'mongodb'
+import { ChangeStream, ChangeStreamInsertDocument, ChangeStreamOptions, ResumeToken, UpdateOptions } from 'mongodb'
 
 import { defineJobs, scheduleJobs } from './JobQueue'
 import { SetIterator } from './SetIterator'
@@ -44,17 +37,6 @@ interface Stats {
   }
 }
 
-const getArchivistStatsIndexes: CollectionIndexFunction = (collectionName: string): IndexDescription[] => {
-  return [
-    {
-      // eslint-disable-next-line sort-keys-fix/sort-keys-fix
-      key: { address: 1 },
-      name: `${collectionName}.UX_address`,
-      unique: true,
-    },
-  ]
-}
-
 const MongoDBDivinerBase = MongoDBModuleMixin(SchemaStatsDiviner)
 
 const moduleName = 'MongoDBSchemaStatsDiviner'
@@ -136,12 +118,7 @@ export class MongoDBSchemaStatsDiviner extends MongoDBDivinerBase implements Job
 
   protected override async startHandler() {
     await super.startHandler()
-    await this.boundWitnesses.useMongo(async (mongo) => {
-      const collection = mongo.db(DATABASES.Archivist).collection<Stats>(COLLECTIONS.ArchivistStats)
-      const { collectionName } = collection
-      const indexes = getArchivistStatsIndexes(collectionName)
-      await collection.createIndexes(indexes)
-    })
+    await this.ensureIndexes()
     await this.registerWithChangeStream()
     defineJobs(this.jobQueue, this.jobs)
     this.jobQueue.once('ready', async () => await scheduleJobs(this.jobQueue, this.jobs))

diff --git a/packages/modules-mongo/packages/module/packages/abstract/src/IndexDescription.ts b/packages/modules-mongo/packages/module/packages/abstract/src/IndexDescription.ts
@@ -0,0 +1,26 @@
+/**
+ * The index direction (1 for ascending, -1 for descending)
+ */
+export type IndexDirection = -1 | 1
+
+/**
+ * Description of index(es) to be created on a store
+ */
+export type IndexDescription = {
+  /**
+   * The key(s) to index
+   */
+  key:
+    | {
+        [key: string]: IndexDirection
+      }
+    | Map<string, IndexDirection>
+  /**
+   * The name of the index
+   */
+  name?: string
+  /**
+   * If true, the index must enforce uniqueness on the key
+   */
+  unique?: boolean
+}