From 82c8b904f99d7c446a34f4b7a06bd1ff496ee96b Mon Sep 17 00:00:00 2001
From: kirillgroshkov <kirill.groshkov@naturalcycles.com>
Date: Mon, 22 Apr 2024 16:45:53 +0200
Subject: [PATCH] feat: default stream concurrency to 32 (up from 16)

---
 src/adapter/file/localFile.persistence.plugin.ts | 2 +-
 src/commondao/common.dao.model.ts                | 2 +-
 src/commondao/common.dao.ts                      | 6 +++---
 src/kv/commonKeyValueDao.ts                      | 4 ++--
 yarn.lock                                        | 6 +++---
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/adapter/file/localFile.persistence.plugin.ts b/src/adapter/file/localFile.persistence.plugin.ts
index 15e6031..a8be645 100644
--- a/src/adapter/file/localFile.persistence.plugin.ts
+++ b/src/adapter/file/localFile.persistence.plugin.ts
@@ -71,7 +71,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
   }
 
   async saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void> {
-    await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 16 })
+    await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 32 })
   }
 
   async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
diff --git a/src/commondao/common.dao.model.ts b/src/commondao/common.dao.model.ts
index bd1d184..b8e7a69 100644
--- a/src/commondao/common.dao.model.ts
+++ b/src/commondao/common.dao.model.ts
@@ -321,7 +321,7 @@ export interface CommonDaoStreamOptions<IN>
 
   /**
    * When chunkSize is set - this option controls how many chunks to run concurrently.
-   * Defaults to 16, "the magic number of JavaScript concurrency".
+   * Defaults to 32.
    */
   chunkConcurrency?: number
 }
diff --git a/src/commondao/common.dao.ts b/src/commondao/common.dao.ts
index 9ec8926..5e6399f 100644
--- a/src/commondao/common.dao.ts
+++ b/src/commondao/common.dao.ts
@@ -920,7 +920,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
    * "Streaming" is implemented by buffering incoming rows into **batches**
    * (of size opt.chunkSize, which defaults to 500),
    * and then executing db.saveBatch(chunk) with the concurrency
-   * of opt.chunkConcurrency (which defaults to 16).
+   * of opt.chunkConcurrency (which defaults to 32).
    */
   streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
     this.requireWriteAccess()
@@ -936,7 +936,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
     const { beforeSave } = this.cfg.hooks!
 
-    const { chunkSize = 500, chunkConcurrency = 16, errorMode } = opt
+    const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt
 
     return [
       transformMap<BM, DBM>(
@@ -1019,7 +1019,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
     let deleted = 0
 
     if (opt.chunkSize) {
-      const { chunkSize, chunkConcurrency = 16 } = opt
+      const { chunkSize, chunkConcurrency = 32 } = opt
 
       await _pipeline([
         this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
diff --git a/src/kv/commonKeyValueDao.ts b/src/kv/commonKeyValueDao.ts
index 7dbb7c9..fdaccdd 100644
--- a/src/kv/commonKeyValueDao.ts
+++ b/src/kv/commonKeyValueDao.ts
@@ -224,7 +224,7 @@ export class CommonKeyValueDao<T> {
         }
       },
       {
-        concurrency: 16,
+        concurrency: 32,
       },
     )
   }
@@ -248,7 +248,7 @@ export class CommonKeyValueDao<T> {
         }
       },
       {
-        concurrency: 16,
+        concurrency: 32,
       },
     )
   }
diff --git a/yarn.lock b/yarn.lock
index a0ac9be..a4ddd54 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -884,9 +884,9 @@
     zod "^3.20.2"
 
 "@naturalcycles/nodejs-lib@^13.0.1", "@naturalcycles/nodejs-lib@^13.0.2", "@naturalcycles/nodejs-lib@^13.1.1":
-  version "13.19.0"
-  resolved "https://registry.yarnpkg.com/@naturalcycles/nodejs-lib/-/nodejs-lib-13.19.0.tgz#c5705a2636718e85b6aef8cb40fe19500d18d5ea"
-  integrity sha512-kc7Sx95amibbj6jHTej7UC1z1BVNBrSraEurcCWbKBnJWBAsp2f52yc5stnehOwD9J8cXm7DJtWdVdwwArpkiw==
+  version "13.20.0"
+  resolved "https://registry.yarnpkg.com/@naturalcycles/nodejs-lib/-/nodejs-lib-13.20.0.tgz#8d97cef2f685b387559d5be78d91f942d411d92f"
+  integrity sha512-qUmYEK2m58N2MwMXri+zqL/bJvc58FqBx1aV31Xtu6BiGMyRUZrqls+SuYCUW+/oJj9ULWFcBL08AHSbLnWC9A==
   dependencies:
     "@naturalcycles/js-lib" "^14.0.0"
     "@types/js-yaml" "^4.0.9"