From 702b7438753a5ff74f58def5b7f798e13453ad6a Mon Sep 17 00:00:00 2001 From: Mark Wylde Date: Mon, 23 Dec 2024 17:33:48 +0000 Subject: [PATCH] feat: implement upsert; implement batchInsert; --- README.md | 26 ++++++++- package-lock.json | 132 +++++++++++++++++++--------------------------- package.json | 6 +-- src/index.ts | 64 ++++++++++++++++++++-- test/index.ts | 77 +++++++++++++++++++++++++++ 5 files changed, 218 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 9eff0c6..17b5a72 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ doubledb.insert({ skills: ['cooking', 'running'] }); -doubledb.get(record.id); +doubledb.read(record.id); doubledb.find('firstName', 'Joe'); doubledb.find('stats.wins', 10); doubledb.find('skills', 'cooking'); @@ -34,9 +34,16 @@ doubledb.filter('firstName', v => v.startsWith('J'), { limit: 10, skip: 20, gt: doubledb.replace(record.id, { firstName: 'Joe', lastName: 'Bloggs' }); doubledb.patch(record.id, { firstName: 'Bob' }); doubledb.remove(record.id); + +// Batch insert multiple documents for better performance +doubledb.batchInsert([ + { firstName: 'Alice', lastName: 'Smith' }, + { firstName: 'Bob', lastName: 'Johnson' }, + { firstName: 'Charlie', lastName: 'Brown' } +]); ``` -### `.get(id)` +### `.read(id)` Get a single record by it's `.id` property. If a record is found, the whole record will be returned. @@ -188,3 +195,18 @@ const records = await doubledb.query({ - **$not**: Matches documents that do not match the specified condition. This query method is powerful and allows combining multiple conditions and operators to fetch the desired records from the database. + +### `.batchInsert(documents)` +Insert multiple documents at once for better performance. + +**Example:** +```javascript +await doubledb.batchInsert([ + { firstName: 'Alice', lastName: 'Smith' }, + { firstName: 'Bob', lastName: 'Johnson' }, + { firstName: 'Charlie', lastName: 'Brown' } +]); +``` + +If the documents are successfully inserted, an array of the inserted documents will be returned. +If the documents array is empty, an error will be thrown. diff --git a/package-lock.json b/package-lock.json index 8be4da1..8218102 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,24 +10,27 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { - "level": "^8.0.1", + "level": "^9.0.0", "uuid": "^11.0.3" }, "devDependencies": { - "@types/node": "^22.10.1", + "@types/node": "^22.10.2", "@types/uuid": "^10.0.0", - "c8": "^10.1.2", + "c8": "^10.1.3", "semistandard": "^17.0.0", "tsx": "^4.19.2", "typescript": "^5.7.2" } }, "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.1.tgz", + "integrity": "sha512-W+a0/JpU28AqH4IKtwUPcEUnUyXMDLALcn5/JLczGGT9fHE2sIby/xP/oQnx3nxkForzgzPy201RAKcB4xPAFQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/@esbuild/aix-ppc64": { "version": "0.23.1", @@ -750,9 +753,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "22.10.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", - "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", + "version": "22.10.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", + "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -774,21 +777,20 @@ "license": "ISC" }, "node_modules/abstract-level": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-1.0.4.tgz", - "integrity": "sha512-eUP/6pbXBkMbXFdx4IH2fVgvB7M0JvR7/lIL33zcs0IBcwjdzSSl31TOJsaCzmKSSDF9h8QYSOJux4Nd4YJqFg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/abstract-level/-/abstract-level-2.0.2.tgz", + "integrity": "sha512-pPJixmXk/kTKLB2sSue7o4Uj6TlLD2XfaP2gWZomHVCC6cuUGX/VslQqKG1yZHfXwBb/3lS6oSTMPGzh1P1iig==", "license": "MIT", "dependencies": { "buffer": "^6.0.3", - "catering": "^2.1.0", "is-buffer": "^2.0.5", - "level-supports": "^4.0.0", + "level-supports": "^6.0.0", "level-transcoder": "^1.0.1", - "module-error": "^1.0.1", - "queue-microtask": "^1.2.3" + "maybe-combine-errors": "^1.0.0", + "module-error": "^1.0.1" }, "engines": { - "node": ">=12" + "node": ">=16" } }, "node_modules/acorn": { @@ -1077,15 +1079,12 @@ } }, "node_modules/browser-level": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-1.0.1.tgz", - "integrity": "sha512-XECYKJ+Dbzw0lbydyQuJzwNXtOpbMSq737qxJN11sIRTErOMShvDpbzTlgju7orJKvx4epULolZAuJGLzCmWRQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/browser-level/-/browser-level-2.0.0.tgz", + "integrity": "sha512-RuYSCHG/jwFCrK+KWA3dLSUNLKHEgIYhO5ORPjJMjCt7T3e+RzpIDmYKWRHxq2pfKGXjlRuEff7y7RESAAgzew==", "license": "MIT", "dependencies": { - "abstract-level": "^1.0.2", - "catering": "^2.1.1", - "module-error": "^1.0.2", - "run-parallel-limit": "^1.1.0" + "abstract-level": "^2.0.1" } }, "node_modules/buffer": { @@ -1123,13 +1122,13 @@ } }, "node_modules/c8": { - "version": "10.1.2", - "resolved": "https://registry.npmjs.org/c8/-/c8-10.1.2.tgz", - "integrity": "sha512-Qr6rj76eSshu5CgRYvktW0uM0CFY0yi4Fd5D0duDXO6sYinyopmftUiJVuzBQxQcwQLor7JWDVRP+dUfCmzgJw==", + "version": "10.1.3", + "resolved": "https://registry.npmjs.org/c8/-/c8-10.1.3.tgz", + "integrity": "sha512-LvcyrOAaOnrrlMpW22n690PUvxiq4Uf9WMhQwNJ9vgagkL/ph1+D4uvjvDA5XCbykrc0sx+ay6pVi9YZ1GnhyA==", "dev": true, "license": "ISC", "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", + "@bcoe/v8-coverage": "^1.0.1", "@istanbuljs/schema": "^0.1.3", "find-up": "^5.0.0", "foreground-child": "^3.1.1", @@ -1186,15 +1185,6 @@ "node": ">=6" } }, - "node_modules/catering": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/catering/-/catering-2.1.1.tgz", - "integrity": "sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -1213,20 +1203,19 @@ } }, "node_modules/classic-level": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/classic-level/-/classic-level-1.4.1.tgz", - "integrity": "sha512-qGx/KJl3bvtOHrGau2WklEZuXhS3zme+jf+fsu6Ej7W7IP/C49v7KNlWIsT1jZu0YnfzSIYDGcEWpCa1wKGWXQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/classic-level/-/classic-level-2.0.0.tgz", + "integrity": "sha512-ftiMvKgCQK+OppXcvMieDoYlYLYWhScK6yZRFBrrlHQRbm4k6Gr+yDgu/wt3V0k1/jtNbuiXAsRmuAFcD0Tx5Q==", "hasInstallScript": true, "license": "MIT", "dependencies": { - "abstract-level": "^1.0.2", - "catering": "^2.1.0", + "abstract-level": "^2.0.0", "module-error": "^1.0.1", "napi-macros": "^2.2.2", "node-gyp-build": "^4.3.0" }, "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/cliui": { @@ -3313,17 +3302,17 @@ } }, "node_modules/level": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/level/-/level-8.0.1.tgz", - "integrity": "sha512-oPBGkheysuw7DmzFQYyFe8NAia5jFLAgEnkgWnK3OXAuJr8qFT+xBQIwokAZPME2bhPFzS8hlYcL16m8UZrtwQ==", + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/level/-/level-9.0.0.tgz", + "integrity": "sha512-n+mVuf63mUEkd8NUx7gwxY+QF5vtkibv6fXTGUgtHWLPDaA5/XZjLcI/Q1nQ8k6OttHT6Ezt+7nSEXsRUfHtOQ==", "license": "MIT", "dependencies": { - "abstract-level": "^1.0.4", - "browser-level": "^1.0.1", - "classic-level": "^1.2.0" + "abstract-level": "^2.0.1", + "browser-level": "^2.0.0", + "classic-level": "^2.0.0" }, "engines": { - "node": ">=12" + "node": ">=18" }, "funding": { "type": "opencollective", @@ -3331,12 +3320,12 @@ } }, "node_modules/level-supports": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-4.0.1.tgz", - "integrity": "sha512-PbXpve8rKeNcZ9C1mUicC9auIYFyGpkV9/i6g76tLgANwWhtG2v7I4xNBUlkn3lE2/dZF3Pi0ygYGtLc4RXXdA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-6.0.0.tgz", + "integrity": "sha512-UU226PsfiFWLRPmuqgB3eADtvZM8WYv+aCnAl93B/2Ca+vgn9+b7o2boA7yOY2ri7Kk5Wk4aHxl3eNimpYZnxw==", "license": "MIT", "engines": { - "node": ">=12" + "node": ">=16" } }, "node_modules/level-transcoder": { @@ -3452,6 +3441,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/maybe-combine-errors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/maybe-combine-errors/-/maybe-combine-errors-1.0.0.tgz", + "integrity": "sha512-eefp6IduNPT6fVdwPp+1NgD0PML1NU5P6j1Mj5nz1nidX8/sWY7119WL8vTAHgqfsY74TzW0w1XPgdYEKkGZ5A==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3938,6 +3936,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, "funding": [ { "type": "github", @@ -4115,29 +4114,6 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/run-parallel-limit": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.1.0.tgz", - "integrity": "sha512-jJA7irRNM91jaKc3Hcl1npHsFLOXOoTkPCUL1JEa1R82O2miplXXRaGdjW/KM/98YQWDhJLiSs793CnXfblJUw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, "node_modules/safe-array-concat": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", diff --git a/package.json b/package.json index 64963b4..b3c2710 100644 --- a/package.json +++ b/package.json @@ -36,15 +36,15 @@ }, "homepage": "https://github.com/markwylde/doubledb#readme", "devDependencies": { - "@types/node": "^22.10.1", + "@types/node": "^22.10.2", "@types/uuid": "^10.0.0", - "c8": "^10.1.2", + "c8": "^10.1.3", "semistandard": "^17.0.0", "tsx": "^4.19.2", "typescript": "^5.7.2" }, "dependencies": { - "level": "^8.0.1", + "level": "^9.0.0", "uuid": "^11.0.3" } } diff --git a/src/index.ts b/src/index.ts index 4044a45..0ca58ec 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,8 @@ export type DoubleDb = { read: (id: string) => Promise; query: (queryObject: object) => Promise; close: () => Promise; + batchInsert: (documents: Document[]) => Promise; + upsert: (id: string, document: Document) => Promise; } function notFoundToUndefined(error: Error & { code?: string }): undefined { @@ -422,9 +424,7 @@ async function createDoubleDb(dataDirectory: string): Promise { async function getIdsForKeyValueRange(key: string, op: string, value: number): Promise> { const ids = new Set(); - const query = - - { + const query = { gte: `indexes.${key}=`, lte: `indexes.${key}=${LastUnicodeCharacter}` }; @@ -515,6 +515,60 @@ async function createDoubleDb(dataDirectory: string): Promise { return new Set([...allIds].filter(id => !excludeIds.has(id))); } + async function batchInsert(documents: Document[]): Promise { + if (!Array.isArray(documents) || documents.length === 0) { + throw new Error('doubledb.batchInsert: documents must be a non-empty array'); + } + + const ops: { type: 'put'; key: string; value: string }[] = []; + for (const doc of documents) { + const id = doc.id || uuid(); + const puttableRecord = { id, ...doc }; + ops.push({ type: 'put', key: id, value: JSON.stringify(puttableRecord) }); + } + + await db.batch(ops); + + // Index each document in parallel + await Promise.all(documents.map(async doc => { + const id = doc.id || uuid(); + const puttableRecord = { id, ...doc }; + await addToIndexes(id, puttableRecord); + })); + + return documents.map(doc => { + const id = doc.id || uuid(); + return { id, ...doc }; + }); + } + + async function upsert(id: string, document: Document): Promise { + if (!id) { + throw new Error('doubledb.upsert: no id was supplied to upsert function'); + } + + if (!document) { + throw new Error('doubledb.upsert: no document was supplied to upsert function'); + } + + const existingDocument = await db.get(id).catch(notFoundToUndefined); + + if (existingDocument) { + await removeIndexesForDocument(id, existingDocument); + } + + const puttableRecord = { + ...JSON.parse(existingDocument || '{}'), + ...document, + id + }; + + await db.put(id, JSON.stringify(puttableRecord)); + await addToIndexes(id, puttableRecord); + + return puttableRecord; + } + return { _level: db, find, @@ -525,7 +579,9 @@ async function createDoubleDb(dataDirectory: string): Promise { remove, read, query, - close: db.close.bind(db) + close: db.close.bind(db), + batchInsert, + upsert }; } diff --git a/test/index.ts b/test/index.ts index 380ac33..8d72db3 100644 --- a/test/index.ts +++ b/test/index.ts @@ -406,3 +406,80 @@ test('remove - found - removes document', async () => { assert.strictEqual(readAfter, undefined); assert.strictEqual(removeResult, undefined); }); + +test('batchInsert - inserts multiple documents at once', async () => { + await fs.rm(testDir, { recursive: true }).catch(() => {}); + const db = await createDoubleDb(testDir); + + const docsToInsert = [ + { id: 'batch1', value: 10 }, + { id: 'batch2', value: 20 }, + { id: 'batch3', value: 30 }, + ]; + const insertedDocs = await db.batchInsert(docsToInsert); + + assert.strictEqual(insertedDocs.length, 3); + assert.deepStrictEqual(await db.read('batch1'), docsToInsert[0]); + assert.deepStrictEqual(await db.read('batch2'), docsToInsert[1]); + assert.deepStrictEqual(await db.read('batch3'), docsToInsert[2]); + + await db.close(); +}); + +test('batchInsert - empty array - throws error', async () => { + await fs.rm(testDir, { recursive: true }).catch(() => {}); + const db = await createDoubleDb(testDir); + + try { + await db.batchInsert([]); + } catch (error) { + await db.close(); + assert.strictEqual((error as Error).message, 'doubledb.batchInsert: documents must be a non-empty array'); + } +}); + +test('upsert - inserts new document if not exists', async () => { + await fs.rm(testDir, { recursive: true }).catch(() => {}); + const db = await createDoubleDb(testDir); + + const upsertedRecord = await db.upsert('id1', { a: 1 }); + + const readRecord = await db.read('id1'); + + await db.close(); + + assert.deepStrictEqual(upsertedRecord, { + id: 'id1', + a: 1 + }); + + assert.deepStrictEqual(readRecord, { + id: 'id1', + a: 1 + }); +}); + +test('upsert - updates existing document if exists', async () => { + await fs.rm(testDir, { recursive: true }).catch(() => {}); + const db = await createDoubleDb(testDir); + + await db.insert({ id: 'id1', a: 1 }); + + const upsertedRecord = await db.upsert('id1', { a: 2, b: 3 }); + + const readRecord = await db.read('id1'); + + await db.close(); + + assert.deepStrictEqual(upsertedRecord, { + id: 'id1', + a: 2, + b: 3 + }); + + assert.deepStrictEqual(readRecord, { + id: 'id1', + a: 2, + b: 3 + }); +});