diff --git a/.github/workflows/prebuild.yml b/.github/workflows/prebuild.yml index 2ef25d04d..64d510ae3 100644 --- a/.github/workflows/prebuild.yml +++ b/.github/workflows/prebuild.yml @@ -2,7 +2,7 @@ name: Test and Prebuild on: [push] jobs: build-test-macos: - if: startsWith(github.ref, 'refs/tags/') + #if: startsWith(github.ref, 'refs/tags/') env: LMDB_DATA_V1: ${{ contains(github.ref, '-v1') }} runs-on: macos-11 @@ -35,7 +35,7 @@ jobs: with: files: prebuild-darwin.tar build-test-win32: - if: startsWith(github.ref, 'refs/tags/') + #if: startsWith(github.ref, 'refs/tags/') env: LMDB_DATA_V1: ${{ contains(github.ref, '-v1') }} runs-on: windows-latest diff --git a/package.json b/package.json index 62f74c327..f9e08e805 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "lmdb", "author": "Kris Zyp", - "version": "2.9.3-beta.2", + "version": "2.10.0-beta.1", "description": "Simple, efficient, scalable, high-performance LMDB interface", "license": "MIT", "repository": { @@ -62,13 +62,13 @@ "build-js": "rollup -c", "prepare": "rollup -c", "before-publish": "rollup -c && cpy index.d.ts . --rename=index.d.cts && prebuildify-ci download && node util/set-optional-deps.cjs && npm run test", - "prebuild-libc-musl": "ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --tag-libc --napi --platform-packages --target 18.17.1", - "prebuild-libc": "prebuildify-platform-packages --tag-libc --target 20.0.0 || true && prebuildify-platform-packages --platform-packages --tag-libc --target 18.17.1 && ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --napi --platform-packages --tag-libc --target 18.17.1", - "prebuild-macos": "prebuildify-platform-packages --target 20.0.0 && prebuildify-platform-packages --platform-packages --target 18.17.1 && ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --napi --platform-packages --target 18.17.1", - "prebuild-win32": "prebuildify-platform-packages --target 20.0.0 && prebuildify-platform-packages --target 18.17.1 && set ENABLE_V8_FUNCTIONS=false&& prebuildify-platform-packages --napi --platform-packages --target 18.17.1", - "prebuild-libc-arm7": "ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --napi --platform-packages --tag-libc --target 18.17.1", - "prebuildify": "prebuildify-platform-packages --napi --target 18.17.1", - "full-publish": "cd prebuilds/win32-x64 && npm publish --access public && cd ../darwin-x64 && npm publish --access public && cd ../darwin-arm64 && npm publish --access public && cd ../linux-x64 && npm publish --access public && cd ../linux-arm64 && npm publish --access public && cd ../linux-arm && npm publish --access public && cd ../.. && npm publish --tag next", + "prebuild-libc-musl": "ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --debug --tag-libc --napi --platform-packages --target 18.17.1", + "prebuild-libc": "prebuildify-platform-packages --debug --tag-libc --target 20.0.0 || true && prebuildify-platform-packages --debug --platform-packages --tag-libc --target 18.17.1 && ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --debug --napi --platform-packages --tag-libc --target 18.17.1", + "prebuild-macos": "prebuildify-platform-packages --debug --target 20.0.0 && prebuildify-platform-packages --debug --platform-packages --target 18.17.1 && ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --debug --napi --platform-packages --target 18.17.1", + "prebuild-win32": "prebuildify-platform-packages --debug --target 20.0.0 && prebuildify-platform-packages --debug --target 18.17.1 && set ENABLE_V8_FUNCTIONS=false&& prebuildify-platform-packages --debug --napi --platform-packages --target 18.17.1", + "prebuild-libc-arm7": "ENABLE_V8_FUNCTIONS=false prebuildify-platform-packages --debug --napi --platform-packages --tag-libc --target 18.17.1", + "prebuildify": "prebuildify-platform-packages --debug --napi --target 18.17.1", + "full-publish": "cd prebuilds/win32-x64 && npm publish --tag next --access public && cd ../darwin-x64 && npm publish --tag next --access public && cd ../darwin-arm64 && npm publish --tag next --access public && cd ../linux-x64 && npm publish --tag next --access public && cd ../linux-arm64 && npm publish --tag next --access public && cd ../linux-arm && npm publish --tag next --access public && cd ../.. && npm publish --tag next", "recompile": "node-gyp clean && node-gyp configure && node-gyp build", "test": "mocha test/**.test.js --expose-gc --recursive", "deno-test": "deno run --allow-ffi --allow-write --allow-read --allow-env --allow-net --unstable test/deno.ts", diff --git a/read.js b/read.js index 2df1ecc72..19a84bca1 100644 --- a/read.js +++ b/read.js @@ -94,6 +94,9 @@ export function addReadMethods(LMDBStore, { let txn = env.writeTxn || (options && options.transaction) || (readTxnRenewed ? readTxn : renewReadTxn(this)); txn.refCount = (txn.refCount || 0) + 1; outstandingReads++; + if (!txn.address) { + throw new Error('Invalid transaction, it has no address'); + } let address = recordReadInstruction(txn.address, this.db.dbi, id, this.writeKey, maxKeySize, ( rc, bufferId, offset, size ) => { if (rc && rc !== 1) callback(lmdbError(rc)); @@ -156,6 +159,7 @@ export function addReadMethods(LMDBStore, { if (!buffer.isGlobal && !env.writeTxn) { let txn = options?.transaction || (readTxnRenewed ? readTxn : renewReadTxn(this)); buffer.txn = txn; + txn.refCount = (txn.refCount || 0) + 1; return data; } else { @@ -412,6 +416,9 @@ export function addReadMethods(LMDBStore, { if (txn.isDone) throw new Error('Can not iterate on range with transaction that is already' + ' done'); txnAddress = txn.address; + if (!txnAddress) { + throw new Error('Invalid transaction, it has no address'); + } cursor = null; } else { let writeTxn = env.writeTxn; @@ -427,7 +434,8 @@ export function addReadMethods(LMDBStore, { cursor = new Cursor(db, txnAddress || 0); } cursorAddress = cursor.address; - txn.refCount = (txn.refCount || 0) + 1; // track transaction so we always use the same one + if (txn.use) txn.use(); // track transaction so we always use the same one + else txn.refCount = (txn.refCount || 0) + 1; if (snapshot === false) { cursorRenewId = renewId; // use shared read transaction txn.renewingRefCount = (txn.renewingRefCount || 0) + 1; // need to know how many are renewing cursors @@ -488,10 +496,13 @@ export function addReadMethods(LMDBStore, { iterable.onDone() if (cursorRenewId) txn.renewingRefCount--; - if (--txn.refCount <= 0 && txn.notCurrent) { - cursor.close(); - txn.abort(); // this is no longer main read txn, abort it now that we are done + if (txn.done) txn.done(); + else if (--txn.refCount <= 0 && txn.notCurrent) { + txn.abort(); txn.isDone = true; + } + if (txn.refCount <= 0 && txn.notCurrent) { + cursor.close(); } else { if (db.availableCursor || txn != readTxn) { cursor.close(); diff --git a/src/env.cpp b/src/env.cpp index 80e45b818..9d8f10829 100644 --- a/src/env.cpp +++ b/src/env.cpp @@ -213,8 +213,7 @@ MDB_txn* EnvWrap::getReadTxn(int64_t tw_address) { if (rc) { if (!txn) fprintf(stderr, "No current read transaction available"); - if (rc != EINVAL) - return nullptr; // if there was a real error, signal with nullptr and let error propagate with last_error + return nullptr; // if there was an error, signal with nullptr and let error propagate with last_error } return txn; } @@ -697,7 +696,7 @@ void EnvWrap::closeEnv(bool hasLock) { if ((envFlags & MDB_OVERLAPPINGSYNC) && envPath->hasWrites) { mdb_env_sync(env, 1); } - //delete (ExtendedEnv*) mdb_env_get_userctx(env); + delete (ExtendedEnv*) mdb_env_get_userctx(env); #endif char* path; mdb_env_get_path(env, (const char**)&path); diff --git a/src/txn.cpp b/src/txn.cpp index f0a45df6d..c0bb3c308 100644 --- a/src/txn.cpp +++ b/src/txn.cpp @@ -62,6 +62,7 @@ TxnWrap::TxnWrap(const Napi::CallbackInfo& info) : ObjectWrap(info) { info.This().As().Set("address", Number::New(info.Env(), 0)); return; } + parentTw = nullptr; parentTxn = nullptr; } int rc = mdb_txn_begin(ew->env, parentTxn, flags, &txn); diff --git a/util/RangeIterable.js b/util/RangeIterable.js index ce2bf6fd4..c20cd017d 100644 --- a/util/RangeIterable.js +++ b/util/RangeIterable.js @@ -3,6 +3,10 @@ const DONE = { value: null, done: true, } +const RETURN_DONE = { // we allow this one to be mutated + value: null, + done: true, +}; if (!Symbol.asyncIterator) { Symbol.asyncIterator = Symbol.for('Symbol.asyncIterator'); } @@ -18,6 +22,7 @@ export class RangeIterable { let iterable = new RangeIterable(); iterable.iterate = (async) => { let iterator = source[async ? Symbol.asyncIterator : Symbol.iterator](); + if (!async) source.isSync = true; let i = 0; return { next(resolvedResult) { @@ -32,9 +37,11 @@ export class RangeIterable { iteratorResult = iterator.next(); if (iteratorResult.then) { if (!async) { - throw new Error('Can not synchronously iterate with asynchronous values'); + this.throw(new Error('Can not synchronously iterate with asynchronous values')); } - return iteratorResult.then(iteratorResult => this.next(iteratorResult), onError); + return iteratorResult.then(iteratorResult => this.next(iteratorResult), (error) => { + this.throw(error); + }); } } if (iteratorResult.done === true) { @@ -42,44 +49,44 @@ export class RangeIterable { if (iterable.onDone) iterable.onDone(); return iteratorResult; } - result = func(iteratorResult.value, i++); - if (result && result.then) { - if (!async) { - throw new Error('Can not synchronously iterate with asynchronous values'); - } + result = func.call(source, iteratorResult.value, i++); + if (result && result.then && async) { + // if async, wait for promise to resolve before returning iterator result return result.then(result => result === SKIP ? this.next() : { value: result - }, onError); + }, (error) => { + this.throw(error); + }); } } while(result === SKIP); if (result === DONE) { - if (iterable.onDone) iterable.onDone(); - return result; + return this.return(); } return { value: result }; } catch(error) { - onError(error); + this.throw(error); } }, - return() { - if (iterable.onDone) iterable.onDone(); - return iterator.return(); + return(value) { + if (!this.done) { + RETURN_DONE.value = value; + this.done = true; + if (iterable.onDone) iterable.onDone(); + iterator.return(); + } + return RETURN_DONE; }, - throw() { - if (iterable.onDone) iterable.onDone(); - return iterator.throw(); + throw(error) { + this.return(); + throw error; } }; }; - function onError(error) { - if (iterable.onDone) iterable.onDone(); - throw error; - } return iterable; } [Symbol.asyncIterator]() { @@ -120,11 +127,14 @@ export class RangeIterable { if (!async) throw new Error('Can not synchronously iterate with asynchronous values'); result.then((result) => { if (result.done()) concatIterable.onDone(); - }, onError); + }, (error) => { + this.return(); + throw error; + }); } else if (result.done) concatIterable.onDone(); } } catch (error) { - onError(error); + this.throw(error); } } else { if (concatIterable.onDone) concatIterable.onDone(); @@ -145,24 +155,26 @@ export class RangeIterable { if (result.done) return iteratorDone(result); return result; } catch (error) { - onError(error); + this.return(); + throw error; } }, return() { - if (concatIterable.onDone) concatIterable.onDone(); - return iterator.return(); + if (!this.done) { + RETURN_DONE.value = value; + this.done = true; + if (concatIterable.onDone) concatIterable.onDone(); + iterator.return(); + } + return RETURN_DONE; + }, - throw() { - if (concatIterable.onDone) concatIterable.onDone(); - return iterator.throw(); + throw(error) { + this.return(); + throw error; } }; }; - function onError(error) { - if (iterable.onDone) iterable.onDone(); - throw error; - } - return concatIterable; } @@ -173,21 +185,46 @@ export class RangeIterable { let isFirst = true; let currentSubIterator; return { - next() { + next(resolvedResult) { try { do { if (currentSubIterator) { - let result = currentSubIterator.next(); + let result; + if (resolvedResult) { + result = resolvedResult; + resolvedResult = undefined; + } else result = currentSubIterator.next(); + if (result.then) { + if (!async) throw new Error('Can not synchronously iterate with asynchronous values'); + return result.then((result) => this.next(result)); + } if (!result.done) { return result; } } - let result = iterator.next(); + let result = resolvedResult ?? iterator.next(); + if (result.then) { + if (!async) throw new Error('Can not synchronously iterate with asynchronous values'); + currentSubIterator = undefined; + return result.then((result) => this.next(result)); + } if (result.done) { if (mappedIterable.onDone) mappedIterable.onDone(); return result; } let value = callback(result.value); + if (value?.then) { + if (!async) throw new Error('Can not synchronously iterate with asynchronous values'); + return value.then((value) => { + if (Array.isArray(value) || value instanceof RangeIterable) { + currentSubIterator = value[Symbol.iterator](); + return this.next(); + } else { + currentSubIterator = null; + return { value }; + } + }) + } if (Array.isArray(value) || value instanceof RangeIterable) currentSubIterator = value[Symbol.iterator](); else { @@ -196,7 +233,8 @@ export class RangeIterable { } } while(true); } catch (error) { - onError(error); + this.return(); + throw error; } }, return() { @@ -213,10 +251,6 @@ export class RangeIterable { } }; }; - function onError(error) { - if (iterable.onDone) iterable.onDone(); - throw error; - } return mappedIterable; }