diff --git a/README.md b/README.md index a9e0c28..92ed387 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,10 @@ await storage.init({ // if you setItem() multiple times to the same key, only the last one would be set, BUT the others would still resolve with the results of the last one, if you turn this to false, each one will execute, but might slow down the writing process. writeQueueWriteOnlyLast: true, + + // Limit the number of concurrently used file descriptors to avoid Error: EMFILE: too many open files + // Defaults to Infinity for maximum performance but YMMV depending on your OS and how you use the library + maxFileDescriptors: 512 }); ``` diff --git a/package-lock.json b/package-lock.json index 461a29b..51eb55e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,16 @@ { "name": "node-persist", - "version": "4.0.1", + "version": "4.0.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "node-persist", - "version": "4.0.1", + "version": "4.0.3", "license": "MIT", + "dependencies": { + "p-limit": "^3.1.0" + }, "devDependencies": { "chai": "^4.1.2", "mocha": "^10.2.0", @@ -701,7 +704,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" }, @@ -995,7 +998,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -1502,7 +1505,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, "requires": { "yocto-queue": "^0.1.0" } @@ -1703,8 +1705,7 @@ "yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" } } } diff --git a/package.json b/package.json index a3c9c4e..2c77e3d 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,9 @@ ], "license": "MIT", "readmeFilename": "README.md", + "dependencies": { + "p-limit": "^3.1.0" + }, "devDependencies": { "chai": "^4.1.2", "mocha": "^10.2.0", diff --git a/src/local-storage.js b/src/local-storage.js index e296449..e4b08bd 100644 --- a/src/local-storage.js +++ b/src/local-storage.js @@ -6,8 +6,10 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); -const pkg = require('../package.json'); const { nextTick } = require('process'); +const pLimit = require('p-limit'); + +const pkg = require('../package.json'); const defaults = { ttl: false, @@ -21,6 +23,7 @@ const defaults = { writeQueue: true, writeQueueIntervalMs: 1000, writeQueueWriteOnlyLast: true, + maxFileDescriptors: Infinity, }; const defaultTTL = 24 * 60 * 60 * 1000; /* if ttl is truthy but it's not a number, use 24h as default */ @@ -127,6 +130,12 @@ LocalStorage.prototype = { this.log = options.logging; options.logging = true; } + + // Update limiter if maxFileDescriptors has changed + if (!this.limit || options.maxFileDescriptors !== this.options?.maxFileDescriptors) { + this.limit = pLimit(options.maxFileDescriptors); + } + this.options = options; }, @@ -307,7 +316,8 @@ LocalStorage.prototype = { try { for (let currentFile of arr) { if (currentFile[0] !== '.') { - data.push(await this.readFile(path.join(this.options.dir, currentFile))); + // Limit concurrent reads + data.push(await this.limit(() => this.readFile(path.join(this.options.dir, currentFile)))); } } } catch (err) { @@ -323,7 +333,7 @@ LocalStorage.prototype = { }, readFile: function (file, options = {}) { - return new Promise((resolve, reject) => { + return this.limit(() => new Promise((resolve, reject) => { fs.readFile(file, this.options.encoding, (err, text) => { if (err) { /* Only throw the error if the error is something else other than the file doesn't exist */ @@ -340,7 +350,7 @@ LocalStorage.prototype = { } resolve(input); }); - }); + })); }, queueWriteFile: async function (file, content) { @@ -411,19 +421,19 @@ LocalStorage.prototype = { }, writeFile: async function (file, content) { - return new Promise((resolve, reject) => { - fs.writeFile(file, this.stringify(content), this.options.encoding, async (err) => { + return this.limit(() => new Promise((resolve, reject) => { + fs.writeFile(file, this.stringify(content), this.options.encoding, (err) => { if (err) { return reject(err); } resolve({file: file, content: content}); this.log('wrote: ' + file); }); - }); + })); }, deleteFile: function (file) { - return new Promise((resolve, reject) => { + return this.limit(() => new Promise((resolve, reject) => { fs.access(file, (accessErr) => { if (!accessErr) { this.log(`Removing file:${file}`); @@ -442,7 +452,7 @@ LocalStorage.prototype = { resolve(result); } }); - }); + })); }, stringify: function (obj) {