diff --git a/packages/plugin-kit/src/fs/deleteProjectFile.ts b/packages/plugin-kit/src/fs/deleteProjectFile.ts index 140bfb51da..a671d3a1b4 100644 --- a/packages/plugin-kit/src/fs/deleteProjectFile.ts +++ b/packages/plugin-kit/src/fs/deleteProjectFile.ts @@ -1,7 +1,6 @@ -import * as fs from "node:fs/promises"; - import { SliceMachineHelpers } from "../createSliceMachineHelpers"; -import { fsLimit } from "./lib/fsLimit"; + +import * as fs from "./lib/fsLimit"; export type DeleteProjectFileArgs = { filename: string; @@ -13,7 +12,7 @@ export const deleteProjectFile = async ( ): Promise => { const filePath = args.helpers.joinPathFromRoot(args.filename); - await fsLimit(() => fs.rm(filePath, { recursive: true })); + await fs.rm(filePath, { recursive: true }); return filePath; }; diff --git a/packages/plugin-kit/src/fs/lib/checkPathExists.ts b/packages/plugin-kit/src/fs/lib/checkPathExists.ts index b26fd0b784..d4c46d7fec 100644 --- a/packages/plugin-kit/src/fs/lib/checkPathExists.ts +++ b/packages/plugin-kit/src/fs/lib/checkPathExists.ts @@ -1,10 +1,10 @@ -import * as fs from "node:fs/promises"; import { PathLike } from "node:fs"; -import { fsLimit } from "./fsLimit"; + +import * as fs from "./fsLimit"; export async function checkPathExists(path: PathLike): Promise { try { - await fsLimit(() => fs.access(path)); + await fs.access(path); return true; } catch { diff --git a/packages/plugin-kit/src/fs/lib/fsLimit.ts b/packages/plugin-kit/src/fs/lib/fsLimit.ts index 12e4abcd89..25f6881e72 100644 --- a/packages/plugin-kit/src/fs/lib/fsLimit.ts +++ b/packages/plugin-kit/src/fs/lib/fsLimit.ts @@ -1,5 +1,13 @@ +import * as fs from "node:fs/promises"; import pLimit from "p-limit"; +/** + * The parsed number value of the SM_FS_LIMIT environment variable. + */ +const SM_FS_LIMIT = Number.isNaN(Number(process.env.SM_FS_LIMIT)) + ? undefined + : Number(process.env.SM_FS_LIMIT); + /** * The maximum number of concurrent file descriptors allowed to adapters to * minimize issues like `EMFILE: too many open files`. @@ -7,9 +15,57 @@ import pLimit from "p-limit"; * - MacOS default limit: 2560 (recent), 256 (old) * - Windows limit (per process): 2048 */ -const CONCURRENT_FILE_DESCRIPTORS_LIMIT = 1024; +const CONCURRENT_FILE_DESCRIPTORS_LIMIT = SM_FS_LIMIT ?? 1024; /** * Limit concurrent file descriptors for adapters. */ -export const fsLimit = pLimit(CONCURRENT_FILE_DESCRIPTORS_LIMIT); +const fsLimit = pLimit(CONCURRENT_FILE_DESCRIPTORS_LIMIT); + +/** + * Wrap a function with `fsLimit()` to limit concurrent calls. All functions + * called with `wrapWithFSLimit()` share the same queue. + * + * @param fn - The function to wrap. + * + * @returns The wrapped function. + */ +const wrapWithFSLimit = < + // eslint-disable-next-line @typescript-eslint/no-explicit-any + TFn extends (...args: any[]) => any, +>( + fn: TFn, +): TFn => { + return ((...args) => fsLimit(() => fn(...args))) as TFn; +}; + +export const access = wrapWithFSLimit(fs.access); +export const appendFile = wrapWithFSLimit(fs.appendFile); +export const chmod = wrapWithFSLimit(fs.chmod); +export const chown = wrapWithFSLimit(fs.chown); +export const copyFile = wrapWithFSLimit(fs.copyFile); +export const cp = wrapWithFSLimit(fs.cp); +export const lchmod = wrapWithFSLimit(fs.lchmod); +export const lchown = wrapWithFSLimit(fs.lchown); +export const link = wrapWithFSLimit(fs.link); +export const lstat = wrapWithFSLimit(fs.lstat); +export const lutimes = wrapWithFSLimit(fs.lutimes); +export const mkdir = wrapWithFSLimit(fs.mkdir); +export const mkdtemp = wrapWithFSLimit(fs.mkdtemp); +export const open = wrapWithFSLimit(fs.open); +export const opendir = wrapWithFSLimit(fs.opendir); +export const readFile = wrapWithFSLimit(fs.readFile); +export const readdir = wrapWithFSLimit(fs.readdir); +export const readlink = wrapWithFSLimit(fs.readlink); +export const realpath = wrapWithFSLimit(fs.realpath); +export const rename = wrapWithFSLimit(fs.rename); +export const rm = wrapWithFSLimit(fs.rm); +export const rmdir = wrapWithFSLimit(fs.rmdir); +export const stat = wrapWithFSLimit(fs.stat); +export const statfs = wrapWithFSLimit(fs.statfs); +export const symlink = wrapWithFSLimit(fs.symlink); +export const truncate = wrapWithFSLimit(fs.truncate); +export const unlink = wrapWithFSLimit(fs.unlink); +export const utimes = wrapWithFSLimit(fs.utimes); +export const watch = wrapWithFSLimit(fs.watch); +export const writeFile = wrapWithFSLimit(fs.writeFile); diff --git a/packages/plugin-kit/src/fs/lib/readJSONFile.ts b/packages/plugin-kit/src/fs/lib/readJSONFile.ts index c0ee9b2f59..e20f64e290 100644 --- a/packages/plugin-kit/src/fs/lib/readJSONFile.ts +++ b/packages/plugin-kit/src/fs/lib/readJSONFile.ts @@ -1,8 +1,7 @@ -import * as fs from "node:fs/promises"; -import { fsLimit } from "./fsLimit"; +import * as fs from "./fsLimit"; export async function readJSONFile(path: string): Promise { - const contents = await fsLimit(() => fs.readFile(path, "utf8")); + const contents = await fs.readFile(path, "utf8"); return JSON.parse(contents); } diff --git a/packages/plugin-kit/src/fs/readCustomTypeLibrary.ts b/packages/plugin-kit/src/fs/readCustomTypeLibrary.ts index 3af60e1da4..9ebeae5ee3 100644 --- a/packages/plugin-kit/src/fs/readCustomTypeLibrary.ts +++ b/packages/plugin-kit/src/fs/readCustomTypeLibrary.ts @@ -1,16 +1,15 @@ -import * as fs from "node:fs/promises"; import * as path from "node:path"; import { checkPathExists } from "./lib/checkPathExists"; import { isCustomTypeModel } from "./lib/isCustomTypeModel"; import { readJSONFile } from "./lib/readJSONFile"; +import * as fs from "./lib/fsLimit"; import { CUSTOM_TYPE_MODEL_FILENAME } from "./constants"; import { buildCustomTypeLibraryDirectoryPath, BuildCustomTypeLibraryDirectoryPathArgs, } from "./buildCustomTypeLibraryDirectoryPath"; -import { fsLimit } from "./lib/fsLimit"; export type ReadCustomTypeLibraryArgs = BuildCustomTypeLibraryDirectoryPathArgs; @@ -32,9 +31,7 @@ export const readCustomTypeLibrary = async ( }; } - const childDirs = await fsLimit(() => - fs.readdir(libraryDir, { withFileTypes: true }), - ); + const childDirs = await fs.readdir(libraryDir, { withFileTypes: true }); /** * Paths to models that could not be read due to invalid JSON. @@ -45,10 +42,11 @@ export const readCustomTypeLibrary = async ( await Promise.all( childDirs.map(async (childDir) => { if (childDir.isDirectory()) { - const childDirContents = await fsLimit(() => - fs.readdir(path.join(libraryDir, childDir.name), { + const childDirContents = await fs.readdir( + path.join(libraryDir, childDir.name), + { withFileTypes: true, - }), + }, ); const isCustomTypeDir = childDirContents.some((entry) => { return entry.isFile() && entry.name === CUSTOM_TYPE_MODEL_FILENAME; diff --git a/packages/plugin-kit/src/fs/readProjectFile.ts b/packages/plugin-kit/src/fs/readProjectFile.ts index 6c37886acb..47f8876d33 100644 --- a/packages/plugin-kit/src/fs/readProjectFile.ts +++ b/packages/plugin-kit/src/fs/readProjectFile.ts @@ -1,7 +1,6 @@ -import * as fs from "node:fs/promises"; - import { SliceMachineHelpers } from "../createSliceMachineHelpers"; -import { fsLimit } from "./lib/fsLimit"; + +import * as fs from "./lib/fsLimit"; type BufferEncoding = Extract[1], string>; @@ -21,5 +20,5 @@ export async function readProjectFile( ): Promise { const filePath = args.helpers.joinPathFromRoot(args.filename); - return await fsLimit(() => fs.readFile(filePath, args.encoding)); + return await fs.readFile(filePath, args.encoding); } diff --git a/packages/plugin-kit/src/fs/readSliceLibrary.ts b/packages/plugin-kit/src/fs/readSliceLibrary.ts index 77e050070c..4b26d0b837 100644 --- a/packages/plugin-kit/src/fs/readSliceLibrary.ts +++ b/packages/plugin-kit/src/fs/readSliceLibrary.ts @@ -1,16 +1,15 @@ -import * as fs from "node:fs/promises"; import * as path from "node:path"; import { checkPathExists } from "./lib/checkPathExists"; import { isSharedSliceModel } from "./lib/isSharedSliceModel"; import { readJSONFile } from "./lib/readJSONFile"; +import * as fs from "./lib/fsLimit"; import { SHARED_SLICE_MODEL_FILENAME } from "./constants"; import { buildSliceLibraryDirectoryPath, BuildSliceLibraryDirectoryPathArgs, } from "./buildSliceLibraryDirectoryPath"; -import { fsLimit } from "./lib/fsLimit"; export type ReadSliceLibraryArgs = BuildSliceLibraryDirectoryPathArgs; @@ -35,9 +34,7 @@ export const readSliceLibrary = async ( }; } - const childDirs = await fsLimit(() => - fs.readdir(libraryDir, { withFileTypes: true }), - ); + const childDirs = await fs.readdir(libraryDir, { withFileTypes: true }); /** * Paths to models that could not be read due to invalid JSON. @@ -48,10 +45,11 @@ export const readSliceLibrary = async ( await Promise.all( childDirs.map(async (childDir) => { if (childDir.isDirectory()) { - const childDirContents = await fsLimit(() => - fs.readdir(path.join(libraryDir, childDir.name), { + const childDirContents = await fs.readdir( + path.join(libraryDir, childDir.name), + { withFileTypes: true, - }), + }, ); const isSliceDir = childDirContents.some((entry) => { return entry.isFile() && entry.name === SHARED_SLICE_MODEL_FILENAME; diff --git a/packages/plugin-kit/src/fs/readSliceModel.ts b/packages/plugin-kit/src/fs/readSliceModel.ts index 617fa041de..9c0d6d7b6b 100644 --- a/packages/plugin-kit/src/fs/readSliceModel.ts +++ b/packages/plugin-kit/src/fs/readSliceModel.ts @@ -1,16 +1,15 @@ import { SharedSlice } from "@prismicio/types-internal/lib/customtypes"; -import * as fs from "node:fs/promises"; import * as path from "node:path"; +import { SliceMachineHelpers } from "../createSliceMachineHelpers"; + import { checkPathExists } from "./lib/checkPathExists"; import { isSharedSliceModel } from "./lib/isSharedSliceModel"; import { readJSONFile } from "./lib/readJSONFile"; - -import { SliceMachineHelpers } from "../createSliceMachineHelpers"; +import * as fs from "./lib/fsLimit"; import { buildSliceLibraryDirectoryPath } from "./buildSliceLibraryDirectoryPath"; import { SHARED_SLICE_MODEL_FILENAME } from "./constants"; -import { fsLimit } from "./lib/fsLimit"; export type ReadSliceModelArgs = { libraryID: string; @@ -32,9 +31,7 @@ export const readSliceModel = async ( }); if (await checkPathExists(libraryDir)) { - const childDirs = await fsLimit(() => - fs.readdir(libraryDir, { withFileTypes: true }), - ); + const childDirs = await fs.readdir(libraryDir, { withFileTypes: true }); /** * Paths to models that could not be read due to invalid JSON. diff --git a/packages/plugin-kit/src/fs/readSliceTemplateLibrary.ts b/packages/plugin-kit/src/fs/readSliceTemplateLibrary.ts index f3444d5130..9886247c4f 100644 --- a/packages/plugin-kit/src/fs/readSliceTemplateLibrary.ts +++ b/packages/plugin-kit/src/fs/readSliceTemplateLibrary.ts @@ -1,14 +1,14 @@ import path from "node:path"; -import fs from "node:fs/promises"; import { SharedSlice } from "@prismicio/types-internal/lib/customtypes"; import { SharedSliceContent } from "@prismicio/types-internal/lib/content"; -import { checkIsTypeScriptProject } from "./checkIsTypeScriptProject"; import { SliceMachineHelpers } from "../createSliceMachineHelpers"; - import { SliceTemplateLibraryReadHookReturnType } from "../hooks/sliceTemplateLibrary-read"; -import { fsLimit } from "./lib/fsLimit"; + +import { checkIsTypeScriptProject } from "./checkIsTypeScriptProject"; + +import * as fs from "./lib/fsLimit"; export type ReadSliceTemplateLibraryArgs = { helpers: SliceMachineHelpers; @@ -47,9 +47,9 @@ export const readSliceTemplateLibrary = async ( const screenshotEntries = Object.entries(screenshotPaths); const screenshotPromises = screenshotEntries.map(([key, filePath]) => { - return fsLimit(() => - fs.readFile(path.join(dirName, filePath)).then((data) => [key, data]), - ); + return fs + .readFile(path.join(dirName, filePath)) + .then((data) => [key, data]); }); const readScreenshots = await Promise.all(screenshotPromises); const screenshots = Object.fromEntries(readScreenshots); @@ -58,8 +58,9 @@ export const readSliceTemplateLibrary = async ( ? componentFileNames.ts : componentFileNames.js; - const componentContentsTemplate = await fsLimit(() => - fs.readFile(path.join(dirName, model.name, fileName), "utf-8"), + const componentContentsTemplate = await fs.readFile( + path.join(dirName, model.name, fileName), + "utf-8", ); return { diff --git a/packages/plugin-kit/src/fs/renameSlice.ts b/packages/plugin-kit/src/fs/renameSlice.ts index a6707965a2..3cf7c034f0 100644 --- a/packages/plugin-kit/src/fs/renameSlice.ts +++ b/packages/plugin-kit/src/fs/renameSlice.ts @@ -7,7 +7,6 @@ import { BuildSliceDirectoryPathArgs, } from "./buildSliceDirectoryPath"; import { writeSliceModel, WriteSliceModelArgs } from "./writeSliceModel"; -import { fsLimit } from "./lib/fsLimit"; export type RenameSliceArgs = { actions: SliceMachineActions; @@ -20,18 +19,16 @@ export const renameSlice = async (args: RenameSliceArgs): Promise => { sliceID: args.model.id, }); - await fsLimit(async () => - fse.move( - await buildSliceDirectoryPath({ - ...args, - model: existingModel, - absolute: true, - }), - await buildSliceDirectoryPath({ - ...args, - absolute: true, - }), - ), + await fse.move( + await buildSliceDirectoryPath({ + ...args, + model: existingModel, + absolute: true, + }), + await buildSliceDirectoryPath({ + ...args, + absolute: true, + }), ); await writeSliceModel(args); diff --git a/packages/plugin-kit/src/fs/writeProjectFile.ts b/packages/plugin-kit/src/fs/writeProjectFile.ts index d37bbd40d7..414102c68e 100644 --- a/packages/plugin-kit/src/fs/writeProjectFile.ts +++ b/packages/plugin-kit/src/fs/writeProjectFile.ts @@ -1,8 +1,8 @@ -import * as fs from "node:fs/promises"; import * as path from "node:path"; import { SliceMachineHelpers } from "../createSliceMachineHelpers"; -import { fsLimit } from "./lib/fsLimit"; + +import * as fs from "./lib/fsLimit"; export type WriteProjectFileArgs = { filename: string; @@ -32,8 +32,8 @@ export const writeProjectFile = async ( ); } - await fsLimit(() => fs.mkdir(path.dirname(filePath), { recursive: true })); - await fsLimit(() => fs.writeFile(filePath, contents)); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, contents); return filePath; };