From 852938df7fe506ece84014abd911b816e6c1bb59 Mon Sep 17 00:00:00 2001 From: Lukas Holzer Date: Mon, 21 Nov 2022 10:54:05 +0100 Subject: [PATCH] fix: fixes the yarn pnp issue for plugins --- packages/build/src/plugins/child/logic.js | 64 ----------- packages/build/src/plugins/child/logic.ts | 85 ++++++++++++++ packages/build/src/plugins/child/main.js | 1 + packages/build/src/plugins/{ipc.js => ipc.ts} | 9 +- packages/build/src/plugins/options.ts | 17 +++ .../src/plugins/{resolve.js => resolve.ts} | 105 +++++++++++++----- .../build/src/plugins/{spawn.js => spawn.ts} | 31 +++++- .../src/utils/{resolve.js => resolve.ts} | 20 ++-- 8 files changed, 224 insertions(+), 108 deletions(-) delete mode 100644 packages/build/src/plugins/child/logic.js create mode 100644 packages/build/src/plugins/child/logic.ts rename packages/build/src/plugins/{ipc.js => ipc.ts} (93%) rename packages/build/src/plugins/{resolve.js => resolve.ts} (59%) rename packages/build/src/plugins/{spawn.js => spawn.ts} (69%) rename packages/build/src/utils/{resolve.js => resolve.ts} (59%) diff --git a/packages/build/src/plugins/child/logic.js b/packages/build/src/plugins/child/logic.js deleted file mode 100644 index fb68cf6af3..0000000000 --- a/packages/build/src/plugins/child/logic.js +++ /dev/null @@ -1,64 +0,0 @@ -import { createRequire } from 'module' -import { pathToFileURL } from 'url' - -import { ROOT_PACKAGE_JSON } from '../../utils/json.js' -import { DEV_EVENTS, EVENTS } from '../events.js' - -import { addTsErrorInfo } from './typescript.js' - -const require = createRequire(import.meta.url) - -// Require the plugin file and fire its top-level function. -// The returned object is the `logic` which includes all event handlers. -export const getLogic = async function ({ pluginPath, inputs, tsNodeService }) { - const logic = await importLogic(pluginPath, tsNodeService) - const logicA = loadLogic({ logic, inputs }) - return logicA -} - -const importLogic = async function (pluginPath, tsNodeService) { - try { - // `ts-node` is not available programmatically for pure ES modules yet, - // which is currently making it impossible for local plugins to use both - // pure ES modules and TypeScript. - if (tsNodeService !== undefined) { - return require(pluginPath) - } - - // `pluginPath` is an absolute file path but `import()` needs URLs. - // Converting those with `pathToFileURL()` is needed especially on Windows - // where the drive letter would not work with `import()`. - const returnValue = await import(pathToFileURL(pluginPath)) - // Plugins should use named exports, but we still support default exports - // for backward compatibility with CommonJS - return returnValue.default === undefined ? returnValue : returnValue.default - } catch (error) { - addTsErrorInfo(error, tsNodeService) - // We must change `error.stack` instead of `error.message` because some - // errors thrown from `import()` access `error.stack` before throwing. - // `error.stack` is lazily instantiated by Node.js, so changing - // `error.message` afterwards would not modify `error.stack`. Therefore, the - // resulting stack trace, which is printed in the build logs, would not - // include the additional message prefix. - error.stack = `Could not import plugin:\n${error.stack}` - throw error - } -} - -const loadLogic = function ({ logic, inputs }) { - if (typeof logic !== 'function') { - return logic - } - - const metadata = { - events: new Set([...DEV_EVENTS, ...EVENTS]), - version: ROOT_PACKAGE_JSON.version, - } - - try { - return logic(inputs, metadata) - } catch (error) { - error.message = `Could not load plugin:\n${error.message}` - throw error - } -} diff --git a/packages/build/src/plugins/child/logic.ts b/packages/build/src/plugins/child/logic.ts new file mode 100644 index 0000000000..7f0e679b09 --- /dev/null +++ b/packages/build/src/plugins/child/logic.ts @@ -0,0 +1,85 @@ +import { readdirSync } from 'fs' +import { createRequire } from 'module' +import { pathToFileURL } from 'url' + +import { execaNode } from 'execa' + +import { ROOT_PACKAGE_JSON } from '../../utils/json.js' +import { DEV_EVENTS, EVENTS } from '../events.js' + +import { addTsErrorInfo } from './typescript.js' + +const require = createRequire(import.meta.url) + +// Require the plugin file and fire its top-level function. +// The returned object is the `logic` which includes all event handlers. +export const getLogic = async function ({ pluginPath, inputs, tsNodeService }) { + const logic = await importLogic(pluginPath, tsNodeService) + const logicA = loadLogic({ logic, inputs }) + return logicA +} + +const importLogic = async function (pluginPath: string, tsNodeService) { + console.log(readdirSync('/Users/lukasholzer/Sites/tmp/effy.space')) + const b = await import('file:///Users/lukasholzer/Sites/tmp/effy.space/.pnp.cjs') + console.log(b.default.resolveRequest('')) + b.default.setup() + + const res = require.resolve('netlify-plugin-inline-source') + console.log(res) + // const c = await execaNode( + // '/Users/lukasholzer/Sites/tmp/effy.space/.yarn/cache/netlify-plugin-inline-source-npm-1.0.4-94542d11f7-edcba2b6aa.zip', + // [], + // { + // stdio: 'inherit', + // }, + // ) + // console.log(process.cwd()) + // const a = await require('netlify-plugin-inline-source') + // console.log(a) + // try { + // // `ts-node` is not available programmatically for pure ES modules yet, + // // which is currently making it impossible for local plugins to use both + // // pure ES modules and TypeScript. + // if (tsNodeService !== undefined) { + // return require(pluginPath) + // } + + // // `pluginPath` is an absolute file path but `import()` needs URLs. + // // Converting those with `pathToFileURL()` is needed especially on Windows + // // where the drive letter would not work with `import()`. + // const returnValue = await import(pathToFileURL(pluginPath)) + // // Plugins should use named exports, but we still support default exports + // // for backward compatibility with CommonJS + // return returnValue.default === undefined ? returnValue : returnValue.default + // } catch (error) { + // addTsErrorInfo(error, tsNodeService) + // // We must change `error.stack` instead of `error.message` because some + // // errors thrown from `import()` access `error.stack` before throwing. + // // `error.stack` is lazily instantiated by Node.js, so changing + // // `error.message` afterwards would not modify `error.stack`. Therefore, the + // // resulting stack trace, which is printed in the build logs, would not + // // include the additional message prefix. + // error.stack = `Could not import plugin:\n${error.stack}` + // throw error + // } + return {} +} + +const loadLogic = function ({ logic, inputs }) { + if (typeof logic !== 'function') { + return logic + } + + const metadata = { + events: new Set([...DEV_EVENTS, ...EVENTS]), + version: ROOT_PACKAGE_JSON.version, + } + + try { + return logic(inputs, metadata) + } catch (error) { + error.message = `Could not load plugin:\n${error.message}` + throw error + } +} diff --git a/packages/build/src/plugins/child/main.js b/packages/build/src/plugins/child/main.js index c957cb28f7..7c8ce52fb1 100644 --- a/packages/build/src/plugins/child/main.js +++ b/packages/build/src/plugins/child/main.js @@ -1,3 +1,4 @@ +console.log('!!!! CHILD \n\n') import { setInspectColors } from '../../log/colors.js' import { sendEventToParent, getEventsFromParent } from '../ipc.js' diff --git a/packages/build/src/plugins/ipc.js b/packages/build/src/plugins/ipc.ts similarity index 93% rename from packages/build/src/plugins/ipc.js rename to packages/build/src/plugins/ipc.ts index 19509b4d49..7bc243bdb3 100644 --- a/packages/build/src/plugins/ipc.js +++ b/packages/build/src/plugins/ipc.ts @@ -1,6 +1,7 @@ import process from 'process' import { promisify } from 'util' +import type { ExecaChildProcess } from 'execa' import { pEvent } from 'p-event' import { v4 as uuidv4 } from 'uuid' @@ -34,7 +35,7 @@ export const callChild = async function ({ childProcess, eventName, payload, log // - child process `exit` // In the later two cases, we propagate the error. // We need to make `p-event` listeners are properly cleaned up too. -export const getEventFromChild = async function (childProcess, callId) { +export const getEventFromChild = async function (childProcess: ExecaChildProcess, callId: 'ready' | string) { if (childProcessHasExited(childProcess)) { throw getChildExitError('Could not receive event from child process because it already exited.') } @@ -124,6 +125,8 @@ export const sendEventToParent = async function (callId, payload, verbose, error // Error static properties are not serializable through `child_process` // (which uses `v8.serialize()` under the hood) so we need to convert from/to // plain objects. +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-expect-error const serializePayload = function ({ error = {}, error: { name } = {}, ...payload }) { if (name === undefined) { return payload @@ -133,11 +136,13 @@ const serializePayload = function ({ error = {}, error: { name } = {}, ...payloa return { ...payload, error: errorA } } +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-expect-error const parsePayload = function ({ error = {}, error: { name } = {}, ...payload }) { if (name === undefined) { return payload } - const errorA = jsonToError(error) + const errorA = jsonToError(error as any) return { ...payload, error: errorA } } diff --git a/packages/build/src/plugins/options.ts b/packages/build/src/plugins/options.ts index 8c52afa15b..fa916dfb73 100644 --- a/packages/build/src/plugins/options.ts +++ b/packages/build/src/plugins/options.ts @@ -3,6 +3,7 @@ import { dirname } from 'path' import { PackageJson } from 'read-pkg-up' import semver from 'semver' +import { Mode } from '../core/types.js' import { addErrorInfo } from '../error/info.js' import { installLocalPluginsDependencies } from '../install/local.js' import { measureDuration } from '../time/main.js' @@ -12,6 +13,15 @@ import { getPackageJson } from '../utils/package.js' import { useManifest } from './manifest/main.js' import { resolvePluginsPath } from './resolve.js' +export type PluginsOptions = { + packageName: string + pluginPath?: unknown + pinnedVersion: undefined + loadedFrom: undefined | 'auto_install' | string + origin: 'config' | string + inputs: Record +} + // Load core plugins and user plugins const tGetPluginsOptions = async function ({ pluginsOptions, @@ -28,6 +38,13 @@ const tGetPluginsOptions = async function ({ sendStatus, testOpts, featureFlags, +}: { + pluginsOptions: PluginsOptions[] + mode: Mode + buildDir: string + nodePath: string + packageJson: PackageJson + [key: string]: any }) { const pluginsOptionsA = await resolvePluginsPath({ pluginsOptions, diff --git a/packages/build/src/plugins/resolve.js b/packages/build/src/plugins/resolve.ts similarity index 59% rename from packages/build/src/plugins/resolve.js rename to packages/build/src/plugins/resolve.ts index 71ab4200fd..6a3b959dc3 100644 --- a/packages/build/src/plugins/resolve.js +++ b/packages/build/src/plugins/resolve.ts @@ -1,16 +1,35 @@ +import { existsSync, readdirSync } from 'fs' +import { createRequire } from 'module' +import { join } from 'path' + +import { PackageJson } from 'read-pkg-up' + +import type { Mode } from '../core/types.js' import { addErrorInfo } from '../error/info.js' import { installMissingPlugins } from '../install/missing.js' import { resolvePath, tryResolvePath } from '../utils/resolve.js' import { addExpectedVersions } from './expected_version.js' import { addPluginsNodeVersion } from './node_version.js' +import type { PluginsOptions } from './options.js' import { addPinnedVersions } from './pinned_version.js' -// Try to find plugins in four places, by priority order: -// - already loaded (core plugins) -// - local plugin -// - external plugin already installed in `node_modules`, most likely through `package.json` -// - automatically installed by us, to `.netlify/plugins/` +const AUTO_PLUGINS_DIR = '.netlify/plugins/' + +/** + * Find the path to the directory used to install plugins automatically. + * It is a subdirectory of `buildDir`, so that the plugin can require the + * project's dependencies (peer dependencies). + */ +const getAutoPluginsDir = (buildDir: string): string => `${buildDir}/${AUTO_PLUGINS_DIR}` + +/** + * Try to find plugins in four places, by priority order: + * - already loaded (core plugins) + * - local plugin + * - external plugin already installed in `node_modules`, most likely through `package.json` + * - automatically installed by us, to `.netlify/plugins/` + */ export const resolvePluginsPath = async function ({ pluginsOptions, siteInfo, @@ -25,6 +44,17 @@ export const resolvePluginsPath = async function ({ sendStatus, testOpts, featureFlags, +}: { + pluginsOptions: PluginsOptions[] + buildDir: string + nodePath: string + packageJson: PackageJson + userNodeVersion: string + mode: Mode + debug: boolean + sendStatus: boolean + featureFlags: Record + [key: string]: any }) { const autoPluginsDir = getAutoPluginsDir(buildDir) const pluginsOptionsA = await Promise.all( @@ -35,7 +65,7 @@ export const resolvePluginsPath = async function ({ nodePath, userNodeVersion, logs, - }) + } as any) const pluginsOptionsC = await addPinnedVersions({ pluginsOptions: pluginsOptionsB, api, siteInfo, sendStatus }) const pluginsOptionsD = await addExpectedVersions({ pluginsOptions: pluginsOptionsC, @@ -56,37 +86,61 @@ export const resolvePluginsPath = async function ({ return pluginsOptionsE } -// Find the path to the directory used to install plugins automatically. -// It is a subdirectory of `buildDir`, so that the plugin can require the -// project's dependencies (peer dependencies). -const getAutoPluginsDir = function (buildDir) { - return `${buildDir}/${AUTO_PLUGINS_DIR}` -} - -const AUTO_PLUGINS_DIR = '.netlify/plugins/' - +/** + * Tries to resolve the plugin locally if the `loadedFrom` is not defined + * - Core plugins have `loadedFrom` set to `undefined` so they are skipped + * - If the `packageName` starts with a `./` we try to resolve it locally + * - other than that we try to get it from the `package.json` + * - if we cannot resolve it we fallback to `auto_install` + * - can happen if the plugin name got misspelled + * - is not in our official list + * - is in our official list but has not been installed by this site yet + * @returns + */ const resolvePluginPath = async function ({ pluginOptions, pluginOptions: { packageName, loadedFrom }, buildDir, autoPluginsDir, +}: { + pluginOptions: PluginsOptions + buildDir: string + autoPluginsDir: string }) { // Core plugins if (loadedFrom !== undefined) { return pluginOptions } - const localPackageName = normalizeLocalPackageName(packageName) - // Local plugins + const localPackageName = normalizeLocalPackageName(packageName) if (localPackageName.startsWith('.')) { const { path: localPath, error } = await tryResolvePath(localPackageName, buildDir) - validateLocalPluginPath(error, localPackageName) + // When requiring a local plugin with an invalid file path + if (error !== undefined) { + error.message = `Plugin could not be found using local path: ${localPackageName}\n${error.message}` + addErrorInfo(error, { type: 'resolveConfig' }) + throw error + } + return { ...pluginOptions, pluginPath: localPath, loadedFrom: 'local' } } + // Yarn Plug and play does not have node_modules anymore + // It is required to load the .pnp.cjs and call the setup function otherwise resolving won't work anymore+ + // https://yarnpkg.com/features/pnp + // + // see https://github.com/netlify/build/issues/1535 + if (existsSync(join(buildDir, '.pnp.cjs'))) { + const { + default: { setup }, + } = await import(join(buildDir, '.pnp.cjs')) + setup() + } + // Plugin added to `package.json` const { path: manualPath } = await tryResolvePath(packageName, buildDir) + if (manualPath !== undefined) { return { ...pluginOptions, pluginPath: manualPath, loadedFrom: 'package.json' } } @@ -104,8 +158,8 @@ const resolvePluginPath = async function ({ return { ...pluginOptions, loadedFrom: 'auto_install' } } -// `packageName` starting with `/` are relative to the build directory -const normalizeLocalPackageName = function (packageName) { +/** Normalizes the `packageName` if it starts with `/` to be relative to the build directory */ +const normalizeLocalPackageName = (packageName: string) => { if (packageName.startsWith('/')) { return `.${packageName}` } @@ -113,15 +167,6 @@ const normalizeLocalPackageName = function (packageName) { return packageName } -// When requiring a local plugin with an invalid file path -const validateLocalPluginPath = function (error, localPackageName) { - if (error !== undefined) { - error.message = `Plugin could not be found using local path: ${localPackageName}\n${error.message}` - addErrorInfo(error, { type: 'resolveConfig' }) - throw error - } -} - // Install plugins from the official list that have not been previously installed. // Print a warning if they have not been installed through the UI. const handleMissingPlugins = async function ({ pluginsOptions, autoPluginsDir, mode, logs }) { @@ -139,7 +184,7 @@ const handleMissingPlugins = async function ({ pluginsOptions, autoPluginsDir, m // Resolve the plugins that just got automatically installed const resolveMissingPluginPath = async function ({ pluginOptions, pluginOptions: { packageName }, autoPluginsDir }) { - if (!isMissingPlugin(pluginOptions)) { + if (!isMissingPlugin(pluginOptions as any)) { return pluginOptions } diff --git a/packages/build/src/plugins/spawn.js b/packages/build/src/plugins/spawn.ts similarity index 69% rename from packages/build/src/plugins/spawn.js rename to packages/build/src/plugins/spawn.ts index 66965976dd..99f1983996 100644 --- a/packages/build/src/plugins/spawn.js +++ b/packages/build/src/plugins/spawn.ts @@ -1,3 +1,5 @@ +import { existsSync } from 'fs' +import { join } from 'path' import { fileURLToPath } from 'url' import { execaNode } from 'execa' @@ -29,15 +31,37 @@ const tStartPlugins = async function ({ pluginsOptions, buildDir, childEnv, logs logIncompatiblePlugins(logs, pluginsOptions) const childProcesses = await Promise.all( - pluginsOptions.map(({ pluginDir, nodePath }) => startPlugin({ pluginDir, nodePath, buildDir, childEnv })), + pluginsOptions.map(({ pluginDir, nodePath }) => startPlugin({ pluginDir, nodePath, buildDir, childEnv, debug })), ) return { childProcesses } } export const startPlugins = measureDuration(tStartPlugins, 'start_plugins') -const startPlugin = async function ({ pluginDir, nodePath, buildDir, childEnv }) { - const childProcess = execaNode(CHILD_MAIN_FILE, { +const startPlugin = async function ({ + pluginDir, + nodePath, + buildDir, + childEnv, + debug, +}: { + pluginDir: string + nodePath: string + buildDir: string + childEnv: Record + debug: boolean +}) { + const nodeOptions: string[] = [] + + // Yarn Plug and play does not have node_modules anymore + // It is required to load the .pnp.cjs and call the setup function otherwise + // importing and resolving dependencies does not work anymore + // https://yarnpkg.com/features/pnp + if (existsSync(join(buildDir, '.pnp.cjs'))) { + nodeOptions.push(`--require ${join(buildDir, '.pnp.cjs')}`) + } + + const childProcess = execaNode(CHILD_MAIN_FILE, [], { cwd: buildDir, preferLocal: true, localDir: pluginDir, @@ -46,6 +70,7 @@ const startPlugin = async function ({ pluginDir, nodePath, buildDir, childEnv }) env: childEnv, extendEnv: false, serialization: 'advanced', + ...(debug && { stdio: 'inherit' }), // in debug mode we want to log out all messages }) try { diff --git a/packages/build/src/utils/resolve.js b/packages/build/src/utils/resolve.ts similarity index 59% rename from packages/build/src/utils/resolve.js rename to packages/build/src/utils/resolve.ts index f5269f3ade..21be6a9ca7 100644 --- a/packages/build/src/utils/resolve.js +++ b/packages/build/src/utils/resolve.ts @@ -6,8 +6,8 @@ import { async as resolveLib } from 'resolve' // flags const require = createRequire(import.meta.url) -// Like `resolvePath()` but does not throw -export const tryResolvePath = async function (path, basedir) { +/** Like `resolvePath()` but does not throw */ +export const tryResolvePath = async function (path: string, basedir: string) { try { const resolvedPath = await resolvePath(path, basedir) return { path: resolvedPath } @@ -16,8 +16,8 @@ export const tryResolvePath = async function (path, basedir) { } } -// This throws if the package cannot be found -export const resolvePath = async function (path, basedir) { +/** This throws if the package cannot be found */ +export const resolvePath = async function (path: string, basedir: string): Promise { try { return await resolvePathWithBasedir(path, basedir) // Fallback. @@ -28,11 +28,13 @@ export const resolvePath = async function (path, basedir) { } } -// Like `require.resolve()` but as an external library. -// We need to use `new Promise()` due to a bug with `utils.promisify()` on -// `resolve`: -// https://github.com/browserify/resolve/issues/151#issuecomment-368210310 -const resolvePathWithBasedir = function (path, basedir) { +/** + * Like `require.resolve()` but as an external library. + * We need to use `new Promise()` due to a bug with `utils.promisify()` on + * `resolve`: + * https://github.com/browserify/resolve/issues/151#issuecomment-368210310 + */ +const resolvePathWithBasedir = function (path: string, basedir: string): Promise { return new Promise((resolve, reject) => { resolveLib(path, { basedir }, (error, resolvedPath) => { if (error) {