diff --git a/packages/next/src/build/build-context.ts b/packages/next/src/build/build-context.ts new file mode 100644 index 0000000000000..0130be1746b16 --- /dev/null +++ b/packages/next/src/build/build-context.ts @@ -0,0 +1,50 @@ +import { LoadedEnvFiles } from '@next/env' +import { Ora } from 'next/dist/compiled/ora' +import { Rewrite } from '../lib/load-custom-routes' +import { __ApiPreviewProps } from '../server/api-utils' +import { NextConfigComplete } from '../server/config-shared' +import { Span } from '../trace' +import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin' + +// a global object to store context for the current build +// this is used to pass data between different steps of the build without having +// to pass it through function arguments. +// Not exhaustive, but should be extended to as needed whilst refactoring +export const NextBuildContext: Partial<{ + // core fields + dir: string + buildId: string + config: NextConfigComplete + appDir: string + pagesDir: string + rewrites: { + fallback: Rewrite[] + afterFiles: Rewrite[] + beforeFiles: Rewrite[] + } + loadedEnvFiles: LoadedEnvFiles + previewProps: __ApiPreviewProps + mappedPages: + | { + [page: string]: string + } + | undefined + mappedAppPages: + | { + [page: string]: string + } + | undefined + mappedRootPaths: { + [page: string]: string + } + + // misc fields + telemetryPlugin: TelemetryPlugin + buildSpinner: Ora + nextBuildSpan: Span + + // cli fields + reactProductionProfiling: boolean + noMangling: boolean + appDirOnly: boolean +}> = {} diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 02ac7dc9762ad..5cc94d0d924a8 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -86,7 +86,7 @@ import { generateBuildId } from './generate-build-id' import { isWriteable } from './is-writeable' import * as Log from './output/log' import createSpinner from './spinner' -import { trace, flushAllTraces, setGlobal, Span } from '../trace' +import { trace, flushAllTraces, setGlobal } from '../trace' import { detectConflictingPaths, computeFromManifest, @@ -104,7 +104,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path' import { NextConfigComplete } from '../server/config-shared' import isError, { NextError } from '../lib/is-error' import { isEdgeRuntime } from '../lib/is-edge-runtime' -import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin' import { MiddlewareManifest } from './webpack/plugins/middleware-plugin' import { recursiveCopy } from '../lib/recursive-copy' import { recursiveReadDir } from '../lib/recursive-readdir' @@ -122,6 +121,7 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths' import { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin' import { RSC, RSC_VARY_HEADER } from '../client/components/app-router-headers' import { webpackBuild } from './webpack-build' +import { NextBuildContext } from './build-context' export type SsgRoute = { initialRevalidateSeconds: number | false @@ -144,13 +144,6 @@ export type PrerenderManifest = { preview: __ApiPreviewProps } -export const NextBuildContext: Partial<{ - telemetryPlugin: TelemetryPlugin - buildSpinner: any - nextBuildSpan: Span - dir: string -}> = {} - /** * typescript will be loaded in "next/lib/verifyTypeScriptSetup" and * then passed to "next/lib/typescript/runTypeCheck" as a parameter. @@ -252,33 +245,40 @@ export default async function build( const nextBuildSpan = trace('next-build', undefined, { version: process.env.__NEXT_VERSION as string, }) + NextBuildContext.nextBuildSpan = nextBuildSpan NextBuildContext.dir = dir + NextBuildContext.appDirOnly = appDirOnly + NextBuildContext.reactProductionProfiling = reactProductionProfiling + NextBuildContext.noMangling = noMangling const buildResult = await nextBuildSpan.traceAsyncFn(async () => { // attempt to load global env values so they are available in next.config.js const { loadedEnvFiles } = nextBuildSpan .traceChild('load-dotenv') .traceFn(() => loadEnvConfig(dir, false, Log)) + NextBuildContext.loadedEnvFiles = loadedEnvFiles const config: NextConfigComplete = await nextBuildSpan .traceChild('load-next-config') .traceAsyncFn(() => loadConfig(PHASE_PRODUCTION_BUILD, dir)) + NextBuildContext.config = config const distDir = path.join(dir, config.distDir) setGlobal('phase', PHASE_PRODUCTION_BUILD) setGlobal('distDir', distDir) - const { target } = config const buildId: string = await nextBuildSpan .traceChild('generate-buildid') .traceAsyncFn(() => generateBuildId(config.generateBuildId, nanoid)) + NextBuildContext.buildId = buildId const customRoutes: CustomRoutes = await nextBuildSpan .traceChild('load-custom-routes') .traceAsyncFn(() => loadCustomRoutes(config)) const { headers, rewrites, redirects } = customRoutes + NextBuildContext.rewrites = rewrites const cacheDir = path.join(distDir, 'cache') if (ciEnvironment.isCI && !ciEnvironment.hasNextSupport) { @@ -325,6 +325,9 @@ export default async function build( }) const { pagesDir, appDir } = findPagesDir(dir, isAppDirEnabled) + NextBuildContext.pagesDir = pagesDir + NextBuildContext.appDir = appDir + const isSrcDir = path .relative(dir, pagesDir || appDir || '') .startsWith('src') @@ -517,6 +520,7 @@ export default async function build( previewModeSigningKey: crypto.randomBytes(32).toString('hex'), previewModeEncryptionKey: crypto.randomBytes(32).toString('hex'), } + NextBuildContext.previewProps = previewProps const mappedPages = nextBuildSpan .traceChild('create-pages-mapping') @@ -529,6 +533,7 @@ export default async function build( pagesDir, }) ) + NextBuildContext.mappedPages = mappedPages let mappedAppPages: { [page: string]: string } | undefined let denormalizedAppPages: string[] | undefined @@ -545,6 +550,7 @@ export default async function build( pagesDir: pagesDir, }) ) + NextBuildContext.mappedAppPages = mappedAppPages } let mappedRootPaths: { [page: string]: string } = {} @@ -557,6 +563,7 @@ export default async function build( pagesDir: pagesDir, }) } + NextBuildContext.mappedRootPaths = mappedRootPaths const pagesPageKeys = Object.keys(mappedPages) @@ -920,32 +927,7 @@ export default async function build( ignore: [] as string[], })) - const webpackBuildDuration = await webpackBuild( - { - buildId, - config, - pagesDir, - reactProductionProfiling, - rewrites, - target, - appDir, - noMangling, - }, - { - buildId, - config, - envFiles: loadedEnvFiles, - isDev: false, - pages: mappedPages, - pagesDir, - previewMode: previewProps, - rootDir: dir, - rootPaths: mappedRootPaths, - appDir, - appPaths: mappedAppPages, - pageExtensions: config.pageExtensions, - } - ) + const webpackBuildDuration = await webpackBuild() telemetry.record( eventBuildCompleted(pagesPaths, { diff --git a/packages/next/src/build/webpack-build.ts b/packages/next/src/build/webpack-build.ts index 13663a060f374..410c35c6be0cc 100644 --- a/packages/next/src/build/webpack-build.ts +++ b/packages/next/src/build/webpack-build.ts @@ -6,6 +6,7 @@ import { COMPILER_NAMES, CLIENT_STATIC_FILES_RUNTIME_MAIN_APP, APP_CLIENT_INTERNALS, + PHASE_PRODUCTION_BUILD, } from '../shared/lib/constants' import { runCompiler } from './compiler' import * as Log from './output/log' @@ -13,10 +14,11 @@ import getBaseWebpackConfig from './webpack-config' import { NextError } from '../lib/is-error' import { injectedClientEntries } from './webpack/plugins/flight-client-entry-plugin' import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin' -import { Rewrite } from '../lib/load-custom-routes' -import { NextConfigComplete } from '../server/config-shared' -import { NextBuildContext } from '.' -import { CreateEntrypointsParams, createEntrypoints } from './entries' +import { NextBuildContext } from './build-context' +import { isMainThread, parentPort, Worker, workerData } from 'worker_threads' +import { createEntrypoints } from './entries' +import loadConfig from '../server/config' +import { trace } from '../trace' type CompilerResult = { errors: webpack.StatsError[] @@ -34,23 +36,7 @@ function isTelemetryPlugin(plugin: unknown): plugin is TelemetryPlugin { return plugin instanceof TelemetryPlugin } -export async function webpackBuild( - commonWebpackOptions: { - buildId: string - config: NextConfigComplete - pagesDir: string | undefined - reactProductionProfiling: boolean - rewrites: { - fallback: Rewrite[] - afterFiles: Rewrite[] - beforeFiles: Rewrite[] - } - target: string - appDir: string | undefined - noMangling: boolean - }, - entrypointsParams: CreateEntrypointsParams -): Promise { +async function webpackBuildImpl(): Promise { let result: CompilerResult | null = { warnings: [], errors: [], @@ -60,128 +46,152 @@ export async function webpackBuild( const nextBuildSpan = NextBuildContext.nextBuildSpan! const buildSpinner = NextBuildContext.buildSpinner const dir = NextBuildContext.dir! - await (async () => { - // IIFE to isolate locals and avoid retaining memory too long - const runWebpackSpan = nextBuildSpan.traceChild('run-webpack-compiler') - - const entrypoints = await nextBuildSpan - .traceChild('create-entrypoints') - .traceAsyncFn(() => createEntrypoints(entrypointsParams)) - - const configs = await runWebpackSpan - .traceChild('generate-webpack-config') - .traceAsyncFn(() => - Promise.all([ - getBaseWebpackConfig(dir, { - ...commonWebpackOptions, - runWebpackSpan, - middlewareMatchers: entrypoints.middlewareMatchers, - compilerType: COMPILER_NAMES.client, - entrypoints: entrypoints.client, - }), - getBaseWebpackConfig(dir, { - ...commonWebpackOptions, - runWebpackSpan, - middlewareMatchers: entrypoints.middlewareMatchers, - compilerType: COMPILER_NAMES.server, - entrypoints: entrypoints.server, - }), - getBaseWebpackConfig(dir, { - ...commonWebpackOptions, - runWebpackSpan, - middlewareMatchers: entrypoints.middlewareMatchers, - compilerType: COMPILER_NAMES.edgeServer, - entrypoints: entrypoints.edgeServer, - }), - ]) - ) - const clientConfig = configs[0] + const runWebpackSpan = nextBuildSpan.traceChild('run-webpack-compiler') + const entrypoints = await nextBuildSpan + .traceChild('create-entrypoints') + .traceAsyncFn(() => + createEntrypoints({ + buildId: NextBuildContext.buildId!, + config: NextBuildContext.config!, + envFiles: NextBuildContext.loadedEnvFiles!, + isDev: false, + rootDir: dir, + pageExtensions: NextBuildContext.config!.pageExtensions!, + pagesDir: NextBuildContext.pagesDir!, + appDir: NextBuildContext.appDir!, + pages: NextBuildContext.mappedPages!, + appPaths: NextBuildContext.mappedAppPages!, + previewMode: NextBuildContext.previewProps!, + rootPaths: NextBuildContext.mappedRootPaths!, + }) + ) - if ( - clientConfig.optimization && - (clientConfig.optimization.minimize !== true || - (clientConfig.optimization.minimizer && - clientConfig.optimization.minimizer.length === 0)) - ) { - Log.warn( - `Production code optimization has been disabled in your project. Read more: https://nextjs.org/docs/messages/minification-disabled` - ) - } + const commonWebpackOptions = { + isServer: false, + buildId: NextBuildContext.buildId!, + config: NextBuildContext.config!, + target: NextBuildContext.config!.target!, + appDir: NextBuildContext.appDir!, + pagesDir: NextBuildContext.pagesDir!, + rewrites: NextBuildContext.rewrites!, + reactProductionProfiling: NextBuildContext.reactProductionProfiling!, + noMangling: NextBuildContext.noMangling!, + } - webpackBuildStart = process.hrtime() + const configs = await runWebpackSpan + .traceChild('generate-webpack-config') + .traceAsyncFn(() => + Promise.all([ + getBaseWebpackConfig(dir, { + ...commonWebpackOptions, + middlewareMatchers: entrypoints.middlewareMatchers, + runWebpackSpan, + compilerType: COMPILER_NAMES.client, + entrypoints: entrypoints.client, + }), + getBaseWebpackConfig(dir, { + ...commonWebpackOptions, + runWebpackSpan, + middlewareMatchers: entrypoints.middlewareMatchers, + compilerType: COMPILER_NAMES.server, + entrypoints: entrypoints.server, + }), + getBaseWebpackConfig(dir, { + ...commonWebpackOptions, + runWebpackSpan, + middlewareMatchers: entrypoints.middlewareMatchers, + compilerType: COMPILER_NAMES.edgeServer, + entrypoints: entrypoints.edgeServer, + }), + ]) + ) - // We run client and server compilation separately to optimize for memory usage - await runWebpackSpan.traceAsyncFn(async () => { - // Run the server compilers first and then the client - // compiler to track the boundary of server/client components. - let clientResult: SingleCompilerResult | null = null + const clientConfig = configs[0] - // During the server compilations, entries of client components will be - // injected to this set and then will be consumed by the client compiler. - injectedClientEntries.clear() + if ( + clientConfig.optimization && + (clientConfig.optimization.minimize !== true || + (clientConfig.optimization.minimizer && + clientConfig.optimization.minimizer.length === 0)) + ) { + Log.warn( + `Production code optimization has been disabled in your project. Read more: https://nextjs.org/docs/messages/minification-disabled` + ) + } - const serverResult = await runCompiler(configs[1], { - runWebpackSpan, - }) - const edgeServerResult = configs[2] - ? await runCompiler(configs[2], { runWebpackSpan }) - : null - - // Only continue if there were no errors - if (!serverResult.errors.length && !edgeServerResult?.errors.length) { - injectedClientEntries.forEach((value, key) => { - const clientEntry = clientConfig.entry as webpack.EntryObject - if (key === APP_CLIENT_INTERNALS) { - clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] = [ - // TODO-APP: cast clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] to type EntryDescription once it's available from webpack - // @ts-expect-error clientEntry['main-app'] is type EntryDescription { import: ... } - ...clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP].import, - value, - ] - } else { - clientEntry[key] = { - dependOn: [CLIENT_STATIC_FILES_RUNTIME_MAIN_APP], - import: value, - } - } - }) + webpackBuildStart = process.hrtime() - clientResult = await runCompiler(clientConfig, { - runWebpackSpan, - }) - } + // We run client and server compilation separately to optimize for memory usage + await runWebpackSpan.traceAsyncFn(async () => { + // Run the server compilers first and then the client + // compiler to track the boundary of server/client components. + let clientResult: SingleCompilerResult | null = null - result = { - warnings: ([] as any[]) - .concat( - clientResult?.warnings, - serverResult?.warnings, - edgeServerResult?.warnings - ) - .filter(nonNullable), - errors: ([] as any[]) - .concat( - clientResult?.errors, - serverResult?.errors, - edgeServerResult?.errors - ) - .filter(nonNullable), - stats: [ - clientResult?.stats, - serverResult?.stats, - edgeServerResult?.stats, - ], - } + // During the server compilations, entries of client components will be + // injected to this set and then will be consumed by the client compiler. + injectedClientEntries.clear() + + const serverResult = await runCompiler(configs[1], { + runWebpackSpan, }) - result = nextBuildSpan - .traceChild('format-webpack-messages') - .traceFn(() => formatWebpackMessages(result, true)) + const edgeServerResult = configs[2] + ? await runCompiler(configs[2], { runWebpackSpan }) + : null + + // Only continue if there were no errors + if (!serverResult.errors.length && !edgeServerResult?.errors.length) { + injectedClientEntries.forEach((value, key) => { + const clientEntry = clientConfig.entry as webpack.EntryObject + if (key === APP_CLIENT_INTERNALS) { + clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] = [ + // TODO-APP: cast clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] to type EntryDescription once it's available from webpack + // @ts-expect-error clientEntry['main-app'] is type EntryDescription { import: ... } + ...clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP].import, + value, + ] + } else { + clientEntry[key] = { + dependOn: [CLIENT_STATIC_FILES_RUNTIME_MAIN_APP], + import: value, + } + } + }) + + clientResult = await runCompiler(clientConfig, { + runWebpackSpan, + }) + } - NextBuildContext.telemetryPlugin = ( - clientConfig as webpack.Configuration - ).plugins?.find(isTelemetryPlugin) - })() + result = { + warnings: ([] as any[]) + .concat( + clientResult?.warnings, + serverResult?.warnings, + edgeServerResult?.warnings + ) + .filter(nonNullable), + errors: ([] as any[]) + .concat( + clientResult?.errors, + serverResult?.errors, + edgeServerResult?.errors + ) + .filter(nonNullable), + stats: [ + clientResult?.stats, + serverResult?.stats, + edgeServerResult?.stats, + ], + } + }) + result = nextBuildSpan + .traceChild('format-webpack-messages') + .traceFn(() => formatWebpackMessages(result, true)) as CompilerResult + + NextBuildContext.telemetryPlugin = ( + clientConfig as webpack.Configuration + ).plugins?.find(isTelemetryPlugin) const webpackBuildEnd = process.hrtime(webpackBuildStart) if (buildSpinner) { @@ -237,3 +247,70 @@ export async function webpackBuild( return webpackBuildEnd[0] } } + +// the main function when this file is run as a worker +async function workerMain() { + const { buildContext } = workerData + // setup new build context from the serialized data passed from the parent + Object.assign(NextBuildContext, buildContext) + + /// load the config because it's not serializable + NextBuildContext.config = await loadConfig( + PHASE_PRODUCTION_BUILD, + NextBuildContext.dir!, + undefined, + undefined, + true + ) + NextBuildContext.nextBuildSpan = trace('next-build') + + try { + const result = await webpackBuildImpl() + parentPort!.postMessage(result) + } catch (e) { + parentPort!.postMessage(e) + } finally { + process.exit(0) + } +} + +if (!isMainThread) { + workerMain() +} + +async function webpackBuildWithWorker() { + const { + config, + telemetryPlugin, + buildSpinner, + nextBuildSpan, + ...prunedBuildContext + } = NextBuildContext + const worker = new Worker(new URL(import.meta.url), { + workerData: { + buildContext: prunedBuildContext, + }, + }) + + const result = await new Promise((resolve, reject) => { + worker.on('message', resolve) + worker.on('error', reject) + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker stopped with exit code ${code}`)) + } + }) + }) + + return result as number +} + +export async function webpackBuild() { + const config = NextBuildContext.config! + + if (config.experimental.webpackBuildWorker) { + return await webpackBuildWithWorker() + } else { + return await webpackBuildImpl() + } +} diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index e367b2ce70cec..812fda8f3e77f 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -438,6 +438,9 @@ const configSchema = { mdxRs: { type: 'boolean', }, + webpackBuildWorker: { + type: 'boolean', + }, turbopackLoaders: { type: 'object', }, diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 3c5d3a6faa54b..bebed4988c13f 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -178,6 +178,10 @@ export interface ExperimentalConfig { memoryLimit?: number } mdxRs?: boolean + /** + * This option is to enable running the Webpack build in a worker thread. + */ + webpackBuildWorker?: boolean } export type ExportPathMap = {