Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

misc: remove IIFE from webpack build fn (5/6) #45459

Merged
merged 2 commits into from
Feb 1, 2023
Merged

misc: remove IIFE from webpack build fn (5/6) #45459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e64f7e
misc: remove IIFE from webpack build fn
feedthejim Jan 31, 2023
92c0e1d
perf: add experimental option to run webpack build in a worker (6/6) …
feedthejim Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
344 changes: 205 additions & 139 deletions packages/next/src/build/webpack-build.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
COMPILER_NAMES,
CLIENT_STATIC_FILES_RUNTIME_MAIN_APP,
APP_CLIENT_INTERNALS,
PHASE_PRODUCTION_BUILD,
} from '../shared/lib/constants'
import { runCompiler } from './compiler'
import * as Log from './output/log'
Expand All @@ -14,8 +15,10 @@ import { NextError } from '../lib/is-error'
import { injectedClientEntries } from './webpack/plugins/flight-client-entry-plugin'
import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
import { NextBuildContext } from './build-context'

import { isMainThread, parentPort, Worker, workerData } from 'worker_threads'
import { createEntrypoints } from './entries'
import loadConfig from '../server/config'
import { trace } from '../trace'

type CompilerResult = {
errors: webpack.StatsError[]
Expand All @@ -33,7 +36,7 @@ function isTelemetryPlugin(plugin: unknown): plugin is TelemetryPlugin {
return plugin instanceof TelemetryPlugin
}

export async function webpackBuild(): Promise<number> {
async function webpackBuildImpl(): Promise<number> {
let result: CompilerResult | null = {
warnings: [],
errors: [],
Expand All @@ -44,155 +47,151 @@ export async function webpackBuild(): Promise<number> {
const buildSpinner = NextBuildContext.buildSpinner
const dir = NextBuildContext.dir!

await (async () => {
// IIFE to isolate locals and avoid retaining memory too long
const runWebpackSpan = nextBuildSpan.traceChild('run-webpack-compiler')

const entrypoints = await nextBuildSpan
.traceChild('create-entrypoints')
.traceAsyncFn(() =>
createEntrypoints({
buildId: NextBuildContext.buildId!,
config: NextBuildContext.config!,
envFiles: NextBuildContext.loadedEnvFiles!,
isDev: false,
rootDir: dir,
pageExtensions: NextBuildContext.config!.pageExtensions!,
pagesDir: NextBuildContext.pagesDir!,
appDir: NextBuildContext.appDir!,
pages: NextBuildContext.mappedPages!,
appPaths: NextBuildContext.mappedAppPages!,
previewMode: NextBuildContext.previewProps!,
rootPaths: NextBuildContext.mappedRootPaths!,
})
)
const runWebpackSpan = nextBuildSpan.traceChild('run-webpack-compiler')
const entrypoints = await nextBuildSpan
.traceChild('create-entrypoints')
.traceAsyncFn(() =>
createEntrypoints({
buildId: NextBuildContext.buildId!,
config: NextBuildContext.config!,
envFiles: NextBuildContext.loadedEnvFiles!,
isDev: false,
rootDir: dir,
pageExtensions: NextBuildContext.config!.pageExtensions!,
pagesDir: NextBuildContext.pagesDir!,
appDir: NextBuildContext.appDir!,
pages: NextBuildContext.mappedPages!,
appPaths: NextBuildContext.mappedAppPages!,
previewMode: NextBuildContext.previewProps!,
rootPaths: NextBuildContext.mappedRootPaths!,
})
)

const commonWebpackOptions = {
isServer: false,
buildId: NextBuildContext.buildId!,
config: NextBuildContext.config!,
target: NextBuildContext.config!.target!,
appDir: NextBuildContext.appDir!,
pagesDir: NextBuildContext.pagesDir!,
rewrites: NextBuildContext.rewrites!,
reactProductionProfiling: NextBuildContext.reactProductionProfiling!,
noMangling: NextBuildContext.noMangling!,
}
const commonWebpackOptions = {
isServer: false,
buildId: NextBuildContext.buildId!,
config: NextBuildContext.config!,
target: NextBuildContext.config!.target!,
appDir: NextBuildContext.appDir!,
pagesDir: NextBuildContext.pagesDir!,
rewrites: NextBuildContext.rewrites!,
reactProductionProfiling: NextBuildContext.reactProductionProfiling!,
noMangling: NextBuildContext.noMangling!,
}

const configs = await runWebpackSpan
.traceChild('generate-webpack-config')
.traceAsyncFn(() =>
Promise.all([
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
middlewareMatchers: entrypoints.middlewareMatchers,
runWebpackSpan,
compilerType: COMPILER_NAMES.client,
entrypoints: entrypoints.client,
}),
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
runWebpackSpan,
middlewareMatchers: entrypoints.middlewareMatchers,
compilerType: COMPILER_NAMES.server,
entrypoints: entrypoints.server,
}),
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
runWebpackSpan,
middlewareMatchers: entrypoints.middlewareMatchers,
compilerType: COMPILER_NAMES.edgeServer,
entrypoints: entrypoints.edgeServer,
}),
])
)
const configs = await runWebpackSpan
.traceChild('generate-webpack-config')
.traceAsyncFn(() =>
Promise.all([
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
middlewareMatchers: entrypoints.middlewareMatchers,
runWebpackSpan,
compilerType: COMPILER_NAMES.client,
entrypoints: entrypoints.client,
}),
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
runWebpackSpan,
middlewareMatchers: entrypoints.middlewareMatchers,
compilerType: COMPILER_NAMES.server,
entrypoints: entrypoints.server,
}),
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
runWebpackSpan,
middlewareMatchers: entrypoints.middlewareMatchers,
compilerType: COMPILER_NAMES.edgeServer,
entrypoints: entrypoints.edgeServer,
}),
])
)

const clientConfig = configs[0]
const clientConfig = configs[0]

if (
clientConfig.optimization &&
(clientConfig.optimization.minimize !== true ||
(clientConfig.optimization.minimizer &&
clientConfig.optimization.minimizer.length === 0))
) {
Log.warn(
`Production code optimization has been disabled in your project. Read more: https://nextjs.org/docs/messages/minification-disabled`
)
}
if (
clientConfig.optimization &&
(clientConfig.optimization.minimize !== true ||
(clientConfig.optimization.minimizer &&
clientConfig.optimization.minimizer.length === 0))
) {
Log.warn(
`Production code optimization has been disabled in your project. Read more: https://nextjs.org/docs/messages/minification-disabled`
)
}

webpackBuildStart = process.hrtime()
webpackBuildStart = process.hrtime()

// We run client and server compilation separately to optimize for memory usage
await runWebpackSpan.traceAsyncFn(async () => {
// Run the server compilers first and then the client
// compiler to track the boundary of server/client components.
let clientResult: SingleCompilerResult | null = null
// We run client and server compilation separately to optimize for memory usage
await runWebpackSpan.traceAsyncFn(async () => {
// Run the server compilers first and then the client
// compiler to track the boundary of server/client components.
let clientResult: SingleCompilerResult | null = null

// During the server compilations, entries of client components will be
// injected to this set and then will be consumed by the client compiler.
injectedClientEntries.clear()
// During the server compilations, entries of client components will be
// injected to this set and then will be consumed by the client compiler.
injectedClientEntries.clear()

const serverResult = await runCompiler(configs[1], {
runWebpackSpan,
})
const edgeServerResult = configs[2]
? await runCompiler(configs[2], { runWebpackSpan })
: null

// Only continue if there were no errors
if (!serverResult.errors.length && !edgeServerResult?.errors.length) {
injectedClientEntries.forEach((value, key) => {
const clientEntry = clientConfig.entry as webpack.EntryObject
if (key === APP_CLIENT_INTERNALS) {
clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] = [
// TODO-APP: cast clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] to type EntryDescription once it's available from webpack
// @ts-expect-error clientEntry['main-app'] is type EntryDescription { import: ... }
...clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP].import,
value,
]
} else {
clientEntry[key] = {
dependOn: [CLIENT_STATIC_FILES_RUNTIME_MAIN_APP],
import: value,
}
const serverResult = await runCompiler(configs[1], {
runWebpackSpan,
})
const edgeServerResult = configs[2]
? await runCompiler(configs[2], { runWebpackSpan })
: null

// Only continue if there were no errors
if (!serverResult.errors.length && !edgeServerResult?.errors.length) {
injectedClientEntries.forEach((value, key) => {
const clientEntry = clientConfig.entry as webpack.EntryObject
if (key === APP_CLIENT_INTERNALS) {
clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] = [
// TODO-APP: cast clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] to type EntryDescription once it's available from webpack
// @ts-expect-error clientEntry['main-app'] is type EntryDescription { import: ... }
...clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP].import,
value,
]
} else {
clientEntry[key] = {
dependOn: [CLIENT_STATIC_FILES_RUNTIME_MAIN_APP],
import: value,
}
})
}
})

clientResult = await runCompiler(clientConfig, {
runWebpackSpan,
})
}
clientResult = await runCompiler(clientConfig, {
runWebpackSpan,
})
}

result = {
warnings: ([] as any[])
.concat(
clientResult?.warnings,
serverResult?.warnings,
edgeServerResult?.warnings
)
.filter(nonNullable),
errors: ([] as any[])
.concat(
clientResult?.errors,
serverResult?.errors,
edgeServerResult?.errors
)
.filter(nonNullable),
stats: [
clientResult?.stats,
serverResult?.stats,
edgeServerResult?.stats,
],
}
})
result = nextBuildSpan
.traceChild('format-webpack-messages')
.traceFn(() => formatWebpackMessages(result, true))
result = {
warnings: ([] as any[])
.concat(
clientResult?.warnings,
serverResult?.warnings,
edgeServerResult?.warnings
)
.filter(nonNullable),
errors: ([] as any[])
.concat(
clientResult?.errors,
serverResult?.errors,
edgeServerResult?.errors
)
.filter(nonNullable),
stats: [
clientResult?.stats,
serverResult?.stats,
edgeServerResult?.stats,
],
}
})
result = nextBuildSpan
.traceChild('format-webpack-messages')
.traceFn(() => formatWebpackMessages(result, true)) as CompilerResult

NextBuildContext.telemetryPlugin = (
clientConfig as webpack.Configuration
).plugins?.find(isTelemetryPlugin)
})()
NextBuildContext.telemetryPlugin = (
clientConfig as webpack.Configuration
).plugins?.find(isTelemetryPlugin)

const webpackBuildEnd = process.hrtime(webpackBuildStart)
if (buildSpinner) {
Expand Down Expand Up @@ -248,3 +247,70 @@ export async function webpackBuild(): Promise<number> {
return webpackBuildEnd[0]
}
}

// the main function when this file is run as a worker
async function workerMain() {
const { buildContext } = workerData
// setup new build context from the serialized data passed from the parent
Object.assign(NextBuildContext, buildContext)

/// load the config because it's not serializable
NextBuildContext.config = await loadConfig(
PHASE_PRODUCTION_BUILD,
NextBuildContext.dir!,
undefined,
undefined,
true
)
NextBuildContext.nextBuildSpan = trace('next-build')

try {
const result = await webpackBuildImpl()
parentPort!.postMessage(result)
} catch (e) {
parentPort!.postMessage(e)
} finally {
process.exit(0)
}
}

if (!isMainThread) {
workerMain()
}

async function webpackBuildWithWorker() {
const {
config,
telemetryPlugin,
buildSpinner,
nextBuildSpan,
...prunedBuildContext
} = NextBuildContext
const worker = new Worker(new URL(import.meta.url), {
workerData: {
buildContext: prunedBuildContext,
},
})

const result = await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
})

return result as number
}

export async function webpackBuild() {
const config = NextBuildContext.config!

if (config.experimental.webpackBuildWorker) {
return await webpackBuildWithWorker()
} else {
return await webpackBuildImpl()
}
}
3 changes: 3 additions & 0 deletions packages/next/src/server/config-schema.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,9 @@ const configSchema = {
mdxRs: {
type: 'boolean',
},
webpackBuildWorker: {
type: 'boolean',
},
turbopackLoaders: {
type: 'object',
},
Expand Down
Loading