Skip to content

Commit

Permalink
misc: refactor webpack build out of build/index (1/6) (#45335)
Browse files Browse the repository at this point in the history 
I'm looking at potentially extracting the webpack compilation step to another process or thread to isolate memory issues, to do this I need to split the build code a bit. This makes it relatively cleaner but there's a lot more that could be done.

I'm moving the content of the webpack span to another file and also created a shared `NextBuildContext` to hold parameters to be shared. This will be useful as we split more of the file.


## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [ ] [e2e](https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs) tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see [`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm build && pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)
  • Loading branch information
@feedthejim
feedthejim authored Feb 1, 2023
1 parent fd1ae6b commit f2d95da
Show file tree
Hide file tree
Showing 2 changed files with 273 additions and 218 deletions.
260 changes: 42 additions & 218 deletions packages/next/src/build/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import findUp from 'next/dist/compiled/find-up'
import { nanoid } from 'next/dist/compiled/nanoid/index.cjs'
import { pathToRegexp } from 'next/dist/compiled/path-to-regexp'
import path from 'path'
import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages'
import {
STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR,
PUBLIC_DIR_MIDDLEWARE_CONFLICT,
Expand Down Expand Up @@ -54,13 +53,10 @@ import {
MIDDLEWARE_MANIFEST,
APP_PATHS_MANIFEST,
APP_PATH_ROUTES_MANIFEST,
COMPILER_NAMES,
APP_BUILD_MANIFEST,
FLIGHT_SERVER_CSS_MANIFEST,
RSC_MODULE_TYPES,
FONT_LOADER_MANIFEST,
CLIENT_STATIC_FILES_RUNTIME_MAIN_APP,
APP_CLIENT_INTERNALS,
SUBRESOURCE_INTEGRITY_MANIFEST,
MIDDLEWARE_BUILD_MANIFEST,
MIDDLEWARE_REACT_LOADABLE_MANIFEST,
Expand All @@ -73,7 +69,6 @@ import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
import { getPagePath } from '../server/require'
import * as ciEnvironment from '../telemetry/ci-info'
import {
eventBuildCompleted,
eventBuildOptimize,
eventCliSession,
eventBuildFeatureUsage,
Expand All @@ -82,16 +77,16 @@ import {
EVENT_BUILD_FEATURE_USAGE,
EventBuildFeatureUsage,
eventPackageUsedInGetServerSideProps,
eventBuildCompleted,
} from '../telemetry/events'
import { Telemetry } from '../telemetry/storage'
import { runCompiler } from './compiler'
import { getPageStaticInfo } from './analysis/get-page-static-info'
import { createEntrypoints, createPagesMapping } from './entries'
import { generateBuildId } from './generate-build-id'
import { isWriteable } from './is-writeable'
import * as Log from './output/log'
import createSpinner from './spinner'
import { trace, flushAllTraces, setGlobal } from '../trace'
import { trace, flushAllTraces, setGlobal, Span } from '../trace'
import {
detectConflictingPaths,
computeFromManifest,
Expand All @@ -103,7 +98,6 @@ import {
isReservedPage,
AppConfig,
} from './utils'
import getBaseWebpackConfig from './webpack-config'
import { PagesManifest } from './webpack/plugins/pages-manifest-plugin'
import { writeBuildId } from './write-build-id'
import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
Expand All @@ -120,14 +114,14 @@ import {
teardownCrashReporter,
loadBindings,
} from './swc'
import { injectedClientEntries } from './webpack/plugins/flight-client-entry-plugin'
import { getNamedRouteRegex } from '../shared/lib/router/utils/route-regex'
import { flatReaddir } from '../lib/flat-readdir'
import { RemotePattern } from '../shared/lib/image-config'
import { eventSwcPlugins } from '../telemetry/events/swc-plugins'
import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
import { AppBuildManifest } from './webpack/plugins/app-build-manifest-plugin'
import { RSC, RSC_VARY_HEADER } from '../client/components/app-router-headers'
import { webpackBuild } from './webpack-build'

export type SsgRoute = {
initialRevalidateSeconds: number | false
Expand All @@ -150,17 +144,18 @@ export type PrerenderManifest = {
preview: __ApiPreviewProps
}

type CompilerResult = {
errors: webpack.StatsError[]
warnings: webpack.StatsError[]
stats: (webpack.Stats | undefined)[]
}

type SingleCompilerResult = {
errors: webpack.StatsError[]
warnings: webpack.StatsError[]
stats: webpack.Stats | undefined
}
export const NextBuildContext: Partial<{
telemetryPlugin: TelemetryPlugin
buildSpinner: any
nextBuildSpan: Span
entrypoints: {
client: webpack.EntryObject
server: webpack.EntryObject
edgeServer: webpack.EntryObject
middlewareMatchers: undefined
}
dir: string
}> = {}

/**
* typescript will be loaded in "next/lib/verifyTypeScriptSetup" and
Expand Down Expand Up @@ -241,10 +236,6 @@ function generateClientSsgManifest(
)
}

function isTelemetryPlugin(plugin: unknown): plugin is TelemetryPlugin {
return plugin instanceof TelemetryPlugin
}

function pageToRoute(page: string) {
const routeRegex = getNamedRouteRegex(page)
return {
Expand All @@ -268,6 +259,8 @@ export default async function build(
const nextBuildSpan = trace('next-build', undefined, {
version: process.env.__NEXT_VERSION as string,
})
NextBuildContext.nextBuildSpan = nextBuildSpan
NextBuildContext.dir = dir

const buildResult = await nextBuildSpan.traceAsyncFn(async () => {
// attempt to load global env values so they are available in next.config.js
Expand Down Expand Up @@ -308,6 +301,7 @@ export default async function build(
}

const telemetry = new Telemetry({ distDir })

setGlobal('telemetry', telemetry)

const publicDir = path.join(dir, 'public')
Expand Down Expand Up @@ -483,6 +477,8 @@ export default async function build(
prefixText: `${Log.prefixes.info} Creating an optimized production build`,
})

NextBuildContext.buildSpinner = buildSpinner

const pagesPaths =
!appDirOnly && pagesDir
? await nextBuildSpan
Expand Down Expand Up @@ -587,6 +583,7 @@ export default async function build(
pageExtensions: config.pageExtensions,
})
)
NextBuildContext.entrypoints = entrypoints

const pagesPageKeys = Object.keys(mappedPages)

Expand Down Expand Up @@ -950,199 +947,24 @@ export default async function build(
ignore: [] as string[],
}))

let result: CompilerResult = {
warnings: [],
errors: [],
stats: [],
}
let webpackBuildStart
let telemetryPlugin
await (async () => {
// IIFE to isolate locals and avoid retaining memory too long
const runWebpackSpan = nextBuildSpan.traceChild('run-webpack-compiler')

const commonWebpackOptions = {
buildId,
config,
pagesDir,
reactProductionProfiling,
rewrites,
runWebpackSpan,
target,
appDir,
noMangling,
middlewareMatchers: entrypoints.middlewareMatchers,
}

const configs = await runWebpackSpan
.traceChild('generate-webpack-config')
.traceAsyncFn(() =>
Promise.all([
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
compilerType: COMPILER_NAMES.client,
entrypoints: entrypoints.client,
}),
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
compilerType: COMPILER_NAMES.server,
entrypoints: entrypoints.server,
}),
getBaseWebpackConfig(dir, {
...commonWebpackOptions,
compilerType: COMPILER_NAMES.edgeServer,
entrypoints: entrypoints.edgeServer,
}),
])
)

const clientConfig = configs[0]

if (
clientConfig.optimization &&
(clientConfig.optimization.minimize !== true ||
(clientConfig.optimization.minimizer &&
clientConfig.optimization.minimizer.length === 0))
) {
Log.warn(
`Production code optimization has been disabled in your project. Read more: https://nextjs.org/docs/messages/minification-disabled`
)
}

webpackBuildStart = process.hrtime()

// We run client and server compilation separately to optimize for memory usage
await runWebpackSpan.traceAsyncFn(async () => {
// Run the server compilers first and then the client
// compiler to track the boundary of server/client components.
let clientResult: SingleCompilerResult | null = null

// During the server compilations, entries of client components will be
// injected to this set and then will be consumed by the client compiler.
injectedClientEntries.clear()

const serverResult = await runCompiler(configs[1], {
runWebpackSpan,
})
const edgeServerResult = configs[2]
? await runCompiler(configs[2], { runWebpackSpan })
: null

// Only continue if there were no errors
if (!serverResult.errors.length && !edgeServerResult?.errors.length) {
injectedClientEntries.forEach((value, key) => {
const clientEntry = clientConfig.entry as webpack.EntryObject
if (key === APP_CLIENT_INTERNALS) {
clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] = [
// TODO-APP: cast clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP] to type EntryDescription once it's available from webpack
// @ts-expect-error clientEntry['main-app'] is type EntryDescription { import: ... }
...clientEntry[CLIENT_STATIC_FILES_RUNTIME_MAIN_APP].import,
value,
]
} else {
clientEntry[key] = {
dependOn: [CLIENT_STATIC_FILES_RUNTIME_MAIN_APP],
import: value,
}
}
})

clientResult = await runCompiler(clientConfig, {
runWebpackSpan,
})
}
const webpackBuildDuration = await webpackBuild({
buildId,
config,
pagesDir,
reactProductionProfiling,
rewrites,
target,
appDir,
noMangling,
middlewareMatchers: entrypoints.middlewareMatchers,
})

result = {
warnings: ([] as any[])
.concat(
clientResult?.warnings,
serverResult?.warnings,
edgeServerResult?.warnings
)
.filter(nonNullable),
errors: ([] as any[])
.concat(
clientResult?.errors,
serverResult?.errors,
edgeServerResult?.errors
)
.filter(nonNullable),
stats: [
clientResult?.stats,
serverResult?.stats,
edgeServerResult?.stats,
],
}
telemetry.record(
eventBuildCompleted(pagesPaths, {
durationInSeconds: webpackBuildDuration,
totalAppPagesCount,
})
result = nextBuildSpan
.traceChild('format-webpack-messages')
.traceFn(() => formatWebpackMessages(result, true))

telemetryPlugin = (clientConfig as webpack.Configuration).plugins?.find(
isTelemetryPlugin
)
})()
const webpackBuildEnd = process.hrtime(webpackBuildStart)
if (buildSpinner) {
buildSpinner.stopAndPersist()
}

if (result.errors.length > 0) {
// Only keep the first few errors. Others are often indicative
// of the same problem, but confuse the reader with noise.
if (result.errors.length > 5) {
result.errors.length = 5
}
let error = result.errors.filter(Boolean).join('\n\n')

console.error(chalk.red('Failed to compile.\n'))

if (
error.indexOf('private-next-pages') > -1 &&
error.indexOf('does not contain a default export') > -1
) {
const page_name_regex = /'private-next-pages\/(?<page_name>[^']*)'/
const parsed = page_name_regex.exec(error)
const page_name = parsed && parsed.groups && parsed.groups.page_name
throw new Error(
`webpack build failed: found page without a React Component as default export in pages/${page_name}\n\nSee https://nextjs.org/docs/messages/page-without-valid-component for more info.`
)
}

console.error(error)
console.error()

if (
error.indexOf('private-next-pages') > -1 ||
error.indexOf('__next_polyfill__') > -1
) {
const err = new Error(
'webpack config.resolve.alias was incorrectly overridden. https://nextjs.org/docs/messages/invalid-resolve-alias'
) as NextError
err.code = 'INVALID_RESOLVE_ALIAS'
throw err
}
const err = new Error(
'Build failed because of webpack errors'
) as NextError
err.code = 'WEBPACK_ERRORS'
throw err
} else {
telemetry.record(
eventBuildCompleted(pagesPaths, {
durationInSeconds: webpackBuildEnd[0],
totalAppPagesCount,
})
)

if (result.warnings.length > 0) {
Log.warn('Compiled with warnings\n')
console.warn(result.warnings.filter(Boolean).join('\n\n'))
console.warn()
} else {
Log.info('Compiled successfully')
}
}
)

// For app directory, we run type checking after build.
if (appDir) {
Expand Down Expand Up @@ -2739,10 +2561,12 @@ export default async function build(
})
)

if (telemetryPlugin) {
const events = eventBuildFeatureUsage(telemetryPlugin)
if (NextBuildContext.telemetryPlugin) {
const events = eventBuildFeatureUsage(NextBuildContext.telemetryPlugin)
telemetry.record(events)
telemetry.record(eventPackageUsedInGetServerSideProps(telemetryPlugin))
telemetry.record(
eventPackageUsedInGetServerSideProps(NextBuildContext.telemetryPlugin)
)
}

if (ssgPages.size > 0 || appDir) {
Expand Down
Loading

0 comments on commit f2d95da

Please sign in to comment.