From bdf42c9a9f8e91dbe594ef49a46e97f68fc872fc Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Wed, 8 Sep 2021 17:12:17 +0200 Subject: [PATCH] feat(gatsby): enable SSR mode in develop & serve (#32896) * feat(gatsby): page engine * add remove-api babel plugin * extract page-data helpers used in page engine to separate module, to not bundle unneeded pieces * enable mode SSR and getServerdata on develop and serve * fix yarn.lock * fix double build * fix develop * update page engine webpack config similarly as done in graphql engine * initial * re-add some packages * remove unused file * revert some renderHTML changes * regen yarn.lock * fixup missing non webpack require type * fixup types Co-authored-by: Michal Piechowiak Co-authored-by: LekoArts --- packages/gatsby/package.json | 2 + packages/gatsby/src/commands/build-html.ts | 3 +- packages/gatsby/src/commands/build.ts | 32 +++++++ packages/gatsby/src/commands/serve.ts | 73 ++++++++++++++- packages/gatsby/src/commands/types.ts | 2 + packages/gatsby/src/constants.ts | 1 + packages/gatsby/src/query/query-runner.ts | 3 + packages/gatsby/src/query/types.ts | 1 + packages/gatsby/src/redux/actions/public.js | 22 +++-- .../gatsby/src/utils/babel-loader-helpers.js | 20 +++++ .../fixtures/remove-apis/arrow-fn/input.mjs | 24 +++++ .../fixtures/remove-apis/arrow-fn/output.mjs | 6 ++ .../remove-apis/combined-export/input.mjs | 30 +++++++ .../remove-apis/combined-export/output.mjs | 9 ++ .../fixtures/remove-apis/function/input.mjs | 23 +++++ .../fixtures/remove-apis/function/output.mjs | 6 ++ .../fixtures/remove-apis/options.json | 13 +++ .../gatsby/src/utils/babel/__tests__/index.js | 3 + .../utils/babel/babel-plugin-remove-api.ts | 63 +++++++++++++ .../gatsby/src/utils/find-page-by-path.ts | 2 +- packages/gatsby/src/utils/get-server-data.ts | 48 ++++++++++ packages/gatsby/src/utils/page-data.ts | 50 ++++++----- .../gatsby/src/utils/page-ssr-module/entry.ts | 46 ++++++++-- packages/gatsby/src/utils/start-server.ts | 88 ++++++++++++++++++- .../gatsby/src/utils/webpack-error-utils.ts | 2 + packages/gatsby/src/utils/webpack.config.js | 55 ++++++++++-- .../gatsby/src/utils/websocket-manager.ts | 9 +- 27 files changed, 587 insertions(+), 49 deletions(-) create mode 100644 packages/gatsby/src/constants.ts create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/input.mjs create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/output.mjs create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/input.mjs create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/output.mjs create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/input.mjs create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/output.mjs create mode 100644 packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/options.json create mode 100644 packages/gatsby/src/utils/babel/__tests__/index.js create mode 100644 packages/gatsby/src/utils/babel/babel-plugin-remove-api.ts create mode 100644 packages/gatsby/src/utils/get-server-data.ts diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index d4e107fbcc9a2..ed7cc3eb47c11 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -13,6 +13,7 @@ "@babel/code-frame": "^7.14.0", "@babel/core": "^7.15.5", "@babel/eslint-parser": "^7.15.4", + "@babel/helper-plugin-utils": "^7.14.5", "@babel/parser": "^7.15.5", "@babel/runtime": "^7.15.4", "@babel/traverse": "^7.15.4", @@ -160,6 +161,7 @@ }, "devDependencies": { "@babel/cli": "^7.15.4", + "@babel/helper-plugin-utils": "^7.14.5", "@babel/register": "^7.15.3", "@types/eslint": "^7.2.6", "@types/micromatch": "^4.0.1", diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts index b9ffa6b655905..871fc6a13fc98 100644 --- a/packages/gatsby/src/commands/build-html.ts +++ b/packages/gatsby/src/commands/build-html.ts @@ -441,7 +441,8 @@ export async function buildHTMLPagesAndDeleteStaleArtifacts({ reporter.info(`There are no new or changed html files to build.`) } - if (!program.keepPageRenderer) { + // TODO move to per page builds in _routes directory + if (!program.keepPageRenderer && _CFLAGS_.GATSBY_MAJOR !== `4`) { try { await deleteRenderer(pageRenderer) } catch (err) { diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index b36c6f4603101..0168a83a3de8d 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -47,6 +47,8 @@ import { mergeWorkerState, runQueriesInWorkersQueue, } from "../utils/worker/pool" +import webpackConfig from "../utils/webpack.config.js" +import { webpack } from "webpack" import { createGraphqlEngineBundle } from "../schema/graphql-engine/bundle-webpack" import { createPageSSRBundle } from "../utils/page-ssr-module/bundle-webpack" import { shouldGenerateEngines } from "../utils/engines-helpers" @@ -253,6 +255,35 @@ module.exports = async function build(program: IBuildArgs): Promise { ) } waitForCompilerCloseBuildHtml = result.waitForCompilerClose + + // TODO Move to page-renderer + if (_CFLAGS_.GATSBY_MAJOR === `4`) { + const routesWebpackConfig = await webpackConfig( + program, + program.directory, + `build-ssr`, + null, + { parentSpan: buildSSRBundleActivityProgress.span } + ) + + await new Promise((resolve, reject) => { + const compiler = webpack(routesWebpackConfig) + compiler.run(err => { + if (err) { + return void reject(err) + } + + compiler.close(error => { + if (error) { + return void reject(error) + } + return void resolve(undefined) + }) + + return undefined + }) + }) + } } catch (err) { buildActivityTimer.panic(structureWebpackErrors(Stage.BuildHTML, err)) } finally { @@ -273,6 +304,7 @@ module.exports = async function build(program: IBuildArgs): Promise { workerPool, buildSpan, }) + const waitWorkerPoolEnd = Promise.all(workerPool.end()) telemetry.addSiteMeasurement(`BUILD_END`, { diff --git a/packages/gatsby/src/commands/serve.ts b/packages/gatsby/src/commands/serve.ts index b665761440078..e523a2b626831 100644 --- a/packages/gatsby/src/commands/serve.ts +++ b/packages/gatsby/src/commands/serve.ts @@ -18,6 +18,7 @@ import { preferDefault } from "../bootstrap/prefer-default" import { IProgram } from "./types" import { IPreparedUrls, prepareUrls } from "../utils/prepare-urls" import { IGatsbyFunction } from "../redux/types" +import { reverseFixedPagePath } from "../utils/page-data" interface IMatchPath { path: string @@ -121,8 +122,6 @@ module.exports = async (program: IServeProgram): Promise => { router.use(compression()) router.use(express.static(`public`, { dotfiles: `allow` })) - const matchPaths = await readMatchPaths(program) - router.use(matchPathRouter(matchPaths, { root })) const compiledFunctionsDir = path.join( program.directory, @@ -226,6 +225,76 @@ module.exports = async (program: IServeProgram): Promise => { ) } + // Handle SSR & DSR Pages + if (_CFLAGS_.GATSBY_MAJOR === `4`) { + try { + const { GraphQLEngine } = require(path.join( + program.directory, + `.cache`, + `query-engine` + )) + const { getData, renderPageData, renderHTML } = require(path.join( + program.directory, + `.cache`, + `page-ssr` + )) + const graphqlEngine = new GraphQLEngine({ + dbPath: path.join(program.directory, `.cache`, `data`, `datastore`), + }) + + app.get( + `/page-data/:pagePath(*)/page-data.json`, + async (req, res, next) => { + const requestedPagePath = req.params.pagePath + if (!requestedPagePath) { + return void next() + } + + const potentialPagePath = reverseFixedPagePath(requestedPagePath) + const page = graphqlEngine.findPageByPath(potentialPagePath) + + if (page && (page.mode === `DSR` || page.mode === `SSR`)) { + const data = await getData({ + pathName: req.path, + graphqlEngine, + req, + }) + const results = await renderPageData({ data }) + return void res.send(results) + } + + return void next() + } + ) + + router.use(async (req, res, next) => { + if (req.accepts(`html`)) { + const potentialPagePath = req.path + const page = graphqlEngine.findPageByPath(potentialPagePath) + + if (page && (page.mode === `DSR` || page.mode === `SSR`)) { + const data = await getData({ + pathName: potentialPagePath, + graphqlEngine, + req, + }) + const results = await renderHTML({ data }) + return res.send(results) + } + + return res.status(404).sendFile(`404.html`, { root }) + } + return next() + }) + } catch (error) { + // TODO: Handle case of engine not being generated + } + } + + const matchPaths = await readMatchPaths(program) + router.use(matchPathRouter(matchPaths, { root })) + + // TODO: Remove/merge with above same block router.use((req, res, next) => { if (req.accepts(`html`)) { return res.status(404).sendFile(`404.html`, { root }) diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts index 1cc9c692aef7c..a2de901f79085 100644 --- a/packages/gatsby/src/commands/types.ts +++ b/packages/gatsby/src/commands/types.ts @@ -50,4 +50,6 @@ export enum Stage { DevelopHTML = `develop-html`, BuildJavascript = `build-javascript`, BuildHTML = `build-html`, + // TODO move to BuildHTML when queryengine pieces are merged + SSR = `build-ssr`, } diff --git a/packages/gatsby/src/constants.ts b/packages/gatsby/src/constants.ts new file mode 100644 index 0000000000000..ce536b612c24c --- /dev/null +++ b/packages/gatsby/src/constants.ts @@ -0,0 +1 @@ +export const ROUTES_DIRECTORY = `.cache/page-ssr/routes` diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index 7c45fc1b2e164..351c6cd0cae0e 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -157,7 +157,10 @@ export async function queryRunner( delete result.pageContext.componentPath delete result.pageContext.context delete result.pageContext.isCreatedByStatefulCreatePages + if (_CFLAGS_.GATSBY_MAJOR === `4`) { + // we shouldn't add matchPath to pageContext but technically this is a breaking change so moving it ot v4 + delete result.pageContext.matchPath delete result.pageContext.mode } } diff --git a/packages/gatsby/src/query/types.ts b/packages/gatsby/src/query/types.ts index 85cd877f8a203..61e33089d4d49 100644 --- a/packages/gatsby/src/query/types.ts +++ b/packages/gatsby/src/query/types.ts @@ -32,4 +32,5 @@ export type PageContext = Record export interface IExecutionResult extends ExecutionResult { pageContext?: PageContext + serverData?: unknown } diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index 115224f4288bd..cb6751753b496 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -11,7 +11,7 @@ const { slash, createContentDigest } = require(`gatsby-core-utils`) const { hasNodeChanged } = require(`../../utils/nodes`) const { getNode } = require(`../../datastore`) const sanitizeNode = require(`../../utils/sanitize-node`) -const { store } = require(`..`) +const { store } = require(`../index`) const { validatePageComponent } = require(`../../utils/validate-page-component`) import { nodeSchema } from "../../joi-schemas/joi" const { generateComponentChunkName } = require(`../../utils/js-chunk-names`) @@ -25,6 +25,7 @@ const { trackCli } = require(`gatsby-telemetry`) const { getNonGatsbyCodeFrame } = require(`../../utils/stack-trace-utils`) import { createJobV2FromInternalJob } from "./internal" import { maybeSendJobToMainProcess } from "../../utils/jobs/worker-messaging" +import fs from "fs-extra" const isNotTestEnv = process.env.NODE_ENV !== `test` const isTestEnv = process.env.NODE_ENV === `test` @@ -281,9 +282,10 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} page.component = pageComponentPath } + const rootPath = store.getState().program.directory const { error, message, panicOnBuild } = validatePageComponent( page, - store.getState().program.directory, + rootPath, name ) @@ -321,10 +323,7 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} trueComponentPath = slash(trueCasePathSync(page.component)) } catch (e) { // systems where user doesn't have access to / - const commonDir = getCommonDir( - store.getState().program.directory, - page.component - ) + const commonDir = getCommonDir(rootPath, page.component) // using `path.win32` to force case insensitive relative path const relativePath = slash( @@ -416,6 +415,17 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} if (page.defer) { pageMode = `DSR` } + + // TODO move to AST Check + const fileContent = fs.readFileSync(page.component).toString() + const isSSR = + fileContent.includes(`exports.getServerData`) || + fileContent.includes(`export const getServerData`) || + fileContent.includes(`export function getServerData`) || + fileContent.includes(`export async function getServerData`) + if (isSSR) { + pageMode = `SSR` + } internalPage.mode = pageMode } diff --git a/packages/gatsby/src/utils/babel-loader-helpers.js b/packages/gatsby/src/utils/babel-loader-helpers.js index 28ae8197e6c4f..2ba66427ce15b 100644 --- a/packages/gatsby/src/utils/babel-loader-helpers.js +++ b/packages/gatsby/src/utils/babel-loader-helpers.js @@ -51,6 +51,26 @@ const prepareOptions = (babel, options = {}, resolve = require.resolve) => { } ), ] + + if ( + _CFLAGS_.GATSBY_MAJOR === `4` && + (stage === `develop` || stage === `build-javascript`) + ) { + requiredPlugins.push( + babel.createConfigItem( + [ + resolve(`./babel/babel-plugin-remove-api`), + { + apis: [`getServerData`], + }, + ], + { + type: `plugin`, + } + ) + ) + } + const requiredPresets = [] // Stage specific plugins to add diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/input.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/input.mjs new file mode 100644 index 0000000000000..583dcb1fb5b69 --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/input.mjs @@ -0,0 +1,24 @@ + +export default function () { + return "test" +} + +export const getServerData = () => { + return { + props: {} + + } +} + +export const config = async () => { + return { + pageContext: { + env: 'test' + }, + } +} + + +export const anotherFunction = () => { + return "test" +} diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/output.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/output.mjs new file mode 100644 index 0000000000000..9203f0dff1994 --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/output.mjs @@ -0,0 +1,6 @@ +export default function () { + return "test"; +} +export const anotherFunction = () => { + return "test"; +}; diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/input.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/input.mjs new file mode 100644 index 0000000000000..d0fd3df57156f --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/input.mjs @@ -0,0 +1,30 @@ + +export default function () { + return "test" +} + +function getServerData() { + return { + props: {} + + } +} + + +async function pageConfig() { + return { + pageContext: { + env: 'test' + }, + } +} + +function anotherFunction() { + return "test" +} + +export { + getServerData, + pageConfig as config, + anotherFunction +} diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/output.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/output.mjs new file mode 100644 index 0000000000000..bf56456933357 --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/output.mjs @@ -0,0 +1,9 @@ +export default function () { + return "test"; +} + +function anotherFunction() { + return "test"; +} + +export { anotherFunction }; diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/input.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/input.mjs new file mode 100644 index 0000000000000..64878086f6245 --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/input.mjs @@ -0,0 +1,23 @@ + +export default function () { + return "test" +} + +export function getServerData() { + return { + props: {} + + } +} + +export async function config() { + return { + pageContext: { + env: 'test' + }, + } +} + +export function anotherFunction() { + return "test" +} diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/output.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/output.mjs new file mode 100644 index 0000000000000..c2d199c55a572 --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/output.mjs @@ -0,0 +1,6 @@ +export default function () { + return "test"; +} +export function anotherFunction() { + return "test"; +} diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/options.json b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/options.json new file mode 100644 index 0000000000000..bcbf672900118 --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/options.json @@ -0,0 +1,13 @@ +{ + "plugins": [ + [ + "../../../babel-plugin-remove-api", + { + "apis": [ + "getServerData", + "config" + ] + } + ] + ] +} \ No newline at end of file diff --git a/packages/gatsby/src/utils/babel/__tests__/index.js b/packages/gatsby/src/utils/babel/__tests__/index.js new file mode 100644 index 0000000000000..f9e81c1ff9e8b --- /dev/null +++ b/packages/gatsby/src/utils/babel/__tests__/index.js @@ -0,0 +1,3 @@ +import runner from "@babel/helper-plugin-test-runner" + +runner(__dirname) diff --git a/packages/gatsby/src/utils/babel/babel-plugin-remove-api.ts b/packages/gatsby/src/utils/babel/babel-plugin-remove-api.ts new file mode 100644 index 0000000000000..d59b710aefc42 --- /dev/null +++ b/packages/gatsby/src/utils/babel/babel-plugin-remove-api.ts @@ -0,0 +1,63 @@ +import { declare } from "@babel/helper-plugin-utils" +import * as t from "@babel/types" +import type { PluginObj, ConfigAPI } from "@babel/core" + +export default declare(function removeApiCalls( + api: ConfigAPI, + options: { apis?: Array } +): PluginObj { + api.assertVersion(7) + + const apisToRemove = options?.apis ?? [] + + if (!apisToRemove.length) { + console.warn( + `No list of APIs was given to remove, check your plugin options.` + ) + } + + return { + name: `remove-api`, + visitor: { + ExportNamedDeclaration(path): void { + const declaration = path.node.declaration + + if (t.isExportNamedDeclaration(path.node)) { + const specifiersToKeep: Array< + | t.ExportDefaultSpecifier + | t.ExportNamespaceSpecifier + | t.ExportSpecifier + > = [] + path.node.specifiers.forEach(specifier => { + if ( + t.isExportSpecifier(specifier) && + t.isIdentifier(specifier.exported) && + apisToRemove.includes(specifier.exported.name) + ) { + path.scope.bindings[specifier.local.name].path.remove() + } else { + specifiersToKeep.push(specifier) + } + }) + + path.node.specifiers = specifiersToKeep + } + + let apiToCheck + if (t.isFunctionDeclaration(declaration) && declaration.id) { + apiToCheck = declaration.id.name + } + if ( + t.isVariableDeclaration(declaration) && + t.isIdentifier(declaration.declarations[0].id) + ) { + apiToCheck = declaration.declarations[0].id.name + } + + if (apiToCheck && apisToRemove.includes(apiToCheck)) { + path.remove() + } + }, + }, + } +}) diff --git a/packages/gatsby/src/utils/find-page-by-path.ts b/packages/gatsby/src/utils/find-page-by-path.ts index bead3d19a0bee..ceeac70bd4d0d 100644 --- a/packages/gatsby/src/utils/find-page-by-path.ts +++ b/packages/gatsby/src/utils/find-page-by-path.ts @@ -46,7 +46,7 @@ export function findPageByPath( } // we didn't find exact static page, time to check matchPaths - for (const [, page] of pages) { + for (const page of pages.values()) { if (page.matchPath && match(page.matchPath, path)) { return page } diff --git a/packages/gatsby/src/utils/get-server-data.ts b/packages/gatsby/src/utils/get-server-data.ts new file mode 100644 index 0000000000000..3cead4d27fbf3 --- /dev/null +++ b/packages/gatsby/src/utils/get-server-data.ts @@ -0,0 +1,48 @@ +import type { Request } from "express" +import type { IGatsbyPage } from "../redux/types" + +import { match } from "@gatsbyjs/reach-router/lib/utils" + +export interface IServerData { + headers?: Record + props?: Record +} + +interface IModuleWithServerData { + getServerData?: (args: { + headers: Map + method: string + url: string + query?: Record + params?: Record + }) => Promise +} + +export async function getServerData( + req: + | Partial> + | undefined, + page: IGatsbyPage, + pagePath: string, + mod: IModuleWithServerData | undefined +): Promise { + if (!mod?.getServerData) { + return {} + } + + const ensuredLeadingSlash = pagePath.startsWith(`/`) + ? pagePath + : `/${pagePath}` + + const { params } = match(page.matchPath || page.path, ensuredLeadingSlash) + + const getServerDataArg = { + headers: new Map(Object.entries(req?.headers ?? {})), + method: req?.method ?? `GET`, + url: req?.url ?? `"req" most likely wasn't passed in`, + query: req?.query ?? {}, + params, + } + + return mod.getServerData(getServerDataArg) +} diff --git a/packages/gatsby/src/utils/page-data.ts b/packages/gatsby/src/utils/page-data.ts index db5c8f2024955..b5b929e482d50 100644 --- a/packages/gatsby/src/utils/page-data.ts +++ b/packages/gatsby/src/utils/page-data.ts @@ -161,9 +161,9 @@ export async function flush(): Promise { staticQueriesByTemplate, queries, } = store.getState() + const isBuild = program?._?.[0] !== `develop` const { pagePaths } = pendingPageDataWrites - const writePageDataActivity = reporter.createProgress( `Writing page-data.json files to public directory`, pagePaths.size, @@ -181,11 +181,7 @@ export async function flush(): Promise { // them, a page might not exist anymore щ（ﾟДﾟщ） // This is why we need this check if (page) { - if ( - (program?._?.[0] === `develop` && - process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) || - (_CFLAGS_.GATSBY_MAJOR === `4` ? page.mode !== `SSG` : false) - ) { + if (!isBuild && process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) { // check if already did run query for this page // with query-on-demand we might have pending page-data write due to // changes in static queries assigned to page template, but we might not @@ -204,24 +200,32 @@ export async function flush(): Promise { } } - const staticQueryHashes = - staticQueriesByTemplate.get(page.componentPath) || [] - - const result = await writePageData( - path.join(program.directory, `public`), - { - ...page, - staticQueryHashes, + // In develop we rely on QUERY_ON_DEMAND so we just go through + // In build we only build these page-json for SSG pages + if ( + _CFLAGS_.GATSBY_MAJOR !== `4` || + !isBuild || + (isBuild && page.mode === `SSG`) + ) { + const staticQueryHashes = + staticQueriesByTemplate.get(page.componentPath) || [] + + const result = await writePageData( + path.join(program.directory, `public`), + { + ...page, + staticQueryHashes, + } + ) + + writePageDataActivity.tick() + + if (!isBuild) { + websocketManager.emitPageData({ + id: pagePath, + result: JSON.parse(result) as IPageDataWithQueryResult, + }) } - ) - - writePageDataActivity.tick() - - if (program?._?.[0] === `develop`) { - websocketManager.emitPageData({ - id: pagePath, - result: JSON.parse(result) as IPageDataWithQueryResult, - }) } } diff --git a/packages/gatsby/src/utils/page-ssr-module/entry.ts b/packages/gatsby/src/utils/page-ssr-module/entry.ts index 6fc7ff86ad108..ce0b1c96a5142 100644 --- a/packages/gatsby/src/utils/page-ssr-module/entry.ts +++ b/packages/gatsby/src/utils/page-ssr-module/entry.ts @@ -4,6 +4,7 @@ import type { IExecutionResult } from "../../query/types" import type { IGatsbyPage } from "../../redux/types" import type { IScriptsAndStyles } from "../client-assets-for-template" import type { IPageDataWithQueryResult } from "../page-data" +import type { Request } from "express" // actual imports import "../engines-fs-provider" @@ -15,6 +16,7 @@ import { } from "../page-data-helpers" // @ts-ignore render-page import will become valid later on (it's marked as external) import htmlComponentRenderer from "./render-page" +import { getServerData } from "../get-server-data" export interface ITemplateDetails { query: string @@ -25,6 +27,7 @@ export interface ISSRData { results: IExecutionResult page: IGatsbyPage templateDetails: ITemplateDetails + potentialPagePath: string } const pageTemplateDetailsMap: Record< @@ -33,12 +36,17 @@ const pageTemplateDetailsMap: Record< // @ts-ignore INLINED_TEMPLATE_TO_DETAILS is being "inlined" by bundler > = INLINED_TEMPLATE_TO_DETAILS +// eslint-disable-next-line @typescript-eslint/naming-convention +declare const __non_webpack_require__: typeof require + export async function getData({ pathName, graphqlEngine, + req, }: { graphqlEngine: GraphQLEngine pathName: string + req?: Partial> }): Promise { const potentialPagePath = getPagePathFromPageDataPath(pathName) || pathName @@ -58,19 +66,45 @@ export async function getData({ ) } + const executionPromises: Array> = [] + // 3. Execute query // query-runner handles case when query is not there - so maybe we should consider using that somehow let results: IExecutionResult = {} + let serverData: any = undefined if (templateDetails.query) { - results = await graphqlEngine.runQuery(templateDetails.query, { - ...page, - ...page.context, - }) + executionPromises.push( + graphqlEngine + .runQuery(templateDetails.query, { + ...page, + ...page.context, + }) + .then(queryResults => { + results = queryResults + }) + ) + } + + // 4. (if SSR) run getServerData + if (page.mode === `SSR`) { + const mod = __non_webpack_require__(`./routes/${page.componentChunkName}`) + executionPromises.push( + getServerData(req, page, potentialPagePath, mod).then( + serverDataResults => { + serverData = serverDataResults + } + ) + ) } + await Promise.all(executionPromises) + + if (serverData) { + results.serverData = serverData.props + } results.pageContext = page.context - return { results, page, templateDetails } + return { results, page, templateDetails, potentialPagePath } } export async function renderPageData({ @@ -81,7 +115,7 @@ export async function renderPageData({ const results = await constructPageDataString( { componentChunkName: data.page.componentChunkName, - path: data.page.path, + path: data.page.mode === `SSR` ? data.potentialPagePath : data.page.path, matchPath: data.page.matchPath, staticQueryHashes: data.templateDetails.staticQueryHashes, }, diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts index c59f02e920c1b..3d98563a24f58 100644 --- a/packages/gatsby/src/utils/start-server.ts +++ b/packages/gatsby/src/utils/start-server.ts @@ -52,6 +52,9 @@ import { writeVirtualLoadingIndicatorModule, } from "./loading-indicator" import { renderDevHTML } from "./dev-ssr/render-dev-html" +import pDefer from "p-defer" +import { getServerData, IServerData } from "./get-server-data" +import { ROUTES_DIRECTORY } from "../constants" type ActivityTracker = any // TODO: Replace this with proper type once reporter is typed @@ -160,12 +163,46 @@ module.exports = { const devConfig = await webpackConfig( program, directory, - `develop`, + Stage.Develop, program.port, - { parentSpan: webpackActivity.span } + { + parentSpan: webpackActivity.span, + } ) const compiler = webpack(devConfig) + let waitForServerCompilation = (): Promise => + Promise.resolve(undefined) + if (_CFLAGS_.GATSBY_MAJOR === `4`) { + const serverDevConfig = await webpackConfig( + program, + directory, + Stage.SSR, + program.port, + { + parentSpan: webpackActivity.span, + } + ) + const serverCompiler = webpack(serverDevConfig) + + const waitForServerCompilationDeferred = pDefer() + waitForServerCompilation = (): Promise => + waitForServerCompilationDeferred.promise + let compileCounter = 0 + serverCompiler.watch( + { + ignored: /node_modules/, + }, + (_, stats) => { + if (compileCounter++ > 0) { + waitForServerCompilation = (): Promise => + Promise.resolve(stats as webpack.Stats) + } else { + waitForServerCompilationDeferred.resolve(stats) + } + } + ) + } /** * Set up the express app. @@ -296,6 +333,7 @@ module.exports = { res.end() }) + const previousHashes: Map = new Map() app.get( `/page-data/:pagePath(*)/page-data.json`, async (req, res, next): Promise => { @@ -310,7 +348,46 @@ module.exports = { if (page) { try { + let serverDataPromise: Promise = Promise.resolve({}) + + if (page.mode === `SSR`) { + // get dynamic serverModule$ + const stats = await waitForServerCompilation() + + if ( + !stats || + !stats.compilation.entrypoints.has(page.componentChunkName) + ) { + report.error( + `Error loading a result for the page query in "${requestedPagePath}" / "${potentialPagePath}". getServerData threw an error.` + ) + } + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const componentHash = stats!.compilation.entrypoints + .get(page.componentChunkName)! + .getEntrypointChunk().hash as string + const modulePath = path.resolve( + `${program.directory}/${ROUTES_DIRECTORY}/${page.componentChunkName}.js` + ) + + // if webpack compilation is diff we delete old cache + if ( + previousHashes.has(page.componentChunkName) && + previousHashes.get(page.componentChunkName) === componentHash + ) { + delete require.cache[modulePath] + } + + const mod = require(modulePath) + + previousHashes.set(page.componentChunkName, componentHash) + + serverDataPromise = getServerData(req, page, potentialPagePath, mod) + } + let pageData: IPageDataWithQueryResult + // TODO move to query-engine if (process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) { const start = Date.now() @@ -327,6 +404,13 @@ module.exports = { ) } + if (page.mode === `SSR`) { + const { props } = await serverDataPromise + + pageData.result.serverData = props + pageData.path = `/${requestedPagePath}` + } + res.status(200).send(pageData) return } catch (e) { diff --git a/packages/gatsby/src/utils/webpack-error-utils.ts b/packages/gatsby/src/utils/webpack-error-utils.ts index ac16ce191921a..4349297bd3a27 100644 --- a/packages/gatsby/src/utils/webpack-error-utils.ts +++ b/packages/gatsby/src/utils/webpack-error-utils.ts @@ -6,6 +6,8 @@ import formatWebpackMessages from "react-dev-utils/formatWebpackMessages" const stageCodeToReadableLabel: Record = { [StageEnum.BuildJavascript]: `Generating JavaScript bundles`, [StageEnum.BuildHTML]: `Generating SSR bundle`, + // TODO remove when part of buildhtml + [StageEnum.SSR]: `Generating SSR bundle`, [StageEnum.DevelopHTML]: `Generating development SSR bundle`, [StageEnum.Develop]: `Generating development JavaScript bundle`, } diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js index 5ef0ca3fb97d4..c4e1610de0075 100644 --- a/packages/gatsby/src/utils/webpack.config.js +++ b/packages/gatsby/src/utils/webpack.config.js @@ -23,6 +23,7 @@ import { ForceCssHMRForEdgeCases } from "./webpack/force-css-hmr-for-edge-cases" import { hasES6ModuleSupport } from "./browserslist" import { builtinModules } from "module" import { shouldGenerateEngines } from "./engines-helpers" +import { ROUTES_DIRECTORY } from "../constants" const { BabelConfigItemsCacheInvalidatorPlugin } = require(`./babel-loader`) const FRAMEWORK_BUNDLES = [`react`, `react-dom`, `scheduler`, `prop-types`] @@ -44,12 +45,18 @@ module.exports = async ( const modulesThatUseGatsby = await getGatsbyDependents() const directoryPath = withBasePath(directory) - process.env.GATSBY_BUILD_STAGE = suppliedStage + // we will converge to build-html later on but for now this was the fastest way to get SSR to work + // TODO remove in v4 - we deprecated this in v3 + process.env.GATSBY_BUILD_STAGE = + suppliedStage === `build-ssr` ? `build-html` : suppliedStage // We combine develop & develop-html stages for purposes of generating the // webpack config. const stage = suppliedStage - const { rules, loaders, plugins } = createWebpackUtils(stage, program) + const { rules, loaders, plugins } = createWebpackUtils( + suppliedStage === `build-ssr` ? `build-html` : stage, + program + ) const { assetPrefix, pathPrefix } = store.getState().config @@ -91,7 +98,9 @@ module.exports = async ( // Don't allow overwriting of NODE_ENV, PUBLIC_DIR as to not break gatsby things envObject.NODE_ENV = JSON.stringify(nodeEnv) envObject.PUBLIC_DIR = JSON.stringify(`${process.cwd()}/public`) - envObject.BUILD_STAGE = JSON.stringify(stage) + envObject.BUILD_STAGE = JSON.stringify( + stage === `build-ssr` ? `build-html` : stage + ) envObject.CYPRESS_SUPPORT = JSON.stringify(process.env.CYPRESS_SUPPORT) envObject.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND = JSON.stringify( !!process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND @@ -167,6 +176,13 @@ module.exports = async ( path: directoryPath(`public`), publicPath: withTrailingSlash(publicPath), } + case `build-ssr`: { + return { + path: directoryPath(ROUTES_DIRECTORY), + filename: `[name].js`, + libraryTarget: `commonjs2`, + } + } default: throw new Error(`The state requested ${stage} doesn't exist.`) } @@ -189,6 +205,15 @@ module.exports = async ( ? directoryPath(`.cache/ssr-develop-static-entry`) : directoryPath(`.cache/develop-static-entry`), } + case `build-ssr`: { + const entries = Object.create(null) + for (const [, { componentPath, componentChunkName }] of store.getState() + .components) { + entries[componentChunkName] = componentPath + } + + return entries + } case `build-html`: return { main: directoryPath(`.cache/static-entry`), @@ -294,6 +319,7 @@ module.exports = async ( // it gives better line and column numbers case `develop-html`: case `build-html`: + case `build-ssr`: case `build-javascript`: return `source-map` default: @@ -388,6 +414,7 @@ module.exports = async ( } case `build-html`: case `develop-html`: + case `build-ssr`: // We don't deal with CSS at all when building the HTML. // The 'null' loader is used to prevent 'module not found' errors. // On the other hand CSS modules loaders are necessary. @@ -461,7 +488,11 @@ module.exports = async ( } const target = - stage === `build-html` || stage === `develop-html` ? `node` : `web` + stage === `build-html` || + stage === `develop-html` || + stage === `build-ssr` + ? `node` + : `web` if (target === `web`) { resolve.alias[`@reach/router`] = path.join( getPackageRoot(`@gatsbyjs/reach-router`), @@ -524,7 +555,11 @@ module.exports = async ( resolve: getResolve(stage), } - if (stage === `build-html` || stage === `develop-html`) { + if ( + stage === `build-html` || + stage === `develop-html` || + stage === `build-ssr` + ) { const [major, minor] = process.version.replace(`v`, ``).split(`.`) config.target = `node12.13` } else { @@ -690,7 +725,11 @@ module.exports = async ( } } - if (stage === `build-html` || stage === `develop-html`) { + if ( + stage === `build-html` || + stage === `develop-html` || + stage === `build-ssr` + ) { // externalize react, react-dom when develop-html or build-html(when not generating engines) const shouldMarkPackagesAsExternal = stage === `develop-html` || @@ -810,6 +849,7 @@ module.exports = async ( if ( stage === `build-javascript` || stage === `build-html` || + stage === `build-ssr` || (process.env.GATSBY_EXPERIMENTAL_DEV_WEBPACK_CACHE && (stage === `develop` || stage === `develop-html`)) ) { @@ -845,7 +885,8 @@ module.exports = async ( await apiRunnerNode(`onCreateWebpackConfig`, { getConfig, - stage, + // we will converge to build-html later on but for now this was the fastest way to get SSR to work + stage: stage === `build-ssr` ? `build-html` : stage, rules, loaders, plugins, diff --git a/packages/gatsby/src/utils/websocket-manager.ts b/packages/gatsby/src/utils/websocket-manager.ts index d7bb127d8bec5..6c3c5543022a8 100644 --- a/packages/gatsby/src/utils/websocket-manager.ts +++ b/packages/gatsby/src/utils/websocket-manager.ts @@ -85,8 +85,15 @@ export class WebsocketManager { newActivePath, fallbackTo404 ) + if (page) { - activePagePath = page.path + // when it's SSR we don't want to return the page path but the actualy url used, + // this is necessary when matchPaths are used. + if (page.mode === `SSR`) { + activePagePath = newActivePath + } else { + activePagePath = page.path + } } } clientInfo.activePath = activePagePath