feat(gatsby): enable SSR mode in develop & serve (#32896)

* feat(gatsby): page engine * add remove-api babel plugin * extract page-data helpers used in page engine to separate module, to not bundle unneeded pieces * enable mode SSR and getServerdata on develop and serve * fix yarn.lock * fix double build * fix develop * update page engine webpack config similarly as done in graphql engine * initial * re-add some packages * remove unused file * revert some renderHTML changes * regen yarn.lock * fixup missing non webpack require type * fixup types Co-authored-by: Michal Piechowiak <[email protected]> Co-authored-by: LekoArts <[email protected]>
gatsbyjs · Sep 8, 2021 · bdf42c9 · bdf42c9
1 parent 7461c7a
commit bdf42c9
Show file tree

Hide file tree

Showing 27 changed files with 587 additions and 49 deletions.
diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
@@ -13,6 +13,7 @@
     "@babel/code-frame": "^7.14.0",
     "@babel/core": "^7.15.5",
     "@babel/eslint-parser": "^7.15.4",
+    "@babel/helper-plugin-utils": "^7.14.5",
     "@babel/parser": "^7.15.5",
     "@babel/runtime": "^7.15.4",
     "@babel/traverse": "^7.15.4",
@@ -160,6 +161,7 @@
   },
   "devDependencies": {
     "@babel/cli": "^7.15.4",
+    "@babel/helper-plugin-utils": "^7.14.5",
     "@babel/register": "^7.15.3",
     "@types/eslint": "^7.2.6",
     "@types/micromatch": "^4.0.1",

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
@@ -441,7 +441,8 @@ export async function buildHTMLPagesAndDeleteStaleArtifacts({
     reporter.info(`There are no new or changed html files to build.`)
   }
 
-  if (!program.keepPageRenderer) {
+  // TODO move to per page builds in _routes directory
+  if (!program.keepPageRenderer && _CFLAGS_.GATSBY_MAJOR !== `4`) {
     try {
       await deleteRenderer(pageRenderer)
     } catch (err) {

diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts
@@ -47,6 +47,8 @@ import {
   mergeWorkerState,
   runQueriesInWorkersQueue,
 } from "../utils/worker/pool"
+import webpackConfig from "../utils/webpack.config.js"
+import { webpack } from "webpack"
 import { createGraphqlEngineBundle } from "../schema/graphql-engine/bundle-webpack"
 import { createPageSSRBundle } from "../utils/page-ssr-module/bundle-webpack"
 import { shouldGenerateEngines } from "../utils/engines-helpers"
@@ -253,6 +255,35 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
       )
     }
     waitForCompilerCloseBuildHtml = result.waitForCompilerClose
+
+    // TODO Move to page-renderer
+    if (_CFLAGS_.GATSBY_MAJOR === `4`) {
+      const routesWebpackConfig = await webpackConfig(
+        program,
+        program.directory,
+        `build-ssr`,
+        null,
+        { parentSpan: buildSSRBundleActivityProgress.span }
+      )
+
+      await new Promise((resolve, reject) => {
+        const compiler = webpack(routesWebpackConfig)
+        compiler.run(err => {
+          if (err) {
+            return void reject(err)
+          }
+
+          compiler.close(error => {
+            if (error) {
+              return void reject(error)
+            }
+            return void resolve(undefined)
+          })
+
+          return undefined
+        })
+      })
+    }
   } catch (err) {
     buildActivityTimer.panic(structureWebpackErrors(Stage.BuildHTML, err))
   } finally {
@@ -273,6 +304,7 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
       workerPool,
       buildSpan,
     })
+
   const waitWorkerPoolEnd = Promise.all(workerPool.end())
 
   telemetry.addSiteMeasurement(`BUILD_END`, {

diff --git a/packages/gatsby/src/commands/serve.ts b/packages/gatsby/src/commands/serve.ts
@@ -18,6 +18,7 @@ import { preferDefault } from "../bootstrap/prefer-default"
 import { IProgram } from "./types"
 import { IPreparedUrls, prepareUrls } from "../utils/prepare-urls"
 import { IGatsbyFunction } from "../redux/types"
+import { reverseFixedPagePath } from "../utils/page-data"
 
 interface IMatchPath {
   path: string
@@ -121,8 +122,6 @@ module.exports = async (program: IServeProgram): Promise<void> => {
 
   router.use(compression())
   router.use(express.static(`public`, { dotfiles: `allow` }))
-  const matchPaths = await readMatchPaths(program)
-  router.use(matchPathRouter(matchPaths, { root }))
 
   const compiledFunctionsDir = path.join(
     program.directory,
@@ -226,6 +225,76 @@ module.exports = async (program: IServeProgram): Promise<void> => {
     )
   }
 
+  // Handle SSR & DSR Pages
+  if (_CFLAGS_.GATSBY_MAJOR === `4`) {
+    try {
+      const { GraphQLEngine } = require(path.join(
+        program.directory,
+        `.cache`,
+        `query-engine`
+      ))
+      const { getData, renderPageData, renderHTML } = require(path.join(
+        program.directory,
+        `.cache`,
+        `page-ssr`
+      ))
+      const graphqlEngine = new GraphQLEngine({
+        dbPath: path.join(program.directory, `.cache`, `data`, `datastore`),
+      })
+
+      app.get(
+        `/page-data/:pagePath(*)/page-data.json`,
+        async (req, res, next) => {
+          const requestedPagePath = req.params.pagePath
+          if (!requestedPagePath) {
+            return void next()
+          }
+
+          const potentialPagePath = reverseFixedPagePath(requestedPagePath)
+          const page = graphqlEngine.findPageByPath(potentialPagePath)
+
+          if (page && (page.mode === `DSR` || page.mode === `SSR`)) {
+            const data = await getData({
+              pathName: req.path,
+              graphqlEngine,
+              req,
+            })
+            const results = await renderPageData({ data })
+            return void res.send(results)
+          }
+
+          return void next()
+        }
+      )
+
+      router.use(async (req, res, next) => {
+        if (req.accepts(`html`)) {
+          const potentialPagePath = req.path
+          const page = graphqlEngine.findPageByPath(potentialPagePath)
+
+          if (page && (page.mode === `DSR` || page.mode === `SSR`)) {
+            const data = await getData({
+              pathName: potentialPagePath,
+              graphqlEngine,
+              req,
+            })
+            const results = await renderHTML({ data })
+            return res.send(results)
+          }
+
+          return res.status(404).sendFile(`404.html`, { root })
+        }
+        return next()
+      })
+    } catch (error) {
+      // TODO: Handle case of engine not being generated
+    }
+  }
+
+  const matchPaths = await readMatchPaths(program)
+  router.use(matchPathRouter(matchPaths, { root }))
+
+  // TODO: Remove/merge with above same block
   router.use((req, res, next) => {
     if (req.accepts(`html`)) {
       return res.status(404).sendFile(`404.html`, { root })

diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts
@@ -50,4 +50,6 @@ export enum Stage {
   DevelopHTML = `develop-html`,
   BuildJavascript = `build-javascript`,
   BuildHTML = `build-html`,
+  // TODO move to BuildHTML when queryengine pieces are merged
+  SSR = `build-ssr`,
 }
diff --git a/packages/gatsby/src/constants.ts b/packages/gatsby/src/constants.ts
@@ -0,0 +1 @@
+export const ROUTES_DIRECTORY = `.cache/page-ssr/routes`
diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts
@@ -157,7 +157,10 @@ export async function queryRunner(
     delete result.pageContext.componentPath
     delete result.pageContext.context
     delete result.pageContext.isCreatedByStatefulCreatePages
+
     if (_CFLAGS_.GATSBY_MAJOR === `4`) {
+      // we shouldn't add matchPath to pageContext but technically this is a breaking change so moving it ot v4
+      delete result.pageContext.matchPath
       delete result.pageContext.mode
     }
   }

diff --git a/packages/gatsby/src/query/types.ts b/packages/gatsby/src/query/types.ts
@@ -32,4 +32,5 @@ export type PageContext = Record<string, any>
 
 export interface IExecutionResult extends ExecutionResult {
   pageContext?: PageContext
+  serverData?: unknown
 }
diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js
@@ -11,7 +11,7 @@ const { slash, createContentDigest } = require(`gatsby-core-utils`)
 const { hasNodeChanged } = require(`../../utils/nodes`)
 const { getNode } = require(`../../datastore`)
 const sanitizeNode = require(`../../utils/sanitize-node`)
-const { store } = require(`..`)
+const { store } = require(`../index`)
 const { validatePageComponent } = require(`../../utils/validate-page-component`)
 import { nodeSchema } from "../../joi-schemas/joi"
 const { generateComponentChunkName } = require(`../../utils/js-chunk-names`)
@@ -25,6 +25,7 @@ const { trackCli } = require(`gatsby-telemetry`)
 const { getNonGatsbyCodeFrame } = require(`../../utils/stack-trace-utils`)
 import { createJobV2FromInternalJob } from "./internal"
 import { maybeSendJobToMainProcess } from "../../utils/jobs/worker-messaging"
+import fs from "fs-extra"
 
 const isNotTestEnv = process.env.NODE_ENV !== `test`
 const isTestEnv = process.env.NODE_ENV === `test`
@@ -281,9 +282,10 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
     page.component = pageComponentPath
   }
 
+  const rootPath = store.getState().program.directory
   const { error, message, panicOnBuild } = validatePageComponent(
     page,
-    store.getState().program.directory,
+    rootPath,
     name
   )
 
@@ -321,10 +323,7 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
         trueComponentPath = slash(trueCasePathSync(page.component))
       } catch (e) {
         // systems where user doesn't have access to /
-        const commonDir = getCommonDir(
-          store.getState().program.directory,
-          page.component
-        )
+        const commonDir = getCommonDir(rootPath, page.component)
 
         // using `path.win32` to force case insensitive relative path
         const relativePath = slash(
@@ -416,6 +415,17 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
     if (page.defer) {
       pageMode = `DSR`
     }
+
+    // TODO move to AST Check
+    const fileContent = fs.readFileSync(page.component).toString()
+    const isSSR =
+      fileContent.includes(`exports.getServerData`) ||
+      fileContent.includes(`export const getServerData`) ||
+      fileContent.includes(`export function getServerData`) ||
+      fileContent.includes(`export async function getServerData`)
+    if (isSSR) {
+      pageMode = `SSR`
+    }
     internalPage.mode = pageMode
   }
 

diff --git a/packages/gatsby/src/utils/babel-loader-helpers.js b/packages/gatsby/src/utils/babel-loader-helpers.js
@@ -51,6 +51,26 @@ const prepareOptions = (babel, options = {}, resolve = require.resolve) => {
       }
     ),
   ]
+
+  if (
+    _CFLAGS_.GATSBY_MAJOR === `4` &&
+    (stage === `develop` || stage === `build-javascript`)
+  ) {
+    requiredPlugins.push(
+      babel.createConfigItem(
+        [
+          resolve(`./babel/babel-plugin-remove-api`),
+          {
+            apis: [`getServerData`],
+          },
+        ],
+        {
+          type: `plugin`,
+        }
+      )
+    )
+  }
+
   const requiredPresets = []
 
   // Stage specific plugins to add

diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/input.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/input.mjs
@@ -0,0 +1,24 @@
+
+export default function () {
+  return "test"
+}
+
+export const getServerData = () => {
+  return {
+    props: {}
+
+  }
+}
+
+export const config = async () => {
+  return {
+    pageContext: {
+      env: 'test'
+    },
+  }
+}
+
+
+export const anotherFunction = () => {
+  return "test"
+}
diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/output.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/arrow-fn/output.mjs
@@ -0,0 +1,6 @@
+export default function () {
+  return "test";
+}
+export const anotherFunction = () => {
+  return "test";
+};
diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/input.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/input.mjs
@@ -0,0 +1,30 @@
+
+export default function () {
+  return "test"
+}
+
+function getServerData() {
+  return {
+    props: {}
+
+  }
+}
+
+
+async function pageConfig() {
+  return {
+    pageContext: {
+      env: 'test'
+    },
+  }
+}
+
+function anotherFunction() {
+  return "test"
+}
+
+export {
+  getServerData,
+  pageConfig as config,
+  anotherFunction
+}
diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/output.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/combined-export/output.mjs
@@ -0,0 +1,9 @@
+export default function () {
+  return "test";
+}
+
+function anotherFunction() {
+  return "test";
+}
+
+export { anotherFunction };
diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/input.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/input.mjs
@@ -0,0 +1,23 @@
+
+export default function () {
+  return "test"
+}
+
+export function getServerData() {
+  return {
+    props: {}
+
+  }
+}
+
+export async function config() {
+  return {
+    pageContext: {
+      env: 'test'
+    },
+  }
+}
+
+export function anotherFunction() {
+  return "test"
+}
diff --git a/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/output.mjs b/packages/gatsby/src/utils/babel/__tests__/fixtures/remove-apis/function/output.mjs
@@ -0,0 +1,6 @@
+export default function () {
+  return "test";
+}
+export function anotherFunction() {
+  return "test";
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		export const ROUTES_DIRECTORY = `.cache/page-ssr/routes`