Merge pull request #460 from hildjj/es-test

Es test

bin/fromMem.js

@@ -0,0 +1,188 @@
+"use strict";
+
+// Import or require module text from memory, rather than disk.  Runs
+// in a node vm, very similar to how node loads modules.
+//
+// Ideas taken from the "module-from-string" and "eval" modules, neither of
+// which were situated correctly to be used as-is.
+
+const vm = require("vm");
+const { Module } = require("module");
+const path = require("path");
+const url = require("url");
+
+// These already exist in a new, blank VM.  Date, JSON, NaN, etc.
+// Things from the core language.
+const vmGlobals = new vm
+  .Script("Object.getOwnPropertyNames(globalThis)")
+  .runInNewContext()
+  .sort();
+vmGlobals.push("global", "globalThis", "sys");
+
+// These are the things that are normally in the environment, that vm doesn't
+// make available.  This that you expect to be available in a node environment
+// that aren't in the laguage itself.
+const neededKeys = Object
+  .getOwnPropertyNames(global)
+  .filter(k => !vmGlobals.includes(k))
+  .sort();
+const globalContext = Object.fromEntries(
+  neededKeys.map(k => [k, global[k]])
+);
+
+// In node <15, console is in vmGlobals.
+globalContext.console = console;
+
+/**
+ * Options for how to process code.
+ *
+ * @typedef {object} FromMemOptions
+ * @property {"amd"|"bare"|"commonjs"|"es"|"globals"|"umd"} [format="commonjs"]
+ *   What format does the code have?  Throws an error if the format is not
+ *   "commonjs", "es", "umd", or "bare".
+ * @property {string} [filename=__filename] What is the fully-qualified synthetic
+ *   filename for the code?  Most important is the directory, which is used to
+ *   find modules that the code import's or require's.
+ * @property {object} [context={}] Variables to make availble in the global
+ *   scope while code is being evaluated.
+ * @property {boolean} [includeGlobals=true] Include the typical global
+ *   properties that node gives to all modules.  (e.g. Buffer, process).
+ * @property {string} [globalExport=null] For type "globals", what name is
+ *   exported from the module?
+ */
+
+/**
+ * Treat the given code as a node module as if require() had been called
+ * on a file containing the code.
+ *
+ * @param {string} code Source code in commonjs format.
+ * @param {string} dirname Used for __dirname.
+ * @param {FromMemOptions} options
+ * @returns {object} The module exports from code
+ */
+function requireString(code, dirname, options) {
+  const m = new Module(options.filename, module); // Current module is parent.
+  // This is the function that will be called by `require()` in the parser.
+  m.require = Module.createRequire(options.filename);
+  const script = new vm.Script(code, { filename: options.filename });
+  return script.runInNewContext({
+    module: m,
+    exports: m.exports,
+    require: m.require,
+    __dirname: dirname,
+    __filename: options.filename,
+    ...options.context,
+  });
+}
+
+/**
+ * If the given specifier starts with a ".", path.resolve it to the given
+ * directory.  Otherwise, it's a fully-qualified path, a node internal
+ * module name, an npm-provided module name, or a URL.
+ *
+ * @param {string} dirname Owning directory
+ * @param {string} specifier String from the rightmost side of an import statement
+ * @returns {string} Resolved path name or original string
+ */
+function resolveIfNeeded(dirname, specifier) {
+  if (specifier.startsWith(".")) {
+    specifier = path.resolve(dirname, specifier);
+  }
+  return specifier;
+}
+
+/**
+ * Treat the given code as a node module as if import had been called
+ * on a file containing the code.
+ *
+ * @param {string} code Source code in es6 format.
+ * @param {string} dirname Where the synthetic file would have lived.
+ * @param {FromMemOptions} options
+ * @returns {object} The module exports from code
+ */
+async function importString(code, dirname, options) {
+  if (!vm.SourceTextModule) {
+    throw new Error("Start node with --experimental-vm-modules for this to work");
+  }
+
+  const [maj, min] = process.version
+    .match(/^v(\d+)\.(\d+)\.(\d+)/)
+    .slice(1)
+    .map(x => parseInt(x, 10));
+  if ((maj < 20) || ((maj === 20) && (min < 8))) {
+    throw new Error("Requires node.js 20.8+ or 21.");
+  }
+
+  const mod = new vm.SourceTextModule(code, {
+    identifier: options.filename,
+    context: vm.createContext(options.context),
+    initializeImportMeta(meta) {
+      meta.url = String(url.pathToFileURL(options.filename));
+    },
+    importModuleDynamically(specifier) {
+      return import(resolveIfNeeded(dirname, specifier));
+    },
+  });
+
+  await mod.link(async(specifier, referencingModule) => {
+    const resolvedSpecifier = resolveIfNeeded(dirname, specifier);
+    const targetModule = await import(resolvedSpecifier);
+    const exports = Object.keys(targetModule);
+
+    // DO NOT change function to () =>, or `this` will be wrong.
+    return new vm.SyntheticModule(exports, function() {
+      for (const e of exports) {
+        this.setExport(e, targetModule[e]);
+      }
+    }, {
+      context: referencingModule.context,
+    });
+  });
+  await mod.evaluate();
+  return mod.namespace;
+}
+
+/**
+ * Import or require the given code from memory.  Knows about the different
+ * Peggy output formats.  Returns the exports of the module.
+ *
+ * @param {string} code Code to import
+ * @param {FromMemOptions} [options] Options.  Most important is filename.
+ * @returns {Promise<object>} The evaluated code.
+ */
+// eslint-disable-next-line require-await -- Always want to return a Promise
+module.exports = async function fromMem(code, options) {
+  options = {
+    format: "commonjs",
+    filename: `${__filename}-string`,
+    context: {},
+    includeGlobals: true,
+    globalExport: null,
+    ...options,
+  };
+
+  if (options.includeGlobals) {
+    options.context = {
+      ...globalContext,
+      ...options.context,
+    };
+  }
+  options.context.global = options.context;
+  options.context.globalThis = options.context;
+
+  options.filename = path.resolve(options.filename);
+  const dirname = path.dirname(options.filename);
+
+  switch (options.format) {
+    case "bare":
+    case "commonjs":
+    case "umd":
+      return requireString(code, dirname, options);
+    case "es":
+      // Returns promise
+      return importString(code, dirname, options);
+    // I don't care enough about amd and globals to figure out how to load them.
+    default:
+      throw new Error(`Unsupported output format: "${options.format}"`);
+  }
+};

bin/peggy-cli.js

@@ -3,12 +3,11 @@
 const {
   Command, CommanderError, InvalidArgumentError, Option,
 } = require("commander");
-const { Module } = require("module");
+const Module = require("module");
 const fs = require("fs");
 const path = require("path");
 const peggy = require("../lib/peg.js");
 const util = require("util");
-const vm = require("vm");
 
 exports.CommanderError = CommanderError;
 exports.InvalidArgumentError = InvalidArgumentError;
@@ -589,39 +588,12 @@ class PeggyCLI extends Command {
       const filename = this.outputJS
         ? path.resolve(this.outputJS)
         : path.join(process.cwd(), "stdout.js"); // Synthetic
-      const dirname = path.dirname(filename);
-      const m = new Module(filename, module);
-      // This is the function that will be called by `require()` in the parser.
-      m.require = Module.createRequire(filename);
-      const script = new vm.Script(source, { filename });
-      const exec = script.runInNewContext({
-        // Anything that is normally in the global scope that we think
-        // might be needed.  Limit to what is available in lowest-supported
-        // engine version.
-
-        // See: https://github.com/nodejs/node/blob/master/lib/internal/bootstrap/node.js
-        // for more things to add.
-        module: m,
-        exports: m.exports,
-        require: m.require,
-        __dirname: dirname,
-        __filename: filename,
-
-        Buffer,
-        TextDecoder: (typeof TextDecoder === "undefined") ? undefined : TextDecoder,
-        TextEncoder: (typeof TextEncoder === "undefined") ? undefined : TextEncoder,
-        URL,
-        URLSearchParams,
-        atob: Buffer.atob,
-        btoa: Buffer.btoa,
-        clearImmediate,
-        clearInterval,
-        clearTimeout,
-        console,
-        process,
-        setImmediate,
-        setInterval,
-        setTimeout,
+
+      const fromMem = require("./fromMem.js");
+      const exec = await fromMem(source, {
+        filename,
+        format: this.argv.format,
+        globalExport: this.argv.exportVar,
       });
 
       const opts = {

bin/peggy.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env -S node --experimental-vm-modules --no-warnings
 
 "use strict";
 

package.json

@@ -29,7 +29,7 @@
     "lint": "eslint . --ext js,ts,mjs",
     "ts": "tsc --build tsconfig.json",
     "docs": "cd docs && npm run build",
-    "test": "jest",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "test:web": "cd web-test && npm test",
     "test:all": "npm run test && npm run test:web",
     "benchmark": "node ./benchmark/run_bench.js",

test/cli/fixtures/imp.peggy

@@ -0,0 +1,12 @@
+{{
+import opts from "./options.mjs";
+// Cause importModuleDynamically to fire
+const opts2 = await import("./options.mjs");
+}}
+
+foo='1' { return [
+  opts.cli_test.words,
+  opts2.default.cli_test.words,
+  // Needs to use import.meta to cause initializeImportMeta to fire.
+  import.meta.url.length > 0
+]; }

test/cli/fixtures/options.mjs

@@ -0,0 +1,9 @@
+export default {
+  cli_test: {
+    words: ["zazzy"],
+  },
+  dependencies: {
+    j: "jest",
+    commander: "commander",
+  },
+};

test/cli/run.spec.ts

@@ -1173,6 +1173,35 @@ Error: Expected "1" but end of input found.
     });
   });
 
+  it("handles tests that import other modules", async() => {
+    if ((await import("vm")).SourceTextModule) {
+      const grammar = path.join(__dirname, "fixtures", "imp.peggy");
+      try {
+        await exec({
+          args: ["--format", "es", "-t", "1", grammar],
+          expected: "[ [ 'zazzy' ], [ 'zazzy' ], true ]\n",
+        });
+      } catch (e) {
+        expect((e as Error).message).toMatch("Requires node.js 20.8+ or 21");
+      }
+      await exec({
+        args: ["--format", "amd", "-t", "1", grammar],
+        error: /Unsupported output format/,
+      });
+      await exec({
+        args: ["--format", "globals", "-t", "1", grammar],
+        error: /Unsupported output format/,
+      });
+      await exec({
+        args: ["--format", "bare", "-t", "1"],
+        stdin: "foo = '1'\n",
+        expected: "'1'\n",
+      });
+    } else {
+      throw new Error("Use --experimental-vm-modules");
+    }
+  });
+
   it("handles grammar errors", async() => {
     await exec({
       stdin: "foo=unknownRule",