support for creating widget bundles

buildtools/bundle.js

@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+
+/**
+ *
+ * This is a script to bundle up some widgets and their
+ * assets into a standalone directory that could be used
+ * offline. It is a little bit string and duct-tape-y,
+ * sorry about that.
+ *
+ * Requires wget to operate. Will only bundle widgets
+ * that have an "archive" field in their package.json.
+ * The "archive" field can be an empty {}, or:
+ *    {
+ *      "domains": ["domain1.com", "domain2.com"],
+ *      "entrypoints: ["https://other.page"]
+ *    }
+ * Material in the specified domains will also be
+ * included in the bundle. Material needed by any
+ * extra entry-point pages will be included as well.
+ *
+ * The bundle will also include grist-plugin-api.js.
+ * This is not intended for use, but rather as a
+ * placeholder for where Grist could serve its own
+ * version of this file.
+ *
+ * The bundle includes a manifest.json file with
+ * relative URLs to the included widgets.
+ *
+ * The bundle also includes a manifest.yml file to
+ * describe the widgets as a plugin to Grist. The
+ * clash in file names is a little unfortunate.
+ *
+ * Call without any arguments. Run from the root of the
+ * repository. Places results in:
+ *    dist/grist-widget-bundle
+ *
+ * Will run a temporary server on port 9990.
+ *
+ * Tested on Linux. The way wget is called may or may
+ * not need tweaking to work on Windows.
+ */
+
+const { spawn, spawnSync } = require('child_process');
+const fs = require('fs');
+const fetch = require('node-fetch');
+const path = require('path');
+
+// This is where we will place our output.
+const TARGET_DIR = 'dist/grist-widget-bundle';
+
+// This is a temporary port number.
+const TMP_PORT = 9990;
+
+/**
+ *
+ * Gather all the steps needed for bundling.
+ *
+ */
+class Bundler {
+  constructor() {
+    this.localServer = null;   // We will briefly serve widgets from here.
+    this.port = TMP_PORT;      // Port for serving widgets.
+    this.host = `localhost:${this.port}`;  // Host for serving widgets.
+    this.renamedHost = 'widgets';          // Final directory name for widgets.
+    this.assetUrl = `http://${this.host}`; // URL for serving widgets.
+    this.widgets = [];         // Bundled widgets will be added here.
+    this.bundledAt = null;     // A single time that applies to whole bundle.
+    this.bundleRoot = null;    // Where bundle will be stored.
+  }
+
+  // Start a widget server.
+  start() {
+    this.localServer = spawn('python', [
+      '-m', 'http.server', this.port,
+    ], {
+      cwd: process.cwd(),
+    });
+  }
+
+  // Wait for asset server to be responsive.
+  // Note: the asset server provided index.html files that list
+  // a directory's contents. This is handy for making entrypoints
+  // work (e.g. providing a link to a directory full off translation
+  // subdirectories can be crawled correctly) but might be a little
+  // unexpected.
+  async wait() {
+    let ct = 0;
+    while (true) {
+      console.log("Waiting for asset server...", this.assetUrl);
+      try {
+        const resp = await fetch(this.assetUrl);
+        if (resp.status === 200) {
+          console.log("Found asset server", this.assetUrl);
+          break;
+        }
+      } catch (e) {
+        // we expect fetch failures initially.
+      }
+      await new Promise(resolve => setTimeout(resolve, 250));
+      ct++;
+    }
+  }
+
+  // Stop widget server.
+  stop() {
+    this.localServer?.kill();
+    this.localServer = null;
+  }
+
+  // Go ahead and bundle widgets, assuming a widget server is running.
+  bundle(targetDir) {
+    // We are going to wipe the directory we bundle into, so
+    // go into a subdirectory of what we were given to reduce
+    // odds of deleting too much unintentially.
+    this.bundleRoot = path.join(targetDir, 'archive');
+    fs.rmSync(this.bundleRoot, { recursive: true, force: true });
+    fs.mkdirSync(this.bundleRoot, { recursive: true });
+
+    // Prepare the manifest file using the regular process
+    // (we will edit it later).
+    this.prepareManifest();
+
+    // Read the manifest.
+    const data = fs.readFileSync(this._manifestFile(), 'utf8');
+    const manifest = JSON.parse(data);
+
+    // Run through the widgets, bundling any marked with an "archive"
+    // field.
+    this.bundledAt = new Date();
+    for (const widget of manifest) {
+      if (!widget.archive) { continue; }
+      console.log(`Bundling: ${widget.url}`);
+      this.downloadUrl(widget.url, widget);
+      // Allow for other "entrypoints" in case there is material
+      // wget doesn't find. Theoretical, unused right now.
+      for (const url of (widget.archive.entrypoints || [])) {
+        this.downloadUrl(url, widget);
+      }
+      this.widgets.push(widget);
+    }
+
+    // Rename material served from our asset server to a
+    // directory called "widgets" instead of "localhost:NNNN".
+    // In theory we should check all files for mention of
+    // "localhost:NNNN" but in practice assets from that server
+    // should only be referenced by other assets from that
+    // server - and wget appears to sensibly make such references
+    // be relative. So we can just rename the directory without
+    // fuss.
+    fs.renameSync(path.join(this.bundleRoot, this.host),
+                  path.join(this.bundleRoot, this.renamedHost));
+    this.reviseManifest();
+
+    fs.writeFileSync(path.join(targetDir, 'manifest.yml'),
+                     'name: Grist Widget Bundle\n' +
+                     'components:\n' +
+                     '  widgets: archive/manifest.json\n');
+  }
+
+  // Write out a manifest file that matches the server we are running.
+  prepareManifest() {
+    const manifestFile = this._manifestFile();
+    const url = `http://localhost:${this.port}`;
+    const cmd = `node ./buildtools/publish.js ${manifestFile} ${url}`;
+    const result = spawnSync(cmd, {shell: true, stdio: 'inherit'});
+    if (result.status !== 0) {
+      throw new Error('failure');
+    }
+  }
+
+  // Rewrite the manifest file with just the bundled widgets, and
+  // with relative URLs.
+  reviseManifest() {
+    console.log(this.widgets);
+    fs.writeFileSync(
+      this._manifestFile(),
+      JSON.stringify(this.widgets, null, 2));
+  }
+
+  // Download the given URL and everything it depends on using
+  // wget.
+  downloadUrl(url, widget) {
+    const archive = widget.archive;
+
+    // Prepare wget cmd.
+    let cmd = 'wget -q --recursive --page-requisites ';
+    cmd += '--no-parent --level=5 --convert-links ';
+    const domains = (archive?.domains || [])
+          .map(domain => this._safeDomain(domain));
+    // domains.push('getgrist.com');
+    domains.push('localhost');
+    cmd += '--span-hosts --domains ' + domains.join(',') + ' ';
+    cmd += `--directory-prefix=${this.bundleRoot} ${url}`;
+
+    // Run the wget command.
+    const result = spawnSync(cmd, {shell: true, stdio: 'inherit'});
+    if (result.status !== 0) {
+      throw new Error('failure');
+    }
+
+    // Fix up the URL in the manifest to be relative to where
+    // the widget material will be moved to.
+    widget.url = widget.url.replace(
+      this.assetUrl,
+      './' + this.renamedHost
+    );
+
+    // Set a timestamp.
+    widget.bundledAt = this.bundledAt.toISOString();
+  }
+
+  // Quick sanity check on domains, since we'll be inserting
+  // them lazily in a shell command.
+  _safeDomain(domain) {
+    const approxDomainNamePattern = /^[a-zA-Z0-9.:-]+$/;
+    domain = String(domain);
+    if (approxDomainNamePattern.test(domain)) {
+      return domain;
+    }
+    throw new Error(`is this a domain: ${domain}`);
+  }
+
+  // Get the path to the manifest file.
+  _manifestFile() {
+    return path.join(this.bundleRoot, 'manifest.json');
+  }
+}
+
+
+// Run a server, do the bundling, and then shut down the server.
+async function main(targetDir) {
+  const bundler = new Bundler();
+  bundler.start();
+  try {
+    await bundler.wait();
+    bundler.bundle(targetDir);
+  } finally {
+    bundler.stop();
+  }
+  console.log(`Results in ${targetDir}`);
+}
+main(TARGET_DIR).catch(e => console.error(e));

buildtools/publish.js

@@ -1,13 +1,23 @@
 #!/usr/bin/env node
 
-// Creates a global manifest with all published widgets.
+// Creates a global manifest with all published widgets. Call as:
+//   node ./buildtools/publish.js manifest.json
+// or:
+//   node ./buildtools/publish.js manifest.json http://localhost:8585
 
 const fs = require('fs');
 const path = require('path');
 
 const rootDir = path.join(__dirname, '..');
 const folders = fs.readdirSync(rootDir);
 
+const manifestFile = process.argv[2];
+const replacementUrl = process.argv[3]
+
+if (!manifestFile) {
+  throw new Error('please call with the file to build');
+}
+
 function isWidgetDir(folder) {
   const indexHtmlFile = path.join(rootDir, folder, 'index.html');
   const packageFile = path.join(rootDir, folder, 'package.json');
@@ -42,15 +52,26 @@ for (const folder of folders) {
       console.log('Publishing ' + config.widgetId);
       // If we have custom server url as a first argument for local testing,
       // replace widget url.
-      if (process.argv[2] && config.url) {
-        config.url = config.url.replace(
-          'https://gristlabs.github.io/grist-widget',
-          process.argv[2]
-        );
+      if (replacementUrl) {
+        if (config.url) {
+          config.url = replaceUrl(replacementUrl, config.url);
+        }
+        if (config.archive?.entrypoints) {
+          config.archive.entrypoints = config.archive.entrypoints.map(
+            e => replaceUrl(replacementUrl, e)
+          );
+        }
       }
       widgets.push(config);
     }
   }
 }
 
-fs.writeFileSync(path.join(rootDir, 'manifest.json'), JSON.stringify(widgets, null, 2));
+fs.writeFileSync(manifestFile, JSON.stringify(widgets, null, 2));
+
+function replaceUrl(replacementUrl, configUrl) {
+  return configUrl.replace(
+    'https://gristlabs.github.io/grist-widget',
+    replacementUrl
+  );
+}

calendar/package.json

@@ -10,7 +10,13 @@
       "widgetId": "@gristlabs/widget-calendar",
       "published": true,
       "accessLevel": "full",
-      "renderAfterReady": true
+      "renderAfterReady": true,
+      "archive": {
+        "domains": ["uicdn.toast.com", "maxcdn.bootstrapcdn.com"],
+        "entrypoints": [
+          "https://gristlabs.github.io/grist-widget/calendar/i18n/"
+        ]
+      }
     }
   ]
 }

package.json

@@ -3,15 +3,16 @@
   "version": "0.0.2",
   "description": "A repository of grist custom widgets that have no back-end requirements.",
   "scripts": {
-    "build": "tsc --build && node ./buildtools/publish.js",
+    "build": "tsc --build && node ./buildtools/publish.js manifest.json",
     "serve": "live-server --port=8585 --no-browser -q",
-    "build:dev": "node ./buildtools/publish.js http://localhost:8585",
+    "build:dev": "node ./buildtools/publish.js manifest.json http://localhost:8585",
     "serve:dev": "live-server --port=8585 --no-browser -q --middleware=$(pwd)/buildtools/rewriteUrl.js",
     "watch": "nodemon --ignore manifest.json -e js,json --exec 'npm run build:dev'",
     "dev": "echo 'Starting local server and watching for changes.\nStart Grist with an environmental variable GRIST_WIDGET_LIST_URL=http://localhost:8585/manifest.json' && npm run watch 1> /dev/null & npm run serve:dev 1> /dev/null",
     "test": "docker image inspect gristlabs/grist --format 'gristlabs/grist image present' && NODE_PATH=_build SELENIUM_BROWSER=chrome mocha -g \"${GREP_TESTS}\" _build/test/*.js",
     "test:ci": "MOCHA_WEBDRIVER_HEADLESS=1 npm run test",
-    "pretest": "docker pull gristlabs/grist && tsc --build && node ./buildtools/publish.js http://localhost:9998"
+    "pretest": "docker pull gristlabs/grist && tsc --build && node ./buildtools/publish.js manifest.json http://localhost:9998",
+    "bundle": "node ./buildtools/bundle.js"
   },
   "devDependencies": {
     "@types/chai": "^4.3.5",