From a7afabce167e6e184d5133746d53d63691a803b3 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Fri, 15 Sep 2023 09:54:43 -0400 Subject: [PATCH 1/3] preliminary support for bundling widgets It can be useful to include custom widgets with Grist, so they can be used offline, or for simpler archiving, or for tighter integration with the rest of the app. This PR includes a script for collecting custom widget files in a structured bundle. It will need some parallel work in Grist itself to serve the bundle. --- buildtools/bundle.js | 242 ++++++++++++++++++++++++++++++++++++++++++ buildtools/publish.js | 35 ++++-- calendar/package.json | 8 +- package.json | 7 +- 4 files changed, 281 insertions(+), 11 deletions(-) create mode 100644 buildtools/bundle.js diff --git a/buildtools/bundle.js b/buildtools/bundle.js new file mode 100644 index 00000000..229bc59b --- /dev/null +++ b/buildtools/bundle.js @@ -0,0 +1,242 @@ +#!/usr/bin/env node + +/** + * + * This is a script to bundle up some widgets and their + * assets into a standalone directory that could be used + * offline. It is a little bit string and duct-tape-y, + * sorry about that. + * + * Requires wget to operate. Will only bundle widgets + * that have an "archive" field in their package.json. + * The "archive" field can be an empty {}, or: + * { + * "domains": ["domain1.com", "domain2.com"], + * "entrypoints: ["https://other.page"] + * } + * Material in the specified domains will also be + * included in the bundle. Material needed by any + * extra entry-point pages will be included as well. + * + * The bundle will also include grist-plugin-api.js. + * This is not intended for use, but rather as a + * placeholder for where Grist could serve its own + * version of this file. + * + * The bundle includes a manifest.json file with + * relative URLs to the included widgets. + * + * The bundle also includes a manifest.yml file to + * describe the widgets as a plugin to Grist. The + * clash in file names is a little unfortunate. + * + * Call without any arguments. Run from the root of the + * repository. Places results in: + * dist/grist-widget-bundle + * + * Will run a temporary server on port 9990. + * + * Tested on Linux. The way wget is called may or may + * not need tweaking to work on Windows. + */ + +const { spawn, spawnSync } = require('child_process'); +const fs = require('fs'); +const fetch = require('node-fetch'); +const path = require('path'); + +// This is where we will place our output. +const TARGET_DIR = 'dist/grist-widget-bundle'; + +// This is a temporary port number. +const TMP_PORT = 9990; + +/** + * + * Gather all the steps needed for bundling. + * + */ +class Bundler { + constructor() { + this.localServer = null; // We will briefly serve widgets from here. + this.port = TMP_PORT; // Port for serving widgets. + this.host = `localhost:${this.port}`; // Host for serving widgets. + this.renamedHost = 'widgets'; // Final directory name for widgets. + this.assetUrl = `http://${this.host}`; // URL for serving widgets. + this.widgets = []; // Bundled widgets will be added here. + this.bundledAt = null; // A single time that applies to whole bundle. + this.bundleRoot = null; // Where bundle will be stored. + } + + // Start a widget server. + start() { + this.localServer = spawn('python', [ + '-m', 'http.server', this.port, + ], { + cwd: process.cwd(), + }); + } + + // Wait for asset server to be responsive. + // Note: the asset server provided index.html files that list + // a directory's contents. This is handy for making entrypoints + // work (e.g. providing a link to a directory full off translation + // subdirectories can be crawled correctly) but might be a little + // unexpected. + async wait() { + let ct = 0; + while (true) { + console.log("Waiting for asset server...", this.assetUrl); + try { + const resp = await fetch(this.assetUrl); + if (resp.status === 200) { + console.log("Found asset server", this.assetUrl); + break; + } + } catch (e) { + // we expect fetch failures initially. + } + await new Promise(resolve => setTimeout(resolve, 250)); + ct++; + } + } + + // Stop widget server. + stop() { + this.localServer?.kill(); + this.localServer = null; + } + + // Go ahead and bundle widgets, assuming a widget server is running. + bundle(targetDir) { + // We are going to wipe the directory we bundle into, so + // go into a subdirectory of what we were given to reduce + // odds of deleting too much unintentially. + this.bundleRoot = path.join(targetDir, 'archive'); + fs.rmSync(this.bundleRoot, { recursive: true, force: true }); + fs.mkdirSync(this.bundleRoot, { recursive: true }); + + // Prepare the manifest file using the regular process + // (we will edit it later). + this.prepareManifest(); + + // Read the manifest. + const data = fs.readFileSync(this._manifestFile(), 'utf8'); + const manifest = JSON.parse(data); + + // Run through the widgets, bundling any marked with an "archive" + // field. + this.bundledAt = new Date(); + for (const widget of manifest) { + if (!widget.archive) { continue; } + console.log(`Bundling: ${widget.url}`); + this.downloadUrl(widget.url, widget); + // Allow for other "entrypoints" in case there is material + // wget doesn't find. Theoretical, unused right now. + for (const url of (widget.archive.entrypoints || [])) { + this.downloadUrl(url, widget); + } + this.widgets.push(widget); + } + + // Rename material served from our asset server to a + // directory called "widgets" instead of "localhost:NNNN". + // In theory we should check all files for mention of + // "localhost:NNNN" but in practice assets from that server + // should only be referenced by other assets from that + // server - and wget appears to sensibly make such references + // be relative. So we can just rename the directory without + // fuss. + fs.renameSync(path.join(this.bundleRoot, this.host), + path.join(this.bundleRoot, this.renamedHost)); + this.reviseManifest(); + + fs.writeFileSync(path.join(targetDir, 'manifest.yml'), + 'name: Grist Widget Bundle\n' + + 'components:\n' + + ' widgets: archive/manifest.json\n'); + } + + // Write out a manifest file that matches the server we are running. + prepareManifest() { + const manifestFile = this._manifestFile(); + const url = `http://localhost:${this.port}`; + const cmd = `node ./buildtools/publish.js ${manifestFile} ${url}`; + const result = spawnSync(cmd, {shell: true, stdio: 'inherit'}); + if (result.status !== 0) { + throw new Error('failure'); + } + } + + // Rewrite the manifest file with just the bundled widgets, and + // with relative URLs. + reviseManifest() { + console.log(this.widgets); + fs.writeFileSync( + this._manifestFile(), + JSON.stringify(this.widgets, null, 2)); + } + + // Download the given URL and everything it depends on using + // wget. + downloadUrl(url, widget) { + const archive = widget.archive; + + // Prepare wget cmd. + let cmd = 'wget -q --recursive --page-requisites '; + cmd += '--no-parent --level=5 --convert-links '; + const domains = (archive?.domains || []) + .map(domain => this._safeDomain(domain)); + // domains.push('getgrist.com'); + domains.push('localhost'); + cmd += '--span-hosts --domains ' + domains.join(',') + ' '; + cmd += `--directory-prefix=${this.bundleRoot} ${url}`; + + // Run the wget command. + const result = spawnSync(cmd, {shell: true, stdio: 'inherit'}); + if (result.status !== 0) { + throw new Error('failure'); + } + + // Fix up the URL in the manifest to be relative to where + // the widget material will be moved to. + widget.url = widget.url.replace( + this.assetUrl, + './' + this.renamedHost + ); + + // Set a timestamp. + widget.bundledAt = this.bundledAt.toISOString(); + } + + // Quick sanity check on domains, since we'll be inserting + // them lazily in a shell command. + _safeDomain(domain) { + const approxDomainNamePattern = /^[a-zA-Z0-9.:-]+$/; + domain = String(domain); + if (approxDomainNamePattern.test(domain)) { + return domain; + } + throw new Error(`is this a domain: ${domain}`); + } + + // Get the path to the manifest file. + _manifestFile() { + return path.join(this.bundleRoot, 'manifest.json'); + } +} + + +// Run a server, do the bundling, and then shut down the server. +async function main(targetDir) { + const bundler = new Bundler(); + bundler.start(); + try { + await bundler.wait(); + bundler.bundle(targetDir); + } finally { + bundler.stop(); + } + console.log(`Results in ${targetDir}`); +} +main(TARGET_DIR).catch(e => console.error(e)); diff --git a/buildtools/publish.js b/buildtools/publish.js index 07e7e719..0b1cf9f7 100755 --- a/buildtools/publish.js +++ b/buildtools/publish.js @@ -1,6 +1,9 @@ #!/usr/bin/env node -// Creates a global manifest with all published widgets. +// Creates a global manifest with all published widgets. Call as: +// node ./buildtools/publish.js manifest.json +// or: +// node ./buildtools/publish.js manifest.json http://localhost:8585 const fs = require('fs'); const path = require('path'); @@ -8,6 +11,13 @@ const path = require('path'); const rootDir = path.join(__dirname, '..'); const folders = fs.readdirSync(rootDir); +const manifestFile = process.argv[2]; +const replacementUrl = process.argv[3] + +if (!manifestFile) { + throw new Error('please call with the file to build'); +} + function isWidgetDir(folder) { const indexHtmlFile = path.join(rootDir, folder, 'index.html'); const packageFile = path.join(rootDir, folder, 'package.json'); @@ -42,15 +52,26 @@ for (const folder of folders) { console.log('Publishing ' + config.widgetId); // If we have custom server url as a first argument for local testing, // replace widget url. - if (process.argv[2] && config.url) { - config.url = config.url.replace( - 'https://gristlabs.github.io/grist-widget', - process.argv[2] - ); + if (replacementUrl) { + if (config.url) { + config.url = replaceUrl(replacementUrl, config.url); + } + if (config.archive?.entrypoints) { + config.archive.entrypoints = config.archive.entrypoints.map( + e => replaceUrl(replacementUrl, e) + ); + } } widgets.push(config); } } } -fs.writeFileSync(path.join(rootDir, 'manifest.json'), JSON.stringify(widgets, null, 2)); +fs.writeFileSync(manifestFile, JSON.stringify(widgets, null, 2)); + +function replaceUrl(replacementUrl, configUrl) { + return configUrl.replace( + 'https://gristlabs.github.io/grist-widget', + replacementUrl + ); +} diff --git a/calendar/package.json b/calendar/package.json index fa1b0220..334812a3 100644 --- a/calendar/package.json +++ b/calendar/package.json @@ -10,7 +10,13 @@ "widgetId": "@gristlabs/widget-calendar", "published": true, "accessLevel": "full", - "renderAfterReady": true + "renderAfterReady": true, + "archive": { + "domains": ["uicdn.toast.com", "maxcdn.bootstrapcdn.com"], + "entrypoints": [ + "https://gristlabs.github.io/grist-widget/calendar/i18n/" + ] + } } ] } diff --git a/package.json b/package.json index ca922762..d9683250 100644 --- a/package.json +++ b/package.json @@ -3,15 +3,16 @@ "version": "0.0.2", "description": "A repository of grist custom widgets that have no back-end requirements.", "scripts": { - "build": "tsc --build && node ./buildtools/publish.js", + "build": "tsc --build && node ./buildtools/publish.js manifest.json", "serve": "live-server --port=8585 --no-browser -q", - "build:dev": "node ./buildtools/publish.js http://localhost:8585", + "build:dev": "node ./buildtools/publish.js manifest.json http://localhost:8585", "serve:dev": "live-server --port=8585 --no-browser -q --middleware=$(pwd)/buildtools/rewriteUrl.js", "watch": "nodemon --ignore manifest.json -e js,json --exec 'npm run build:dev'", "dev": "echo 'Starting local server and watching for changes.\nStart Grist with an environmental variable GRIST_WIDGET_LIST_URL=http://localhost:8585/manifest.json' && npm run watch 1> /dev/null & npm run serve:dev 1> /dev/null", "test": "docker image inspect gristlabs/grist --format 'gristlabs/grist image present' && NODE_PATH=_build SELENIUM_BROWSER=chrome mocha -g \"${GREP_TESTS}\" _build/test/*.js", "test:ci": "MOCHA_WEBDRIVER_HEADLESS=1 npm run test", - "pretest": "docker pull gristlabs/grist && tsc --build && node ./buildtools/publish.js http://localhost:9998" + "pretest": "docker pull gristlabs/grist && tsc --build && node ./buildtools/publish.js manifest.json http://localhost:9998", + "bundle": "node ./buildtools/bundle.js" }, "devDependencies": { "@types/chai": "^4.3.5", From 7a351bc9fce0d7a1864fd2bdce6a50dd6c6c0e64 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Fri, 3 Nov 2023 13:44:14 -0400 Subject: [PATCH 2/3] bundle getgrist.com material by default; add dist to gitignore --- .gitignore | 3 +++ buildtools/bundle.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 71b8374d..9f1fe616 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,6 @@ jspm_packages/ .env pivottable/.eslintrc.yml + +# generated bundles +dist diff --git a/buildtools/bundle.js b/buildtools/bundle.js index 229bc59b..1d4bf1cd 100644 --- a/buildtools/bundle.js +++ b/buildtools/bundle.js @@ -187,7 +187,7 @@ class Bundler { cmd += '--no-parent --level=5 --convert-links '; const domains = (archive?.domains || []) .map(domain => this._safeDomain(domain)); - // domains.push('getgrist.com'); + domains.push('getgrist.com'); domains.push('localhost'); cmd += '--span-hosts --domains ' + domains.join(',') + ' '; cmd += `--directory-prefix=${this.bundleRoot} ${url}`; From b0516fe0d94295583ec5e25b1b21837e7faf3d04 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Mon, 6 Nov 2023 10:37:19 -0500 Subject: [PATCH 3/3] be explicit about using python3 --- buildtools/bundle.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtools/bundle.js b/buildtools/bundle.js index 1d4bf1cd..39ebbe7a 100644 --- a/buildtools/bundle.js +++ b/buildtools/bundle.js @@ -70,7 +70,7 @@ class Bundler { // Start a widget server. start() { - this.localServer = spawn('python', [ + this.localServer = spawn('python3', [ '-m', 'http.server', this.port, ], { cwd: process.cwd(),