Introduce "Multi Wiki Server" Plugin

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
     branches:
       - master
       - tiddlywiki-com
+      - multi-wiki-support
 env:
   NODE_VERSION: "22"
 jobs:
@@ -82,3 +83,14 @@ jobs:
       - run: "./bin/build-tw-org.sh"
         env:
           GH_TOKEN: ${{ secrets.GITHUBPUSHTOKEN }}
+  build-mws-tiddlywiki-com:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/multi-wiki-support'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: "${{ env.NODE_VERSION }}"
+      - run: "./bin/build-mws-site.sh"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUBPUSHTOKEN }}

.gitignore

@@ -5,7 +5,9 @@
 tmp/
 output/
 node_modules/
+store/
 /test-results/
 /playwright-report/
 /playwright/.cache/
 $__StoryList.tid
+/editions/test/test-store/*

TODO BEFORE MERGING THIS PR.md

@@ -0,0 +1,4 @@
+The `multi-wiki-support` branch includes some changes that are not intended to be merged into the `master` branch:
+
+* Readme update (see `editions/tw5.com/tiddlers/readme/ReadMe.tid`)
+* Remove `multiwikiserver` plugin from `readme-bld.sh` (see `bin/readme-bld.sh`)

bin/build-mws-site.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# Build mws.tiddlywiki.com assets.
+
+# Default to the version of TiddlyWiki installed in this repo
+
+if [  -z "$MWSTWCOM_BUILD_TIDDLYWIKI" ]; then
+    MWSTWCOM_BUILD_TIDDLYWIKI=./tiddlywiki.js
+fi
+
+echo "Using MWSTWCOM_BUILD_TIDDLYWIKI as [$MWSTWCOM_BUILD_TIDDLYWIKI]"
+
+# Set up the build details
+
+if [  -z "$MWSTWCOM_BUILD_DETAILS" ]; then
+    MWSTWCOM_BUILD_DETAILS="$(git symbolic-ref --short HEAD)-$(git rev-parse HEAD) from $(git remote get-url origin)"
+fi
+
+echo "Using MWSTWCOM_BUILD_DETAILS as [$MWSTWCOM_BUILD_DETAILS]"
+
+if [  -z "$MWSTWCOM_BUILD_COMMIT" ]; then
+	MWSTWCOM_BUILD_COMMIT="$(git rev-parse HEAD)"
+fi
+
+echo "Using MWSTWCOM_BUILD_COMMIT as [$MWSTWCOM_BUILD_COMMIT]"
+
+# Set up the build output directory
+
+if [  -z "$MWSTWCOM_BUILD_OUTPUT" ]; then
+    MWSTWCOM_BUILD_OUTPUT=$(mktemp -d)
+fi
+
+mkdir -p $MWSTWCOM_BUILD_OUTPUT
+
+if [  ! -d "$MWSTWCOM_BUILD_OUTPUT" ]; then
+    echo 'A valid MWSTWCOM_BUILD_OUTPUT environment variable must be set'
+    exit 1
+fi
+
+echo "Using MWSTWCOM_BUILD_OUTPUT as [$MWSTWCOM_BUILD_OUTPUT]"
+
+# Pull existing GitHub pages content
+
+git clone --depth=1 --branch=main "https://github.com/TiddlyWiki/mws.tiddlywiki.com-gh-pages.git" $MWSTWCOM_BUILD_OUTPUT
+
+# Make the CNAME file that GitHub Pages requires
+
+echo "mws.tiddlywiki.com" > $MWSTWCOM_BUILD_OUTPUT/CNAME
+
+# Delete any existing static content
+
+mkdir -p $MWSTWCOM_BUILD_OUTPUT/static
+rm $MWSTWCOM_BUILD_OUTPUT/static/*
+
+# Put the build details into a .tid file so that it can be included in each build (deleted at the end of this script)
+
+echo -e -n "title: $:/build\ncommit: $MWSTWCOM_BUILD_COMMIT\n\n$MWSTWCOM_BUILD_DETAILS\n" > $MWSTWCOM_BUILD_OUTPUT/build.tid
+
+######################################################
+#
+# mws.tiddlywiki.com distribution
+#
+######################################################
+
+# /index.html			Main site
+# /favicon.ico			Favicon for main site
+# /static.html			Static rendering of default tiddlers
+# /alltiddlers.html		Static rendering of all tiddlers
+# /static/*				Static single tiddlers
+# /static/static.css	Static stylesheet
+# /static/favicon.ico	Favicon for static pages
+node $MWSTWCOM_BUILD_TIDDLYWIKI \
+	editions/multiwikidocs \
+	--verbose \
+	--version \
+	--load $MWSTWCOM_BUILD_OUTPUT/build.tid \
+	--output $MWSTWCOM_BUILD_OUTPUT \
+	--build favicon static index \
+	|| exit 1
+
+# Delete the temporary build tiddler
+
+rm $MWSTWCOM_BUILD_OUTPUT/build.tid || exit 1
+
+# Push output back to GitHub
+
+# Exit script immediately if any command fails
+set -e
+
+pushd $MWSTWCOM_BUILD_OUTPUT
+git config --global user.email "[email protected]"
+git config --global user.name "GitHub Actions"
+git add -A .
+git commit --message "GitHub build: $GITHUB_RUN_NUMBER of $TW5_BUILD_BRANCH ($(date +'%F %T %Z'))"
+git remote add deploy "https://[email protected]/TiddlyWiki/mws.tiddlywiki.com-gh-pages.git" &>/dev/null
+git push deploy main &>/dev/null
+popd

bin/ci-test.sh

@@ -2,6 +2,8 @@
 
 # test TiddlyWiki5 for tiddlywiki.com
 
+npm install
+
 node ./tiddlywiki.js \
 	./editions/test \
 	--verbose \

bin/readme-bld.sh

@@ -10,6 +10,7 @@ fi
 
 # tw5.com readmes
 node $TW5_BUILD_TIDDLYWIKI \
+	+plugins/tiddlywiki/multiwikiserver \
 	editions/tw5.com \
 	--verbose \
 	--output . \

core/modules/commander.js

@@ -38,6 +38,13 @@ Commander.prototype.log = function(str) {
 	}
 };
 
+/*
+Clear pending commands
+*/
+Commander.prototype.clearCommands = function() {
+	this.commandTokens = this.commandTokens.slice(0,this.nextToken);
+};
+
 /*
 Write a string if verbose flag is set
 */

core/modules/commands/listen.js

@@ -16,7 +16,7 @@ var Server = require("$:/core/modules/server/server.js").Server;
 
 exports.info = {
 	name: "listen",
-	synchronous: true,
+	synchronous: false,
 	namedParameterMode: true,
 	mandatoryParameters: []
 };
@@ -38,7 +38,11 @@ Command.prototype.execute = function() {
 		wiki: this.commander.wiki,
 		variables: self.params
 	});
-	var nodeServer = this.server.listen();
+	var nodeServer = this.server.listen(null,null,null,{
+		callback: function() {
+			self.callback();
+		}
+	});
 	$tw.hooks.invokeHook("th-server-command-post-start",this.server,nodeServer,"tiddlywiki");
 	return null;
 };

core/modules/commands/quit.js

@@ -0,0 +1,37 @@
+/*\
+title: $:/core/modules/commands/quit.js
+type: application/javascript
+module-type: command
+
+Immediately ends the TiddlyWiki process
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.info = {
+	name: "quit",
+	synchronous: true
+};
+
+var Command = function(params,commander,callback) {
+	var self = this;
+	this.params = params;
+	this.commander = commander;
+	this.callback = callback;
+};
+
+Command.prototype.execute = function() {
+	// Clear any pending commands
+	this.commander.clearCommands();
+	// We don't actually quit, we just issue the "th-quit" hook to give listeners a chance to exit
+	$tw.hooks.invokeHook("th-quit");
+	return null;
+};
+
+exports.Command = Command;
+
+})();

core/modules/server/server.js

@@ -364,6 +364,11 @@ Server.prototype.listen = function(port,host,prefix) {
 	}
 	// Display the port number after we've started listening (the port number might have been specified as zero, in which case we will get an assigned port)
 	server.on("listening",function() {
+		// Stop listening when we get the "th-quit" hook
+		$tw.hooks.addHook("th-quit",function() {
+			server.close();
+		});
+		// Log listening details
 		var address = server.address(),
 			url = self.protocol + "://" + (address.family === "IPv6" ? "[" + address.address + "]" : address.address) + ":" + address.port + prefix;
 		$tw.utils.log("Serving on " + url,"brown/orange");

core/modules/syncer.js

@@ -257,7 +257,11 @@ Save an incoming tiddler in the store, and updates the associated tiddlerInfo
 Syncer.prototype.storeTiddler = function(tiddlerFields) {
 	// Save the tiddler
 	var tiddler = new $tw.Tiddler(tiddlerFields);
-	this.wiki.addTiddler(tiddler);
+	// Only save the tiddler if it has changed
+	var existingTiddler = this.wiki.getTiddler(tiddlerFields.title);
+	if(!existingTiddler || !existingTiddler.isEqual(tiddler)) {
+		this.wiki.addTiddler(tiddler);
+	}
 	// Save the tiddler revision and changeCount details
 	this.tiddlerInfo[tiddlerFields.title] = {
 		revision: this.getTiddlerRevision(tiddlerFields.title),
@@ -556,6 +560,7 @@ SaveTiddlerTask.prototype.run = function(callback) {
 			// Invoke the callback
 			callback(null);
 		},{
+			syncer: self.syncer,
 			tiddlerInfo: self.syncer.tiddlerInfo[self.title]
 		});
 	} else {
@@ -586,6 +591,7 @@ DeleteTiddlerTask.prototype.run = function(callback) {
 		// Invoke the callback
 		callback(null);
 	},{
+		syncer: self.syncer,
 		tiddlerInfo: self.syncer.tiddlerInfo[this.title]
 	});
 };
@@ -614,6 +620,8 @@ LoadTiddlerTask.prototype.run = function(callback) {
 		}
 		// Invoke the callback
 		callback(null);
+	},{
+		syncer: self.syncer
 	});
 };
 

core/modules/widgets/image.js

@@ -163,4 +163,4 @@ ImageWidget.prototype.refresh = function(changedTiddlers) {
 
 exports.image = ImageWidget;
 
-})();
+})();

editions/multiwikidocs/tiddlers/Bags and Recipes.tid

@@ -0,0 +1,41 @@
+created: 20240309135835396
+modified: 20240309142156125
+title: Bags and Recipes
+type: text/vnd.tiddlywiki
+
+The bags and recipes model is a reference architecture for how tiddlers can be shared between multiple wikis. It was first introduced by TiddlyWeb in 2008.
+
+The principles of bags and recipes can be simply stated:
+
+# Tiddlers are stored in named "bags"
+# Bags have access controls that determines which users can read or write to them
+# Recipes are named lists of bags, ordered from lowest priority to highest
+# The tiddlers within a recipe are accumulated in turn from each bag in the recipe in order of increasing priority. Thus, if there are multiple tiddlers with the same title in different bags then the one from the highest priority bag will be used as the recipe tiddler
+# Wikis are composed by splicing the tiddlers from the corresponding recipe into the standard TW5 HTML template
+
+A very simple example of the recipe/bag model might be for a single user who maintains the following bags:
+
+* ''recipes'' - tiddlers related to cooking recipes
+* ''work'' - tiddlers related to work
+* ''app'' - common tiddlers for customising TiddlyWiki
+
+Those bags would be used with the following recipes:
+
+* ''recipes'' --> recipes, app - wiki for working with recipes, with common custom components
+* ''work'' --> work, app - wiki for working with work, with common custom components
+* ''app'' --> app - wiki for maintaining custom components
+
+All of this will work dynamically, so changes to the app bag will instantly ripple into the affected hosted wikis.
+
+A more complex example might be for a teacher working with a group of students:
+
+* ''student-{name}'' bag for each students work
+* ''teacher-course'' bag for the coursework, editable by the teacher
+* ''teacher-tools'' bag for custom tools used by the teacher
+
+Those bags would be exposed through the following hosted wikis:
+
+* ''student-{name}'' hosted wiki for each students work, including the coursework material
+* ''teacher-course'' hosted wiki for the coursework, editable by the teacher
+* ''teacher'' hosted wiki for the teacher, bringing together all the bags, giving them an overview of all the students work
+

editions/multiwikidocs/tiddlers/Database Engines.tid

@@ -0,0 +1,15 @@
+title: Database Engines
+tags: Reference
+
+MWS uses [[SQLite]] for data storage. It supports choosing between two "database engines" that are based on two different npm modules:
+
+* [[better-sqlite3|https://www.npmjs.com/package/better-sqlite3]] is written partially in C/C++ and so requires compilation for the target platform
+* [[node-sqlite3-wasm|https://www.npmjs.com/package/node-sqlite3-wasm]] is written entirely in JavaScript and does not require compilation, but does require a WebAssembly-capable Node.js host. This is not currently possible on some platforms such as iOS/iPadOS
+
+By default `npm install` will install both database engines. By default it will use `better-sqlite3`. To switch to using `node-sqlite3-wasm`, set the system configuration tiddler `$:/config/MultiWikiServer/Engine` to `wasm` (the default value is `better`). Note that this tiddler resides in the [[Administration Wiki]].
+
+!! Avoiding Installation Errors
+
+If you encounter errors during `npm install` related to `gyp` or `prebuild`, you may be able to avoid them by using `node-sqlite3-wasm` instead of `better-sqlite3`. However, it will be necessary to manually install `node-sqlite3-wasm` and its dependencies. This can be done by running the following commands in your terminal:
+
+<<.copy-code-to-clipboard "npm install node-sqlite3-wasm">>

editions/multiwikidocs/tiddlers/DefaultTiddlers.tid

@@ -0,0 +1,6 @@
+title: $:/DefaultTiddlers
+
+HelloThere
+Installation
+Usage
+Reference

editions/multiwikidocs/tiddlers/HelloThere.tid

@@ -0,0 +1,19 @@
+title: HelloThere
+tags: TableOfContents
+
+!! ~TiddlyWiki is Growing Up
+
+<span class="tc-float-right">[img width=200 [MWS Banner.png]]</span>
+~MultiWikiServer is a new development that drastically improves ~TiddlyWiki's capabilities when running as a server under Node.js. It brings ~TiddlyWiki up to par with common web-based tools like ~WordPress or ~MediaWiki by supporting multiple wikis and multiple users at the same time.
+
+Features under development include:
+
+* Hosting multiple wikis at once, using the [[Bags and Recipes]] mechanism for sharing data between them
+* Based on [[SQLite|MWS and SQLite]] for performance and reliability
+* Robust built-in synchronisation handlers for syncing data to the filesystem
+* Flexible authentication and authorisation options
+* Improved handling of file uploads and attachments, allowing gigabyte video files to be uploaded and streamed
+* Instantaneous synchronisation of changes between the server and all connected clients
+* Workflow processing on the server, for example to automatically compress images, or to archive webpages
+
+MWS is currently [[under development at GitHub|https://github.com/TiddlyWiki/TiddlyWiki5/pull/7915]] but it is already functional and usable.

editions/multiwikidocs/tiddlers/Installation using Git.tid

@@ -0,0 +1,19 @@
+title: Installation using Git
+tags: Installation
+modified: 20241105133737778
+created: 20241105133737778
+
+These instructions require basic knowledge both of the terminal and of Git. There are also [[alternative instructions without using Git|Installation]].
+
+# Clone the code from GitHub with: <<.copy-code-to-clipboard "git clone -b multi-wiki-support --single-branch https://github.com/TiddlyWiki/TiddlyWiki5">>
+# Open a terminal window and set the current directory to the root of the downloaded folder
+# Install the dependencies with the command: <<.copy-code-to-clipboard "npm install">>
+# Start the server with the command: <<.copy-code-to-clipboard "npm start">>
+# To use MWS, visit http://localhost:8080 in a browser on the same computer
+# When you have finished using MWS, stop the server with <kbd>ctrl-C</kbd>
+
+See [[Troubleshooting]] if you encounter any errors.
+
+To update your copy of MWS with newer changes from ~GitHub, run the following command:
+
+<<.copy-code-to-clipboard "git pull">>