Merge pull request #428 from a-type/sqlite

Alternative persistence support alpha

.changeset/pre.json

@@ -0,0 +1,17 @@
+{
+  "mode": "pre",
+  "tag": "alpha",
+  "initialVersions": {
+    "@verdant-web/cli": "4.7.0",
+    "@verdant-web/common": "2.6.0",
+    "@verdant-web/create-app": "0.6.1",
+    "@verdant-web/s3-file-storage": "1.0.30",
+    "@verdant-web/persistence-capacitor-sqlite": "0.1.0-alpha.1",
+    "@verdant-web/persistence-sqlite": "0.1.0-alpha.1",
+    "@verdant-web/react": "39.0.0",
+    "@verdant-web/react-router": "0.6.5",
+    "@verdant-web/server": "3.3.3",
+    "@verdant-web/store": "4.0.0"
+  },
+  "changesets": []
+}

.changeset/six-deers-destroy.md

@@ -0,0 +1,6 @@
+---
+'@verdant-web/common': minor
+'@verdant-web/store': minor
+---
+
+Beginning of support for alternative persistence implementations. This involves major internal refactoring and some undocumented internal-use-only library API changes.

packages/common/src/files.ts

@@ -25,9 +25,13 @@ export type FileData = {
 	name: string;
 	type: string;
 	/** A local File instance, if available. */
-	file?: Blob;
+	file?: Blob | null;
 	/** The server URL of this file. */
-	url?: string;
+	url?: string | null;
+	/** Local filesystem path for the file. Only used in environments with direct fs access. */
+	localPath?: string;
+	/** The time this file was added, if available. */
+	timestamp?: string;
 };
 
 export function getAllFileFields(snapshot: any): [string, FileRef][] {

packages/common/src/indexes.test.ts

@@ -185,7 +185,6 @@ describe('all indexes', () => {
 			),
 		).toEqual({
 			foobar: 'foo' + COMPOUND_INDEX_SEPARATOR + 'foobar',
-			'@@@snapshot': JSON.stringify({ foo: 'foo' }),
 			foo: 'foo',
 			bar: 'foobar',
 		});

packages/common/src/indexes.ts

@@ -122,17 +122,14 @@ export function computeCompoundIndices(
 ): any {
 	return Object.entries(schema.compounds || {}).reduce<
 		Record<string, CompoundIndexValue>
-	>(
-		(acc, [indexKey, index]) => {
-			acc[indexKey] = createCompoundIndexValue(
-				...(index as CollectionCompoundIndex<any, any>).of.map(
-					(key) => doc[key] as string | number,
-				),
-			);
-			return acc;
-		},
-		{} as Record<string, CompoundIndexValue>,
-	);
+	>((acc, [indexKey, index]) => {
+		acc[indexKey] = createCompoundIndexValue(
+			...(index as CollectionCompoundIndex<any, any>).of.map(
+				(key) => doc[key] as string | number,
+			),
+		);
+		return acc;
+	}, {} as Record<string, CompoundIndexValue>);
 }
 
 function computeIndexedFields(schema: StorageCollectionSchema, doc: any) {
@@ -161,7 +158,6 @@ export function getIndexValues(
 		basicIndexes,
 		computeCompoundIndices(schema, { ...doc, ...basicIndexes }),
 	);
-	basicIndexes['@@@snapshot'] = JSON.stringify(doc);
 	return basicIndexes;
 }
 

packages/common/src/migration.ts

@@ -8,12 +8,12 @@ import {
 	StorageDocumentInit,
 	removeExtraProperties,
 	assert,
-	hasOid,
 	assignOid,
 	getOid,
 	hasDefault,
 	validateEntity,
 	AuthorizationKey,
+	isMultiValueIndex,
 } from './index.js';
 
 /**@deprecated */
@@ -27,9 +27,9 @@ export interface PreservedCollectionMigrationStrategy<
 	Old extends StorageCollectionSchema<any, any, any>,
 	New extends StorageCollectionSchema<any, any, any>,
 > {
-	(
-		old: StorageDocument<Old>,
-	): StorageDocument<New> | Promise<StorageDocument<New>>;
+	(old: StorageDocument<Old>):
+		| StorageDocument<New>
+		| Promise<StorageDocument<New>>;
 }
 /** @deprecated */
 type MigrationStrategy<
@@ -182,6 +182,7 @@ export interface MigrationEngine {
 	/** Deletes all documents in a collection - used for removed collections */
 	deleteCollection: (collection: string) => Promise<void>;
 	log: (...messages: any[]) => void;
+	close: () => Promise<void>;
 }
 /** @deprecated */
 type DeprecatedMigrationProcedure<
@@ -367,9 +368,27 @@ export function migrate(
 
 export interface MigrationIndexDescription {
 	name: string;
+	/**
+	 * The index writes multiple entries. Any value which matches
+	 * a lookup on this index should return the associated document.
+	 */
 	multiEntry: boolean;
+	/**
+	 * Whether this index is a synthetic index. Synthetic indexes are
+	 * computed from other fields.
+	 */
 	synthetic: boolean;
+	/**
+	 * Whether this index is a compound index. Compound indexes are
+	 * created from multiple fields in the collection merged into one
+	 * value.
+	 */
 	compound: boolean;
+	/**
+	 * The base type of the values for this index. Multientry indexes
+	 * store multiple values of this type.
+	 */
+	type: 'string' | 'number' | 'boolean';
 }
 
 export interface Migration<
@@ -406,23 +425,35 @@ function getIndexes<Coll extends StorageCollectionSchema<any, any, any>>(
 	if (!collection) return [];
 
 	return [
-		...Object.keys(collection.indexes || {}).map((key) => ({
-			name: key,
-			multiEntry: ['array', 'string[]', 'number[]', 'boolean[]'].includes(
-				collection.indexes[key].type,
-			),
-			synthetic: true,
-			compound: false,
-		})),
+		...Object.keys(collection.indexes || {}).map((key) => {
+			// lookup name-based indexes to get type from original
+			// field.
+			const index = collection.indexes[key];
+			let indexType = index.type;
+			if (!indexType && index.field) {
+				indexType = collection.fields[index.field].type;
+			}
+			assert(
+				indexType,
+				`Could not determine type of index ${collection}.${key}. Index must have a type. Perhaps your schema is malformed?`,
+			);
+
+			const multiEntry = isMultiValueIndex(collection, key);
+
+			return {
+				name: key,
+				multiEntry,
+				synthetic: true,
+				compound: false,
+				type: indexType?.replace('[]', '') as any,
+			};
+		}),
 		...Object.keys(collection.compounds || {}).map((key) => ({
 			name: key,
-			multiEntry: collection.compounds[key].of.some(
-				(fieldName: string) =>
-					(collection.fields[fieldName] || collection.indexes[fieldName])
-						.type === 'array',
-			),
+			multiEntry: isMultiValueIndex(collection, key),
 			synthetic: false,
 			compound: true,
+			type: 'string' as const,
 		})),
 	];
 }

packages/common/src/schema/indexFilters.ts

@@ -5,6 +5,7 @@ import {
 	RangeCollectionIndexFilter,
 	SortIndexFilter,
 	StartsWithIndexFilter,
+	StorageCollectionSchema,
 } from './types.js';
 
 export function isMatchIndexFilter(
@@ -47,3 +48,33 @@ export function isSortIndexFilter(
 		(filter as any).order
 	);
 }
+
+export function isMultiValueIndex(
+	collectionSchema: StorageCollectionSchema,
+	indexName: string,
+): boolean {
+	const compound = collectionSchema.compounds?.[indexName];
+	if (compound) {
+		return compound.of.some((fieldOrIndexName) => {
+			return isMultiValueIndex(collectionSchema, fieldOrIndexName);
+		});
+	}
+	const index = collectionSchema.indexes?.[indexName];
+	if (index) {
+		if ('type' in index) {
+			return isMultiEntryIndexType(index.type);
+		}
+		if ('field' in index) {
+			const field = collectionSchema.fields[index.field];
+			if (!field) return false;
+			return isMultiEntryIndexType(field.type);
+		}
+	}
+	const field = collectionSchema.fields[indexName];
+	if (!field) return false;
+	return isMultiEntryIndexType(field.type);
+}
+
+function isMultiEntryIndexType(type: string) {
+	return type === 'array' || type.endsWith('[]');
+}

packages/persistence-capacitor-sqlite/package.json

@@ -0,0 +1,44 @@
+{
+	"name": "@verdant-web/persistence-capacitor-sqlite",
+	"version": "0.1.0-alpha.1",
+	"type": "module",
+	"exports": {
+		".": {
+			"development": "./src/index.ts",
+			"import": "./dist/esm/index.js",
+			"types": "./dist/esm/index.d.ts"
+		}
+	},
+	"publishConfig": {
+		"exports": {
+			".": {
+				"import": "./dist/esm/index.js",
+				"types": "./dist/esm/index.d.ts"
+			}
+		},
+		"access": "public"
+	},
+	"files": [
+		"dist/",
+		"src/"
+	],
+	"scripts": {
+		"test": "vitest",
+		"build": "tsc",
+		"prepublishOnly": "pnpm run build"
+	},
+	"dependencies": {
+		"@capacitor-community/sqlite": "^6.0.2",
+		"@capacitor/filesystem": "^6.0.1",
+		"@verdant-web/persistence-sqlite": "workspace:*",
+		"capacitor-sqlite-kysely": "^1.0.1",
+		"kysely": "^0.27.4"
+	},
+	"peerDependencies": {
+		"@verdant-web/store": "^4.0.0"
+	},
+	"devDependencies": {
+		"@verdant-web/store": "workspace:*",
+		"vitest": "2.1.3"
+	}
+}

packages/persistence-capacitor-sqlite/src/index.ts

@@ -0,0 +1,66 @@
+import { CapacitorSQLite, SQLiteConnection } from '@capacitor-community/sqlite';
+import {
+	SqlitePersistence,
+	FilesystemImplementation,
+} from '@verdant-web/persistence-sqlite';
+import CapacitorSQLiteKyselyDialect from 'capacitor-sqlite-kysely';
+import { Kysely } from 'kysely';
+import { Directory, Filesystem } from '@capacitor/filesystem';
+import path from 'path';
+
+function getKysely(databaseFile: string) {
+	return new Kysely({
+		dialect: new CapacitorSQLiteKyselyDialect(
+			new SQLiteConnection(CapacitorSQLite),
+			{ name: databaseFile },
+		) as any,
+	});
+}
+
+class CapacitorFilesystem implements FilesystemImplementation {
+	copyDirectory = async (options: { from: string; to: string }) => {
+		await Filesystem.copy({
+			from: options.from,
+			to: options.to,
+		});
+	};
+	deleteFile = (path: string) => Filesystem.deleteFile({ path });
+	readDirectory = async (path: string) => {
+		const result = await Filesystem.readdir({ path });
+		return result.files.map((f) => f.name);
+	};
+	writeFile = async (path: string, data: Blob) => {
+		await Filesystem.writeFile({
+			path,
+			data,
+		});
+	};
+	copyFile = async (options: { from: string; to: string }) => {
+		await Filesystem.copy({
+			from: options.from,
+			to: options.to,
+		});
+	};
+	readFile = async (path: string) => {
+		const res = await Filesystem.readFile({
+			path,
+		});
+		if (typeof res.data === 'string') {
+			throw new Error(
+				"Verdant doesn't support non-Web Capacitor runtime environments.",
+			);
+		}
+		return res.data;
+	};
+}
+
+export class CapacitorSQLitePersistence extends SqlitePersistence {
+	constructor() {
+		super({
+			getKysely,
+			filesystem: new CapacitorFilesystem(),
+			databaseDirectory: path.resolve(Directory.Data, 'databases'),
+			userFilesDirectory: path.resolve(Directory.Data, 'userFiles'),
+		});
+	}
+}

packages/persistence-capacitor-sqlite/tsconfig.json

@@ -0,0 +1,11 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"rootDir": "./src",
+		"outDir": "./dist/esm",
+		"lib": ["esnext", "dom"],
+		"moduleResolution": "bundler",
+		"module": "es2020"
+	},
+	"include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.mts", "./typings"]
+}

packages/persistence-capacitor-sqlite/vitest.config.ts

@@ -0,0 +1,13 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+	test: {
+		environment: 'jsdom',
+		clearMocks: true,
+
+		setupFiles: ['src/__tests__/setup/indexedDB.ts'],
+	},
+	resolve: {
+		conditions: ['development', 'default'],
+	},
+});