Merge pull request #423 from a-type/persistence-refactor

Persistence layer refactoring overhaul

.changeset/light-cups-poke.md

@@ -0,0 +1,8 @@
+---
+'@verdant-web/store': major
+'@verdant-web/common': minor
+'@verdant-web/server': patch
+'@verdant-web/cli': patch
+---
+
+Major internal refactoring of persistence layer. Verdant still only supports IndexedDB for now, but this is a huge step toward configurable storage. Prereleasing this version to test in real-world environments.

.changeset/pre.json

@@ -0,0 +1,15 @@
+{
+  "mode": "pre",
+  "tag": "next",
+  "initialVersions": {
+    "@verdant-web/cli": "4.6.1",
+    "@verdant-web/common": "2.5.2",
+    "@verdant-web/create-app": "0.6.0",
+    "@verdant-web/s3-file-storage": "1.0.29",
+    "@verdant-web/react": "38.0.0",
+    "@verdant-web/react-router": "0.6.4",
+    "@verdant-web/server": "3.3.2",
+    "@verdant-web/store": "3.12.1"
+  },
+  "changesets": []
+}

configs/package.json

@@ -0,0 +1,5 @@
+{
+	"devDependencies": {
+		"vitest": "2.1.3"
+	}
+}

configs/vitest.workspace.ts

@@ -0,0 +1 @@
+export default ['../packages/*/vitest.config.ts', '../test'];

packages/cli/src/typings.ts

@@ -73,9 +73,11 @@ export class Client<Presence = any, Profile = any> {
 	__manualRebase: () => Promise<void>;
 }
 
-export interface ClientDescriptorOptions<Presence = any, Profile = any> extends Omit<BaseClientDescriptorOptions<Presence, Profile>, 'schema' | 'migrations'> {
+export interface ClientDescriptorOptions<Presence = any, Profile = any> extends Omit<BaseClientDescriptorOptions<Presence, Profile>, 'schema' | 'migrations' | 'oldSchemas'> {
   /** WARNING: overriding the schema is dangerous and almost definitely not what you want. */
   schema?: StorageSchema;
+	/** WARNING: overriding old schemas is dangerous and almost definitely not what you want. */
+	oldSchemas?: StorageSchema[];
 	/** WARNING: overriding the migrations is dangerous and almost definitely not what you want. */
 	migrations?: Migration[];
 }

packages/cli/test/__snapshots__/generated.test.ts.snap

@@ -69,10 +69,12 @@ export class Client<Presence = any, Profile = any> {
 export interface ClientDescriptorOptions<Presence = any, Profile = any>
   extends Omit<
     BaseClientDescriptorOptions<Presence, Profile>,
-    "schema" | "migrations"
+    "schema" | "migrations" | "oldSchemas"
   > {
   /** WARNING: overriding the schema is dangerous and almost definitely not what you want. */
   schema?: StorageSchema;
+  /** WARNING: overriding old schemas is dangerous and almost definitely not what you want. */
+  oldSchemas?: StorageSchema[];
   /** WARNING: overriding the migrations is dangerous and almost definitely not what you want. */
   migrations?: Migration[];
 }

packages/common/src/error.ts

@@ -9,6 +9,9 @@ export enum VerdantErrorCode {
 	Unexpected = 5000,
 	ConfigurationError = 5010,
 	NoFileStorage = 5011,
+	// Client errors
+	MigrationPathNotFound = 7001,
+	ImportFailed = 7002,
 }
 
 export class VerdantError extends Error {

packages/common/src/patch.ts

@@ -2,6 +2,7 @@
  * High-level patch creation for use with complex nested objects.
  */
 
+import { AuthorizationKey } from './authz.js';
 import { FileRef } from './files.js';
 import { createRef, createSubOid, ObjectIdentifier } from './oids.js';
 import {
@@ -32,8 +33,19 @@ export class PatchCreator {
 		return diffToPatches(from, to, this.getNow, this.createSubId, [], options);
 	};
 
-	createInitialize = (obj: any, oid: ObjectIdentifier) => {
-		return initialToPatches(obj, oid, this.getNow, this.createSubId);
+	createInitialize = (
+		obj: any,
+		oid: ObjectIdentifier,
+		authz?: AuthorizationKey,
+	) => {
+		return initialToPatches(
+			obj,
+			oid,
+			this.getNow,
+			this.createSubId,
+			undefined,
+			authz ? { authz } : undefined,
+		);
 	};
 
 	createSet = (
@@ -294,13 +306,17 @@ export class PatchCreator {
 		];
 	};
 
-	createDeleteAll = (oids: ObjectIdentifier[]): Operation[] => {
+	createDeleteAll = (
+		oids: ObjectIdentifier[],
+		authz?: AuthorizationKey,
+	): Operation[] => {
 		return oids.map((oid) => ({
 			oid,
 			timestamp: this.getNow(),
 			data: {
 				op: 'delete',
 			},
+			authz,
 		}));
 	};
 }

packages/common/src/utils.ts

@@ -185,3 +185,5 @@ export function throttle<T extends (...args: any[]) => any>(
 		}
 	} as any;
 }
+
+export function noop() {}

packages/react/package.json

@@ -32,7 +32,7 @@
 		"typecheck": "tsc --noEmit"
 	},
 	"peerDependencies": {
-		"@verdant-web/store": "^3.12.0",
+		"@verdant-web/store": "^3.12.1",
 		"react": "^18.2.0"
 	},
 	"dependencies": {

packages/server/src/Server.ts

@@ -575,6 +575,7 @@ export class Server extends EventEmitter implements MessageSender {
 					await this.storage.ready;
 					await this.storage.fileMetadata.put(info.libraryId, lofiFileInfo);
 					fs.put(stream, lofiFileInfo);
+					this.log('info', 'File uploading', lofiFileInfo);
 				} catch (e) {
 					reject(e);
 				}

packages/server/src/files/FileStorage.ts

@@ -26,16 +26,20 @@ export interface FileStorage {
 export class LocalFileStorage implements FileStorage {
 	private readonly host: string;
 	private readonly rootDirectory: string;
+	private log: (level: string, ...args: any[]) => void;
 
 	constructor({
 		rootDirectory,
 		host,
+		log,
 	}: {
 		rootDirectory: string;
 		host: string;
+		log?: (level: string, ...args: any[]) => void;
 	}) {
 		this.host = this.prepareHost(host);
 		this.rootDirectory = rootDirectory;
+		this.log = log || (() => {});
 	}
 
 	private prepareHost = (host: string) => {
@@ -53,8 +57,10 @@ export class LocalFileStorage implements FileStorage {
 		await fs.promises.mkdir(containingDirectory, {
 			recursive: true,
 		});
-		const dest = fs.createWriteStream(this.getStorageLocation(data));
+		const location = this.getStorageLocation(data);
+		const dest = fs.createWriteStream(location);
 		fileStream.pipe(dest);
+		this.log('info', 'File saving to', location);
 	}
 
 	getUrl(data: FileInfo): string {

packages/store/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @lo-fi/web
 
+## 3.12.1
+
+### Patch Changes
+
+- 66041118: Make getAll() result readonly
+
 ## 3.12.0
 
 ### Minor Changes

packages/store/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@verdant-web/store",
-	"version": "3.12.0",
+	"version": "3.12.1",
 	"access": "public",
 	"type": "module",
 	"exports": {

packages/store/src/__tests__/batching.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, vi, MockedFunction } from 'vitest';
+import { describe, it, expect, vi } from 'vitest';
 import { createTestStorage } from './fixtures/testStorage.js';
 import { Entity } from '../entities/Entity.js';
 
@@ -59,14 +59,15 @@ describe('batching operations', () => {
 	it('should overwrite superseded sequential set operations on the same key where applicable', async () => {
 		const onOperation = vi.fn();
 		const client = await createTestStorage({
-			onOperation,
+			// log: console.log,
 		});
+		client.subscribe('operation', onOperation);
 
 		const item = await client.todos.put({
 			content: 'hello world',
 			category: 'general',
 		});
-		onOperation.mockReset();
+		onOperation.mockClear();
 
 		item.set('content', 'hello world 2');
 		item.set('content', 'hello world 3');
@@ -90,9 +91,8 @@ describe('batching operations', () => {
 
 	it('should not interfere with other fields when superseding', async () => {
 		const onOperation = vi.fn();
-		const client = await createTestStorage({
-			onOperation,
-		});
+		const client = await createTestStorage();
+		client.subscribe('operation', onOperation);
 
 		const item = await client.todos.put({
 			content: 'hello world',

packages/store/src/__tests__/entities.test.ts

@@ -686,7 +686,7 @@ describe('entities', () => {
 
 	it('should allow subscribing to one field and not break on deletion', async () => {
 		const storage = await createTestStorage({
-			log: console.log,
+			// log: console.log,
 		});
 		const item = await storage.todos.put({
 			content: 'item',

packages/store/src/__tests__/fixtures/testStorage.ts

@@ -1,14 +1,8 @@
-import {
-	ClientMessage,
-	createMigration,
-	Operation,
-	schema,
-} from '@verdant-web/common';
+import { createMigration, schema } from '@verdant-web/common';
 // @ts-ignore
 import { IDBFactory } from 'fake-indexeddb';
 import { ClientWithCollections, ClientDescriptor } from '../../index.js';
 import { METADATA_VERSION_KEY } from '../../client/constants.js';
-import { Sync } from '../../sync/Sync.js';
 
 export const todoCollection = schema.collection({
 	name: 'todo',
@@ -102,23 +96,21 @@ export function createTestStorage({
 	disableRebasing = false,
 	metadataVersion,
 	log,
-	onOperation,
 }: {
 	idb?: IDBFactory;
 	disableRebasing?: boolean;
 	metadataVersion?: number;
 	log?: (...args: any[]) => void;
-	onOperation?: (op: Operation) => void;
 } = {}) {
 	const storage = new ClientDescriptor({
 		schema: testSchema,
 		migrations: [createMigration<{}>(testSchema)],
 		indexedDb: idb,
 		namespace: 'test',
 		disableRebasing,
+		oldSchemas: [testSchema],
 		log,
 		[METADATA_VERSION_KEY]: metadataVersion,
-		onOperation,
 	}).open();
 	return storage as Promise<ClientWithCollections>;
 }