From a116ee4d1616d62ceb660f38db381b6796e7d4cd Mon Sep 17 00:00:00 2001 From: Dillon Nys Date: Tue, 15 Oct 2024 18:33:23 -0700 Subject: [PATCH] fix(runtime): Persist database in local environment When running locally, persist DBs to the filesystem so that data is present between restarts. --- .github/workflows/celest.yaml | 2 + packages/celest/CHANGELOG.md | 1 + packages/celest/build.yaml | 5 + packages/celest/dart_test.yaml | 2 + .../celest/lib/src/runtime/data/connect.dart | 3 +- .../lib/src/runtime/data/connect.io.dart | 39 +++ .../lib/src/runtime/data/connect.web.dart | 9 + packages/celest/pubspec.yaml | 3 + .../test/runtime/data/connect_test.dart | 105 ++++-- .../test/runtime/data/test_database.dart | 16 + .../test/runtime/data/test_database.g.dart | 312 ++++++++++++++++++ 11 files changed, 473 insertions(+), 24 deletions(-) create mode 100644 packages/celest/build.yaml create mode 100644 packages/celest/dart_test.yaml create mode 100644 packages/celest/test/runtime/data/test_database.dart create mode 100644 packages/celest/test/runtime/data/test_database.g.dart diff --git a/.github/workflows/celest.yaml b/.github/workflows/celest.yaml index 1cb98c64..23178f9f 100644 --- a/.github/workflows/celest.yaml +++ b/.github/workflows/celest.yaml @@ -25,6 +25,8 @@ jobs: cache: true - name: Setup Melos run: dart pub global activate melos + - name: Setup Turso + run: curl -sSfL https://get.tur.so/install.sh | bash - name: Get Packages run: melos bootstrap - name: Analyze diff --git a/packages/celest/CHANGELOG.md b/packages/celest/CHANGELOG.md index 799c9583..20f1924f 100644 --- a/packages/celest/CHANGELOG.md +++ b/packages/celest/CHANGELOG.md @@ -2,6 +2,7 @@ - chore: Remove assumptions about where the project is deployed - chore: Allow connecting to locally running libSQL servers +- fix: Persist database in local environment ## 1.0.0 diff --git a/packages/celest/build.yaml b/packages/celest/build.yaml new file mode 100644 index 00000000..33d5cf7b --- /dev/null +++ b/packages/celest/build.yaml @@ -0,0 +1,5 @@ +targets: + $default: + sources: + include: + - 'test/runtime/data/**' diff --git a/packages/celest/dart_test.yaml b/packages/celest/dart_test.yaml new file mode 100644 index 00000000..3a018ba2 --- /dev/null +++ b/packages/celest/dart_test.yaml @@ -0,0 +1,2 @@ +tags: + e2e: {} diff --git a/packages/celest/lib/src/runtime/data/connect.dart b/packages/celest/lib/src/runtime/data/connect.dart index 92936939..ee212b17 100644 --- a/packages/celest/lib/src/runtime/data/connect.dart +++ b/packages/celest/lib/src/runtime/data/connect.dart @@ -25,9 +25,10 @@ Future connect( required Database Function(QueryExecutor) factory, required env hostnameVariable, required secret tokenSecret, + String? path, }) async { if (context.environment == Environment.local) { - final executor = await inMemoryExecutor(); + final executor = await localExecutor(name: name, path: path); return _checkConnection(factory(executor)); } final host = context.get(hostnameVariable); diff --git a/packages/celest/lib/src/runtime/data/connect.io.dart b/packages/celest/lib/src/runtime/data/connect.io.dart index 0b95986e..44d2833b 100644 --- a/packages/celest/lib/src/runtime/data/connect.io.dart +++ b/packages/celest/lib/src/runtime/data/connect.io.dart @@ -1,5 +1,44 @@ +import 'dart:io'; +import 'dart:isolate'; + import 'package:drift/drift.dart'; import 'package:drift/native.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as p; + +final Logger _logger = Logger('Celest.Data'); /// A [QueryExecutor] for an in-memory database. Future inMemoryExecutor() async => NativeDatabase.memory(); + +/// A [QueryExecutor] with local persistence. +Future localExecutor({ + required String name, + String? path, +}) async { + if (path == null) { + final packageConfig = await Isolate.packageConfig; + if (packageConfig == null) { + _logger.warning( + 'Failed to determine package config path. ' + 'Falling back to in-memory database.', + ); + return inMemoryExecutor(); + } + path = p.join( + p.dirname(p.fromUri(packageConfig)), + 'celest', + '$name.db', + ); + } + _logger.info('Opening database at $path'); + final databaseFile = File(path); + if (!databaseFile.existsSync()) { + _logger.info('Database does not exist. Creating new database.'); + } + return NativeDatabase( + databaseFile, + cachePreparedStatements: true, + enableMigrations: true, + ); +} diff --git a/packages/celest/lib/src/runtime/data/connect.web.dart b/packages/celest/lib/src/runtime/data/connect.web.dart index a2987134..10adb11c 100644 --- a/packages/celest/lib/src/runtime/data/connect.web.dart +++ b/packages/celest/lib/src/runtime/data/connect.web.dart @@ -8,3 +8,12 @@ Future inMemoryExecutor() async { sqlite3.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); return WasmDatabase.inMemory(sqlite3); } + +/// A [QueryExecutor] with local persistence. +Future localExecutor({ + required String name, + String? path, +}) async { + // TODO(dnys1): We don't have a use case for this yet. + return inMemoryExecutor(); +} diff --git a/packages/celest/pubspec.yaml b/packages/celest/pubspec.yaml index 4b09d379..197dbbcd 100644 --- a/packages/celest/pubspec.yaml +++ b/packages/celest/pubspec.yaml @@ -29,6 +29,7 @@ dependencies: http_sfv: ^0.1.0 logging: ^1.2.0 meta: ^1.11.0 + path: ^1.9.0 platform: ^3.1.5 shelf: ^1.4.1 shelf_router: ^1.1.4 @@ -40,8 +41,10 @@ dependencies: x509: ^0.2.4 dev_dependencies: + build_runner: ^2.4.13 celest_lints: path: ../celest_lints + drift_dev: '>=2.21.0 <2.22.0' pub_semver: ^2.1.4 stream_transform: ^2.1.0 test: ^1.25.1 diff --git a/packages/celest/test/runtime/data/connect_test.dart b/packages/celest/test/runtime/data/connect_test.dart index bef9e978..35fddf3e 100644 --- a/packages/celest/test/runtime/data/connect_test.dart +++ b/packages/celest/test/runtime/data/connect_test.dart @@ -2,17 +2,22 @@ @Tags(['e2e']) library; +import 'dart:async'; +import 'dart:convert'; import 'dart:io'; +import 'dart:isolate'; import 'package:celest/src/config/config_values.dart'; import 'package:celest/src/core/context.dart'; import 'package:celest/src/runtime/data/connect.dart'; -import 'package:drift/drift.dart'; +import 'package:drift/native.dart'; import 'package:drift_hrana/drift_hrana.dart'; +import 'package:path/path.dart' as p; import 'package:platform/platform.dart'; -import 'package:test/fake.dart'; import 'package:test/test.dart'; +import 'test_database.dart'; + Future _findOpenPort() async { final server = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); final port = server.port; @@ -20,17 +25,32 @@ Future _findOpenPort() async { return port; } -Future _startLibSqlServer() async { +Future _startSqld() async { final port = await _findOpenPort(); final process = await Process.start('turso', ['dev', '--port', '$port']); addTearDown(process.kill); + final running = Completer(); + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((event) { + if (event.contains('sqld listening on')) { + running.complete(); + } + }); + await running.future.timeout( + const Duration(seconds: 5), + onTimeout: () { + throw StateError('Failed to start sqld'); + }, + ); return Uri.parse('ws://localhost:$port'); } void main() { group('connect', () { - test('allows local libSQL URIs', () async { - final host = await _startLibSqlServer(); + test('allows local sqld URIs', () async { + final host = await _startSqld(); final platform = FakePlatform( environment: { 'CELEST_DATABASE_HOST': host.toString(), @@ -38,34 +58,73 @@ void main() { ); Context.current.put(env.environment, 'production'); Context.current.put(ContextKey.platform, platform); - await connect( + final database = await connect( Context.current, name: 'test', factory: expectAsync1((executor) { expect(executor, isA()); - return _FakeDatabase(); + return TestDatabase(executor); }), hostnameVariable: const env('CELEST_DATABASE_HOST'), tokenSecret: const secret('CELEST_DATABASE_TOKEN'), ); + await database.close(); }); }); -} -final class _FakeDatabase extends Fake implements GeneratedDatabase { - @override - Selectable customSelect( - String query, { - List> variables = const [], - Set> readsFrom = const {}, - }) { - return _FakeSelectable(); - } -} + group('localExecutor', () { + test('uses package config when path=null', () async { + final packageConfig = await Isolate.packageConfig; + final file = File.fromUri(packageConfig!.resolve('./celest/test.db')); + if (file.existsSync()) { + file.deleteSync(); + } + addTearDown(() { + if (file.existsSync()) { + file.deleteSync(); + } + }); + + final platform = FakePlatform(); + Context.current.put(env.environment, 'local'); + Context.current.put(ContextKey.platform, platform); + final database = await connect( + Context.current, + name: 'test', + factory: expectAsync1((executor) { + expect(executor, isA()); + return TestDatabase(executor); + }), + hostnameVariable: const env('CELEST_DATABASE_HOST'), + tokenSecret: const secret('CELEST_DATABASE_TOKEN'), + ); + addTearDown(database.close); -final class _FakeSelectable extends Fake implements Selectable { - @override - Future> get() { - return Future.value([]); - } + expect(file.existsSync(), isTrue); + }); + + test('path != null', () async { + final tmpDir = await Directory.systemTemp.createTemp('celest_test'); + addTearDown(() => tmpDir.delete(recursive: true)); + + final platform = FakePlatform(); + Context.current.put(env.environment, 'local'); + Context.current.put(ContextKey.platform, platform); + final database = await connect( + Context.current, + name: 'test', + factory: expectAsync1((executor) { + expect(executor, isA()); + return TestDatabase(executor); + }), + hostnameVariable: const env('CELEST_DATABASE_HOST'), + tokenSecret: const secret('CELEST_DATABASE_TOKEN'), + path: p.join(tmpDir.path, 'test.db'), + ); + addTearDown(database.close); + + final file = File.fromUri(tmpDir.uri.resolve('./test.db')); + expect(file.existsSync(), isTrue); + }); + }); } diff --git a/packages/celest/test/runtime/data/test_database.dart b/packages/celest/test/runtime/data/test_database.dart new file mode 100644 index 00000000..2d26130e --- /dev/null +++ b/packages/celest/test/runtime/data/test_database.dart @@ -0,0 +1,16 @@ +import 'package:drift/drift.dart'; + +part 'test_database.g.dart'; + +class Items extends Table { + IntColumn get id => integer().autoIncrement()(); + TextColumn get name => text().nullable()(); +} + +@DriftDatabase(tables: [Items]) +class TestDatabase extends _$TestDatabase { + TestDatabase(super.e); + + @override + int get schemaVersion => 1; +} diff --git a/packages/celest/test/runtime/data/test_database.g.dart b/packages/celest/test/runtime/data/test_database.g.dart new file mode 100644 index 00000000..df95418b --- /dev/null +++ b/packages/celest/test/runtime/data/test_database.g.dart @@ -0,0 +1,312 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'test_database.dart'; + +// ignore_for_file: type=lint +class $ItemsTable extends Items with TableInfo<$ItemsTable, Item> { + @override + final GeneratedDatabase attachedDatabase; + final String? _alias; + $ItemsTable(this.attachedDatabase, [this._alias]); + static const VerificationMeta _idMeta = const VerificationMeta('id'); + @override + late final GeneratedColumn id = GeneratedColumn( + 'id', aliasedName, false, + hasAutoIncrement: true, + type: DriftSqlType.int, + requiredDuringInsert: false, + defaultConstraints: + GeneratedColumn.constraintIsAlways('PRIMARY KEY AUTOINCREMENT')); + static const VerificationMeta _nameMeta = const VerificationMeta('name'); + @override + late final GeneratedColumn name = GeneratedColumn( + 'name', aliasedName, true, + type: DriftSqlType.string, requiredDuringInsert: false); + @override + List get $columns => [id, name]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'items'; + @override + VerificationContext validateIntegrity(Insertable instance, + {bool isInserting = false}) { + final context = VerificationContext(); + final data = instance.toColumns(true); + if (data.containsKey('id')) { + context.handle(_idMeta, id.isAcceptableOrUnknown(data['id']!, _idMeta)); + } + if (data.containsKey('name')) { + context.handle( + _nameMeta, name.isAcceptableOrUnknown(data['name']!, _nameMeta)); + } + return context; + } + + @override + Set get $primaryKey => {id}; + @override + Item map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return Item( + id: attachedDatabase.typeMapping + .read(DriftSqlType.int, data['${effectivePrefix}id'])!, + name: attachedDatabase.typeMapping + .read(DriftSqlType.string, data['${effectivePrefix}name']), + ); + } + + @override + $ItemsTable createAlias(String alias) { + return $ItemsTable(attachedDatabase, alias); + } +} + +class Item extends DataClass implements Insertable { + final int id; + final String? name; + const Item({required this.id, this.name}); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['id'] = Variable(id); + if (!nullToAbsent || name != null) { + map['name'] = Variable(name); + } + return map; + } + + ItemsCompanion toCompanion(bool nullToAbsent) { + return ItemsCompanion( + id: Value(id), + name: name == null && nullToAbsent ? const Value.absent() : Value(name), + ); + } + + factory Item.fromJson(Map json, + {ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return Item( + id: serializer.fromJson(json['id']), + name: serializer.fromJson(json['name']), + ); + } + @override + Map toJson({ValueSerializer? serializer}) { + serializer ??= driftRuntimeOptions.defaultSerializer; + return { + 'id': serializer.toJson(id), + 'name': serializer.toJson(name), + }; + } + + Item copyWith({int? id, Value name = const Value.absent()}) => Item( + id: id ?? this.id, + name: name.present ? name.value : this.name, + ); + Item copyWithCompanion(ItemsCompanion data) { + return Item( + id: data.id.present ? data.id.value : this.id, + name: data.name.present ? data.name.value : this.name, + ); + } + + @override + String toString() { + return (StringBuffer('Item(') + ..write('id: $id, ') + ..write('name: $name') + ..write(')')) + .toString(); + } + + @override + int get hashCode => Object.hash(id, name); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is Item && other.id == this.id && other.name == this.name); +} + +class ItemsCompanion extends UpdateCompanion { + final Value id; + final Value name; + const ItemsCompanion({ + this.id = const Value.absent(), + this.name = const Value.absent(), + }); + ItemsCompanion.insert({ + this.id = const Value.absent(), + this.name = const Value.absent(), + }); + static Insertable custom({ + Expression? id, + Expression? name, + }) { + return RawValuesInsertable({ + if (id != null) 'id': id, + if (name != null) 'name': name, + }); + } + + ItemsCompanion copyWith({Value? id, Value? name}) { + return ItemsCompanion( + id: id ?? this.id, + name: name ?? this.name, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (id.present) { + map['id'] = Variable(id.value); + } + if (name.present) { + map['name'] = Variable(name.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('ItemsCompanion(') + ..write('id: $id, ') + ..write('name: $name') + ..write(')')) + .toString(); + } +} + +abstract class _$TestDatabase extends GeneratedDatabase { + _$TestDatabase(QueryExecutor e) : super(e); + $TestDatabaseManager get managers => $TestDatabaseManager(this); + late final $ItemsTable items = $ItemsTable(this); + @override + Iterable> get allTables => + allSchemaEntities.whereType>(); + @override + List get allSchemaEntities => [items]; +} + +typedef $$ItemsTableCreateCompanionBuilder = ItemsCompanion Function({ + Value id, + Value name, +}); +typedef $$ItemsTableUpdateCompanionBuilder = ItemsCompanion Function({ + Value id, + Value name, +}); + +class $$ItemsTableFilterComposer extends Composer<_$TestDatabase, $ItemsTable> { + $$ItemsTableFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnFilters get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnFilters(column)); + + ColumnFilters get name => $composableBuilder( + column: $table.name, builder: (column) => ColumnFilters(column)); +} + +class $$ItemsTableOrderingComposer + extends Composer<_$TestDatabase, $ItemsTable> { + $$ItemsTableOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + ColumnOrderings get id => $composableBuilder( + column: $table.id, builder: (column) => ColumnOrderings(column)); + + ColumnOrderings get name => $composableBuilder( + column: $table.name, builder: (column) => ColumnOrderings(column)); +} + +class $$ItemsTableAnnotationComposer + extends Composer<_$TestDatabase, $ItemsTable> { + $$ItemsTableAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + GeneratedColumn get id => + $composableBuilder(column: $table.id, builder: (column) => column); + + GeneratedColumn get name => + $composableBuilder(column: $table.name, builder: (column) => column); +} + +class $$ItemsTableTableManager extends RootTableManager< + _$TestDatabase, + $ItemsTable, + Item, + $$ItemsTableFilterComposer, + $$ItemsTableOrderingComposer, + $$ItemsTableAnnotationComposer, + $$ItemsTableCreateCompanionBuilder, + $$ItemsTableUpdateCompanionBuilder, + (Item, BaseReferences<_$TestDatabase, $ItemsTable, Item>), + Item, + PrefetchHooks Function()> { + $$ItemsTableTableManager(_$TestDatabase db, $ItemsTable table) + : super(TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + $$ItemsTableFilterComposer($db: db, $table: table), + createOrderingComposer: () => + $$ItemsTableOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + $$ItemsTableAnnotationComposer($db: db, $table: table), + updateCompanionCallback: ({ + Value id = const Value.absent(), + Value name = const Value.absent(), + }) => + ItemsCompanion( + id: id, + name: name, + ), + createCompanionCallback: ({ + Value id = const Value.absent(), + Value name = const Value.absent(), + }) => + ItemsCompanion.insert( + id: id, + name: name, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + )); +} + +typedef $$ItemsTableProcessedTableManager = ProcessedTableManager< + _$TestDatabase, + $ItemsTable, + Item, + $$ItemsTableFilterComposer, + $$ItemsTableOrderingComposer, + $$ItemsTableAnnotationComposer, + $$ItemsTableCreateCompanionBuilder, + $$ItemsTableUpdateCompanionBuilder, + (Item, BaseReferences<_$TestDatabase, $ItemsTable, Item>), + Item, + PrefetchHooks Function()>; + +class $TestDatabaseManager { + final _$TestDatabase _db; + $TestDatabaseManager(this._db); + $$ItemsTableTableManager get items => + $$ItemsTableTableManager(_db, _db.items); +}