feat: fts (#30)

README.md

@@ -26,6 +26,7 @@ async function bootstrap() {
 
   // You can implement your own event repository. It just needs to implement a few methods.
   const eventRepository = new EventRepositorySqlite();
+  await eventRepository.init();
   const relay = new NostrRelay(eventRepository);
   const validator = new Validator();
 

packages/event-repository-sqlite/__test__/event-repository-sqlite.spec.ts

@@ -9,6 +9,7 @@ describe('EventRepositorySqlite', () => {
 
   beforeEach(async () => {
     eventRepository = new EventRepositorySqlite();
+    await eventRepository.init();
     database = eventRepository.getDatabase();
   });
 
@@ -20,6 +21,7 @@ describe('EventRepositorySqlite', () => {
     it('should support create by better-sqlite3.Database', async () => {
       const db = new BetterSqlite3(':memory:');
       const newEventRepository = new EventRepositorySqlite(db);
+      await newEventRepository.init();
       expect(newEventRepository.getDatabase()).toBe(db);
       expect(newEventRepository.getDefaultLimit()).toBe(100);
       await newEventRepository.destroy();
@@ -29,14 +31,15 @@ describe('EventRepositorySqlite', () => {
       const newEventRepository = new EventRepositorySqlite(':memory:', {
         defaultLimit: 10,
       });
+      await newEventRepository.init();
       expect(newEventRepository.getDefaultLimit()).toBe(10);
       await newEventRepository.destroy();
     });
   });
 
   describe('isSearchSupported', () => {
-    it('should return false', () => {
-      expect(eventRepository.isSearchSupported()).toBe(false);
+    it('should return true', () => {
+      expect(eventRepository.isSearchSupported()).toBe(true);
     });
   });
 
@@ -394,11 +397,4 @@ describe('EventRepositorySqlite', () => {
       expect(eventRepository.getDefaultLimit()).toBe(10);
     });
   });
-
-  describe('migrate', () => {
-    it('should not run migration if already migrated', async () => {
-      const result = (eventRepository as any).migrate();
-      expect(result.executedMigrations).toHaveLength(0);
-    });
-  });
 });

packages/event-repository-sqlite/src/event-repository-sqlite.ts

@@ -6,14 +6,16 @@ import {
   Filter,
 } from '@nostr-relay/common';
 import * as BetterSqlite3 from 'better-sqlite3';
-import { readFileSync, readdirSync } from 'fs';
 import {
   JSONColumnType,
   Kysely,
+  Migrator,
   SelectQueryBuilder,
+  sql,
   SqliteDialect,
 } from 'kysely';
-import * as path from 'path';
+import { CustomMigrationProvider } from './migrations';
+import { extractSearchableContent } from './search';
 
 const DEFAULT_LIMIT = 100;
 const MAX_LIMIT_MULTIPLIER = 10;
@@ -24,6 +26,7 @@ export type EventRepositorySqliteOptions = {
 
 export interface Database {
   events: EventTable;
+  events_fts: EventFtsTable;
   generic_tags: GenericTagTable;
 }
 
@@ -39,6 +42,11 @@ interface EventTable {
   d_tag_value: string | null;
 }
 
+interface EventFtsTable {
+  id: string;
+  content: string;
+}
+
 interface GenericTagTable {
   tag: string;
   author: string;
@@ -80,12 +88,15 @@ export class EventRepositorySqlite extends EventRepository {
     this.db = new Kysely<Database>({
       dialect: new SqliteDialect({ database: this.betterSqlite3 }),
     });
-    this.migrate();
 
     this.defaultLimit = options?.defaultLimit ?? DEFAULT_LIMIT;
     this.maxLimit = this.defaultLimit * MAX_LIMIT_MULTIPLIER;
   }
 
+  async init(): Promise<void> {
+    await this.migrate();
+  }
+
   getDatabase(): BetterSqlite3.Database {
     return this.betterSqlite3;
   }
@@ -95,7 +106,7 @@ export class EventRepositorySqlite extends EventRepository {
   }
 
   isSearchSupported(): boolean {
-    return false;
+    return true;
   }
 
   async upsert(event: Event): Promise<EventRepositoryUpsertResult> {
@@ -106,6 +117,17 @@ export class EventRepositorySqlite extends EventRepository {
       const { numInsertedOrUpdatedRows } = await this.db
         .transaction()
         .execute(async trx => {
+          let oldEventId: string | undefined;
+          if (dTagValue !== null) {
+            const row = await trx
+              .selectFrom('events')
+              .select(['id'])
+              .where('author', '=', author)
+              .where('kind', '=', event.kind)
+              .where('d_tag_value', '=', dTagValue)
+              .executeTakeFirst();
+            oldEventId = row ? row.id : undefined;
+          }
           const eventInsertResult = await trx
             .insertInto('events')
             .values({
@@ -171,6 +193,24 @@ export class EventRepositorySqlite extends EventRepository {
               .executeTakeFirst();
           }
 
+          if (oldEventId) {
+            await trx
+              .deleteFrom('events_fts')
+              .where('id', '=', oldEventId)
+              .execute();
+          }
+
+          const searchableContent = extractSearchableContent(event);
+          if (searchableContent) {
+            await trx
+              .insertInto('events_fts')
+              .values({
+                id: event.id,
+                content: searchableContent,
+              })
+              .execute();
+          }
+
           return eventInsertResult;
         });
 
@@ -183,6 +223,21 @@ export class EventRepositorySqlite extends EventRepository {
     }
   }
 
+  async insertToSearch(event: Event): Promise<number> {
+    const searchableContent = extractSearchableContent(event);
+    if (searchableContent) {
+      await this.db
+        .insertInto('events_fts')
+        .values({
+          id: event.id,
+          content: searchableContent,
+        })
+        .execute();
+      return 1;
+    }
+    return 0;
+  }
+
   async find(filter: Filter): Promise<Event[]> {
     const limit = this.getLimitFrom(filter);
     if (limit === 0) return [];
@@ -240,6 +295,15 @@ export class EventRepositorySqlite extends EventRepository {
   private createSelectQuery(filter: Filter): eventSelectQueryBuilder {
     let query = this.db.selectFrom('events as e');
 
+    const searchStr = filter.search?.trim();
+    if (searchStr) {
+      query = query.innerJoin('events_fts as fts', join =>
+        join
+          .onRef('fts.id', '=', 'e.id')
+          .on('fts.content', sql`match`, searchStr),
+      );
+    }
+
     const genericTagsCollection = this.extractGenericTagsCollectionFrom(filter);
     if (genericTagsCollection.length) {
       const [firstGenericTagsFilter, secondGenericTagsFilter] =
@@ -360,59 +424,56 @@ export class EventRepositorySqlite extends EventRepository {
       : Math.min(filter.limit, this.maxLimit);
   }
 
-  private migrate(): {
-    lastMigration: string | undefined;
-    executedMigrations: string[];
-  } {
-    this.betterSqlite3.exec(`
-      CREATE TABLE IF NOT EXISTS nostr_relay_migrations (
-        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        name TEXT NOT NULL,
-        created_at INTEGER NOT NULL
-      )
-    `);
+  private async migrate(): Promise<void> {
+    this.migrateOldMigrationTable();
 
-    const lastMigration = this.betterSqlite3
-      .prepare(`SELECT * FROM nostr_relay_migrations ORDER BY id DESC LIMIT 1`)
-      .get() as { name: string } | undefined;
+    const migrator = new Migrator({
+      db: this.db,
+      provider: new CustomMigrationProvider(),
+      migrationTableName: 'nostr_relay_sqlite_migrations',
+    });
+
+    const { error } = await migrator.migrateToLatest();
 
-    const migrationFileNames = readdirSync(
-      path.join(__dirname, '../migrations'),
-    ).filter(fileName => fileName.endsWith('.sql'));
-
-    const migrationsToRun = (
-      lastMigration
-        ? migrationFileNames.filter(fileName => fileName > lastMigration.name)
-        : migrationFileNames
-    ).sort();
-
-    if (migrationsToRun.length === 0) {
-      return {
-        lastMigration: lastMigration?.name,
-        executedMigrations: [],
-      };
+    if (error) {
+      throw error;
     }
+  }
 
-    const runMigrations = this.betterSqlite3.transaction(() => {
-      migrationsToRun.forEach(fileName => {
-        const migration = readFileSync(
-          path.join(__dirname, '../migrations', fileName),
-          'utf8',
-        );
-        this.betterSqlite3.exec(migration);
-        this.betterSqlite3
-          .prepare(
-            `INSERT INTO nostr_relay_migrations (name, created_at) VALUES (?, ?)`,
-          )
-          .run(fileName, Date.now());
-      });
-    });
-    runMigrations();
+  private migrateOldMigrationTable(): void {
+    const oldMigrationsTable = this.betterSqlite3
+      .prepare(
+        `SELECT name FROM sqlite_master WHERE type='table' AND name='nostr_relay_migrations'`,
+      )
+      .get() as { name: string } | undefined;
 
-    return {
-      lastMigration: migrationsToRun[migrationsToRun.length - 1],
-      executedMigrations: migrationsToRun,
-    };
+    if (oldMigrationsTable) {
+      this.betterSqlite3.exec(`
+        CREATE TABLE IF NOT EXISTS nostr_relay_sqlite_migrations (
+          name TEXT NOT NULL PRIMARY KEY,
+          timestamp TEXT NOT NULL
+        )
+      `);
+
+      const oldMigrations = this.betterSqlite3
+        .prepare(`SELECT * FROM nostr_relay_migrations`)
+        .all() as { name: string; created_at: number }[];
+
+      const runMigrations = this.betterSqlite3.transaction(() => {
+        oldMigrations.forEach(migration => {
+          this.betterSqlite3
+            .prepare(
+              `INSERT INTO nostr_relay_sqlite_migrations (name, timestamp) VALUES (?, ?)`,
+            )
+            .run(
+              migration.name.replace('.sql', ''),
+              new Date(migration.created_at).toISOString(),
+            );
+        });
+        this.betterSqlite3.exec(`DROP TABLE nostr_relay_migrations`);
+      });
+      runMigrations();
+    }
   }
 
   private toEvent(row: any): Event {