Implement rag prototpye

.eslintignore

@@ -45,6 +45,8 @@ version-bump.mjs
 versions.json
 package.json
 manifest.json
+compile-migration.js
+import-meta-url-shim.js
 
 # Obsidian
 .obsidian

.gitignore

@@ -19,4 +19,4 @@ main.js
 data.json
 
 # Exclude macOS Finder (System Explorer) View States
-.DS_Store
+.DS_Store

compile-migration.js

@@ -0,0 +1,12 @@
+const { readMigrationFiles } = require('drizzle-orm/migrator')
+const fs = require('node:fs/promises')
+
+async function compileMigrations() {
+  const migrations = readMigrationFiles({ migrationsFolder: './drizzle/' })
+
+  await fs.writeFile('./src/db/migrations.json', JSON.stringify(migrations))
+
+  console.log('Migrations compiled!')
+}
+
+compileMigrations().catch(console.error)

drizzle.config.ts

@@ -0,0 +1,6 @@
+import { defineConfig } from 'drizzle-kit'
+
+export default defineConfig({
+  dialect: 'postgresql',
+  schema: './src/db/schema.ts',
+})

drizzle/0000_add_vector_extension.sql

@@ -0,0 +1,2 @@
+-- Custom SQL migration file, put you code below! --
+CREATE EXTENSION IF NOT EXISTS vector;

drizzle/0001_create_vector_data_tables.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS "vector_data_text_embedding_3_small" (
+	"id" serial PRIMARY KEY NOT NULL,
+	"path" text NOT NULL,
+	"mtime" bigint NOT NULL,
+	"content" text NOT NULL,
+	"embedding" vector(1536),
+	"metadata" jsonb NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "vector_data_text_embedding_3_large" (
+	"id" serial PRIMARY KEY NOT NULL,
+	"path" text NOT NULL,
+	"mtime" bigint NOT NULL,
+	"content" text NOT NULL,
+	"embedding" vector(3072),
+	"metadata" jsonb NOT NULL
+);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "embeddingIndex_text_embedding_3_small" ON "vector_data_text_embedding_3_small" USING hnsw ("embedding" vector_cosine_ops);

drizzle/meta/0000_snapshot.json

@@ -0,0 +1,16 @@
+{
+  "id": "8344dba7-b655-4009-9c56-894062c17ea6",
+  "prevId": "00000000-0000-0000-0000-000000000000",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {},
+  "enums": {},
+  "schemas": {},
+  "views": {},
+  "sequences": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

drizzle/meta/0001_snapshot.json

@@ -0,0 +1,128 @@
+{
+  "id": "1084161a-8de5-4452-ba8b-6ffc3549d411",
+  "prevId": "8344dba7-b655-4009-9c56-894062c17ea6",
+  "version": "7",
+  "dialect": "postgresql",
+  "tables": {
+    "public.vector_data_text_embedding_3_small": {
+      "name": "vector_data_text_embedding_3_small",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "serial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "path": {
+          "name": "path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mtime": {
+          "name": "mtime",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content": {
+          "name": "content",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "embedding": {
+          "name": "embedding",
+          "type": "vector(1536)",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "metadata": {
+          "name": "metadata",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {
+        "embeddingIndex_text_embedding_3_small": {
+          "name": "embeddingIndex_text_embedding_3_small",
+          "columns": [
+            {
+              "expression": "embedding",
+              "isExpression": false,
+              "asc": true,
+              "nulls": "last",
+              "opclass": "vector_cosine_ops"
+            }
+          ],
+          "isUnique": false,
+          "concurrently": false,
+          "method": "hnsw",
+          "with": {}
+        }
+      },
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    },
+    "public.vector_data_text_embedding_3_large": {
+      "name": "vector_data_text_embedding_3_large",
+      "schema": "",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "serial",
+          "primaryKey": true,
+          "notNull": true
+        },
+        "path": {
+          "name": "path",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "mtime": {
+          "name": "mtime",
+          "type": "bigint",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "content": {
+          "name": "content",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true
+        },
+        "embedding": {
+          "name": "embedding",
+          "type": "vector(3072)",
+          "primaryKey": false,
+          "notNull": false
+        },
+        "metadata": {
+          "name": "metadata",
+          "type": "jsonb",
+          "primaryKey": false,
+          "notNull": true
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {},
+      "checkConstraints": {}
+    }
+  },
+  "enums": {},
+  "schemas": {},
+  "sequences": {},
+  "views": {},
+  "_meta": {
+    "columns": {},
+    "schemas": {},
+    "tables": {}
+  }
+}

drizzle/meta/_journal.json

@@ -0,0 +1,20 @@
+{
+  "version": "7",
+  "dialect": "postgresql",
+  "entries": [
+    {
+      "idx": 0,
+      "version": "7",
+      "when": 1729509950412,
+      "tag": "0000_add_vector_extension",
+      "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "7",
+      "when": 1729509994653,
+      "tag": "0001_create_vector_data_tables",
+      "breakpoints": true
+    }
+  ]
+}

esbuild.config.mjs

@@ -1,3 +1,4 @@
+import path from 'path'
 import esbuild from 'esbuild'
 import process from 'process'
 import builtins from 'builtin-modules'
@@ -14,7 +15,7 @@ const context = await esbuild.context({
   banner: {
     js: banner,
   },
-  entryPoints: ['src/main.ts'], // Change this line
+  entryPoints: ['src/main.ts'],
   bundle: true,
   external: [
     'obsidian',
@@ -33,12 +34,19 @@ const context = await esbuild.context({
     ...builtins,
   ],
   format: 'cjs',
-  target: 'es2018',
+  define: {
+    'import.meta.url': 'import_meta_url',
+    process: '{}',
+    'process.env.NODE_ENV': JSON.stringify(prod ? 'production' : 'development'),
+  },
+  inject: [path.resolve('import-meta-url-shim.js')],
+  target: 'es2020',
   logLevel: 'info',
   sourcemap: prod ? false : 'inline',
   treeShaking: true,
-  outfile: 'main.js', // Changed back to main.js
+  outfile: 'main.js',
   minify: prod,
+  // logLevel: 'debug',
 })
 
 if (prod) {

import-meta-url-shim.js

@@ -0,0 +1,7 @@
+const import_meta_url =
+  typeof document === 'undefined'
+    ? require('url').pathToFileURL(__filename).href
+    : (document.currentScript && document.currentScript.src) ||
+      new URL('main.js', document.baseURI).href
+
+export { import_meta_url }