Monorepo

.github/workflows/build.yml

@@ -13,14 +13,17 @@ jobs:
 
         strategy:
             matrix:
-                node-version: [18.x, 20.x]
+                node-version: [18.x, 20.x, 22.x]
 
         steps:
-            - uses: actions/checkout@v4
+            - name: Checkout repository
+              uses: actions/checkout@v4
             - name: Use Node.js ${{ matrix.node-version }}
               uses: actions/setup-node@v3
               with:
                   node-version: ${{ matrix.node-version }}
                   cache: 'npm'
-            - run: npm ci
-            - run: npm run build:esm
+            - name: Install dependencies
+              run: npm ci
+            - name: Test build
+              run: npm run build:esm

.github/workflows/commitlint.yml

@@ -0,0 +1,16 @@
+name: Lint commit messages
+
+on:
+    pull_request:
+        branches: ['main']
+
+permissions:
+    contents: read
+    pull-requests: read
+
+jobs:
+    commitlint:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+            - uses: wagoid/commitlint-github-action@v6

.gitignore

@@ -62,4 +62,8 @@ kafka/
 zookeeper/
 acme.json
 temp/
-storage/
+storage/
+lmdb/
+
+#env
+.env

core/embedjs-interfaces/src/index.ts

@@ -6,5 +6,6 @@ import { BaseModel } from './interfaces/base-model.js';
 import { BaseConversation } from './interfaces/base-conversations.js';
 
 export * from './types.js';
+export * from './loaders.js';
 export * from './constants.js';
 export { BaseDb, BaseCache, BaseLoader, BaseEmbeddings, BaseModel, BaseConversation };

core/embedjs-interfaces/src/loaders.ts

@@ -0,0 +1,21 @@
+// import { BaseLoader } from './interfaces/base-loader.js';
+import { importBaseLoader } from './util.js';
+
+await importBaseLoader('ConfluenceLoader', '@llm-tools/embedjs-loader-confluence');
+// export type ConfluenceLoaderParam = ConstructorParameters<typeof ConfluenceLoader>[0];
+
+// export type LoaderParam = string | BaseLoader | ({ type: 'Confluence' } & ConfluenceLoaderParam);
+// | ({ type: 'Web' } & ConstructorParameters<typeof WebLoader>[0])
+// | ({ type: 'Doc' } & ConstructorParameters<typeof DocxLoader>[0])
+// | ({ type: 'Excel' } & ConstructorParameters<typeof ExcelLoader>[0])
+// | ({ type: 'Json' } & ConstructorParameters<typeof JsonLoader>[0])
+// | ({ type: 'Pdf' } & ConstructorParameters<typeof PdfLoader>[0])
+// | ({ type: 'Ppt' } & ConstructorParameters<typeof PptLoader>[0])
+// | ({ type: 'Sitemap' } & ConstructorParameters<typeof SitemapLoader>[0])
+// | ({ type: 'Text' } & ConstructorParameters<typeof TextLoader>[0])
+// | ({ type: 'YoutubeChannel' } & ConstructorParameters<typeof YoutubeChannelLoader>[0])
+// | ({ type: 'Youtube' } & ConstructorParameters<typeof YoutubeLoader>[0])
+// | ({ type: 'YoutubeSearch' } & ConstructorParameters<typeof YoutubeSearchLoader>[0])
+// | ({ type: 'LocalPath' } & ConstructorParameters<typeof LocalPathLoader>[0])
+// | ({ type: 'Url' } & ConstructorParameters<typeof UrlLoader>[0])
+// | ({ type: 'Csv' } & ConstructorParameters<typeof CsvLoader>[0]);

core/embedjs-interfaces/src/util.ts

@@ -0,0 +1,25 @@
+import createDebugMessages from 'debug';
+import { BaseLoader } from './interfaces/base-loader.js';
+
+export async function importBaseLoader(moduleName: string, modulePath: string): Promise<BaseLoader> {
+    createDebugMessages('embedjs:interfaces:importBaseLoader')(
+        `Starting import of ${moduleName} from path ${modulePath}`,
+    );
+
+    try {
+        const importedModule = await import(modulePath);
+        createDebugMessages('embedjs:interfaces:importBaseLoader')(`Imported ${modulePath}`);
+
+        if (!importedModule[moduleName] || typeof importedModule[moduleName] !== 'function') {
+            createDebugMessages('embedjs:interfaces:importBaseLoader')(
+                `Module ${moduleName} not found at ${modulePath}`,
+            );
+            return null;
+        }
+
+        return importedModule[moduleName];
+    } catch (e) {
+        createDebugMessages('embedjs:interfaces:importBaseLoader')(`Module ${modulePath} not installed!`, e);
+        return null;
+    }
+}

core/embedjs-interfaces/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

core/embedjs-utils/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

core/embedjs/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-astra/package.json

@@ -1,5 +1,5 @@
 {
-    "name": "@llm-tools/embedjs-astra",
+    "name": "@llm-tools/embedjs-astradb",
     "version": "0.1.1",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {

databases/embedjs-astra/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-cosmos/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-hnswlib/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-lancedb/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-lmdb/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-mongodb/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-pinecone/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-qdrant/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-redis/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

databases/embedjs-weaviate/tsconfig.json

@@ -1,8 +1,8 @@
 {
     "extends": "../../tsconfig.base.json",
     "compilerOptions": {
-        "target": "ES2021",
-        "lib": ["ES2021", "ES2022.Object"],
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
         "module": "NodeNext",
         "moduleResolution": "nodenext",
         "esModuleInterop": true,

examples/simple/eslint.config.js

@@ -0,0 +1,3 @@
+import baseConfig from '../../eslint.config.js';
+
+export default [...baseConfig];

examples/simple/package.json

@@ -0,0 +1,11 @@
+{
+    "name": "@llm-tools/embedjs-examples-simple",
+    "version": "0.1.1",
+    "type": "module",
+    "dependencies": {
+        "dotenv": "^16.4.5"
+    },
+    "scripts": {
+        "start": "nx run examples-simple:serve"
+    }
+}

examples/simple/project.json

@@ -0,0 +1,57 @@
+{
+    "name": "examples-simple",
+    "$schema": "../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "examples/simple/src",
+    "projectType": "application",
+    "tags": [],
+    "targets": {
+        "build": {
+            "executor": "@nx/esbuild:esbuild",
+            "outputs": ["{options.outputPath}"],
+            "defaultConfiguration": "development",
+            "options": {
+                "platform": "node",
+                "outputPath": "dist/examples/simple",
+                "format": ["esm"],
+                "bundle": true,
+                "main": "examples/simple/src/main.ts",
+                "tsConfig": "examples/simple/tsconfig.app.json",
+                "generatePackageJson": false,
+                "esbuildOptions": {
+                    "sourcemap": true,
+                    "outExtension": {
+                        ".js": ".js"
+                    }
+                }
+            },
+            "configurations": {
+                "development": {},
+                "production": {
+                    "esbuildOptions": {
+                        "sourcemap": false,
+                        "outExtension": {
+                            ".js": ".js"
+                        }
+                    }
+                }
+            }
+        },
+        "serve": {
+            "executor": "@nx/js:node",
+            "defaultConfiguration": "development",
+            "dependsOn": ["build"],
+            "options": {
+                "buildTarget": "examples-simple:build",
+                "runBuildTargetDependencies": true
+            },
+            "configurations": {
+                "development": {
+                    "buildTarget": "examples-simple:build:development"
+                },
+                "production": {
+                    "buildTarget": "examples-simple:build:production"
+                }
+            }
+        }
+    }
+}

examples/simple/src/main.ts

@@ -0,0 +1,16 @@
+import 'dotenv/config';
+import { RAGApplicationBuilder } from '@llm-tools/embedjs';
+import { OpenAi, OpenAi3SmallEmbeddings } from '@llm-tools/embedjs-openai';
+import { WebLoader } from '@llm-tools/embedjs-loader-web';
+import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
+
+const llmApplication = await new RAGApplicationBuilder()
+    .setModel(new OpenAi({ modelName: 'gpt-4o' }))
+    .setEmbeddingModel(new OpenAi3SmallEmbeddings())
+    .setVectorDb(new HNSWDb())
+    .setSearchResultCount(30)
+    .build();
+
+await llmApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+
+console.log(await llmApplication.query('Who founded Tesla?'));

examples/simple/tsconfig.app.json

@@ -0,0 +1,9 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"],
+    "include": ["src/**/*.ts"]
+}