llm-tools · adhityan · Nov 13, 2024 · Nov 6, 2024 · Nov 13, 2024 · Nov 13, 2024
diff --git a/core/embedjs-interfaces/CHANGELOG.md b/core/embedjs-interfaces/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-interfaces to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@llm-tools/embedjs-interfaces",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Interfaces for extending the embedjs ecosystem",
     "dependencies": {
         "@langchain/core": "^0.3.17",

diff --git a/core/embedjs-utils/CHANGELOG.md b/core/embedjs-utils/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-utils to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-utils",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Useful util functions when extending the embedjs ecosystem",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17"
+        "@llm-tools/embedjs-interfaces": "0.1.18"
     },
     "type": "module",
     "main": "./src/index.js",

diff --git a/core/embedjs/CHANGELOG.md b/core/embedjs/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/core/embedjs/package.json b/core/embedjs/package.json
@@ -1,12 +1,12 @@
 {
     "type": "module",
     "name": "@llm-tools/embedjs",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets",
     "dependencies": {
         "@langchain/textsplitters": "^0.1.0",
-        "@llm-tools/embedjs-interfaces": "0.1.17",
-        "@llm-tools/embedjs-utils": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
+        "@llm-tools/embedjs-utils": "0.1.18",
         "debug": "^4.3.7",
         "langchain": "^0.3.5",
         "md5": "^2.3.0",

diff --git a/core/embedjs/src/core/rag-application.ts b/core/embedjs/src/core/rag-application.ts
@@ -183,6 +183,7 @@ export class RAGApplication {
         await loader.init();
         const chunks = await loader.getChunks();
 
+        this.debug('Chunks generator received', uniqueId);
         const { newInserts } = await this.batchLoadChunks(uniqueId, chunks);
         this.debug(`Add loader completed with ${newInserts} new entries for`, uniqueId);
 
@@ -226,20 +227,20 @@ export class RAGApplication {
      * The function `batchLoadChunks` processes chunks of data in batches and formats them for insertion.
      * @param {string} uniqueId - The `uniqueId` parameter is a string that represents a unique
      * identifier for loader being processed.
-     * @param incrementalGenerator - The `incrementalGenerator` parameter in the `batchLoadChunks`
+     * @param generator - The `incrementalGenerator` parameter in the `batchLoadChunks`
      * function is an asynchronous generator that yields `LoaderChunk` objects.
      * @returns The `batchLoadChunks` function returns an object with two properties:
      * 1. `newInserts`: The total number of new inserts made during the batch loading process.
      * 2. `formattedChunks`: An array containing the formatted chunks that were processed during the
      * batch loading process.
      */
-    private async batchLoadChunks(uniqueId: string, incrementalGenerator: AsyncGenerator<LoaderChunk, void, void>) {
+    private async batchLoadChunks(uniqueId: string, generator: AsyncGenerator<LoaderChunk, void, void>) {
         let i = 0,
             batchSize = 0,
             newInserts = 0,
             formattedChunks: Chunk[] = [];
 
-        for await (const chunk of incrementalGenerator) {
+        for await (const chunk of generator) {
             batchSize++;
 
             const formattedChunk = {

diff --git a/databases/embedjs-astra/CHANGELOG.md b/databases/embedjs-astra/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-astra to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-astradb",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add AstraDB support to embedjs",
     "dependencies": {
         "@datastax/astra-db-ts": "^1.5.0",
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-cosmos/CHANGELOG.md b/databases/embedjs-cosmos/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-cosmos to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-cosmos",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add CosmosDB support to embedjs",
     "dependencies": {
         "@azure/cosmos": "^4.1.1",
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "debug": "^4.3.7"
     },
     "type": "module",

diff --git a/databases/embedjs-hnswlib/CHANGELOG.md b/databases/embedjs-hnswlib/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-hnswlib to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-hnswlib",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add HNSWLib support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "debug": "^4.3.7",
         "hnswlib-node": "^3.0.0"
     },

diff --git a/databases/embedjs-lancedb/CHANGELOG.md b/databases/embedjs-lancedb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-lancedb to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-lancedb",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add LanceDb support to embedjs",
     "dependencies": {
         "@lancedb/lancedb": "^0.12.0",
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "compute-cosine-similarity": "^1.1.0"
     },
     "type": "module",

diff --git a/databases/embedjs-lmdb/CHANGELOG.md b/databases/embedjs-lmdb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-lmdb to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-lmdb",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add LMDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "lmdb": "^3.1.4"
     },
     "type": "module",

diff --git a/databases/embedjs-mongodb/CHANGELOG.md b/databases/embedjs-mongodb/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-mongodb to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-mongodb",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add MongoDB support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "debug": "^4.3.7",
         "mongodb": "^6.10.0"
     },

diff --git a/databases/embedjs-pinecone/CHANGELOG.md b/databases/embedjs-pinecone/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-pinecone to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-pinecone",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add Pinecone support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "@pinecone-database/pinecone": "^4.0.0",
         "debug": "^4.3.7"
     },

diff --git a/databases/embedjs-qdrant/CHANGELOG.md b/databases/embedjs-qdrant/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-qdrant to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-qdrant",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add Qdrant support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "@qdrant/js-client-rest": "^1.12.0",
         "debug": "^4.3.7",
         "uuid": "^11.0.2"

diff --git a/databases/embedjs-redis/CHANGELOG.md b/databases/embedjs-redis/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-redis to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-redis",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add Redis support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "ioredis": "^5.4.1"
     },
     "type": "module",

diff --git a/databases/embedjs-weaviate/CHANGELOG.md b/databases/embedjs-weaviate/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-weaviate to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json
@@ -1,9 +1,9 @@
 {
     "name": "@llm-tools/embedjs-weaviate",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Add Weaviate support to embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
         "compute-cosine-similarity": "^1.1.0",
         "debug": "^4.3.7",
         "weaviate-ts-client": "^2.2.0"

diff --git a/docs/get-started/faq.mdx b/docs/get-started/faq.mdx
@@ -60,8 +60,8 @@ const ragApplication = await new RAGApplicationBuilder()
     .setEmbeddingModel(new OllamaEmbeddings({ model: 'nomic-embed-text', baseUrl: 'http://localhost:11434' }))
     .build();
 
-ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })
-ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' })
+await ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })
+await ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' })
 
 await ragApplication.query('What is the net worth of Elon Musk today?')
 ```

diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx
@@ -45,8 +45,8 @@ const ragApplication = await new RAGApplicationBuilder()
 .setVectorDatabase(new HNSWDb())
 .build();
 
-ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })
-ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' })
+await ragApplication.addLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })
+await ragApplication.addLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' })
 
 await ragApplication.query('What is the net worth of Elon Musk today?')
 //Answer: The net worth of Elon Musk today is $258.7 billion.
@@ -63,7 +63,7 @@ In this section, we will use both an LLM and embedding model from OpenAI.
 
 <CodeGroup>
 ```ts openai_demo.ts
-import { RAGApplicationBuilder } from '@llm-tools/embedjs';
+import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs';
 import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { WebLoader } from '@llm-tools/embedjs-loader-web';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
@@ -77,8 +77,8 @@ const ragApplication = await new RAGApplicationBuilder()
 .setVectorDatabase(new HNSWDb())
 .build();
 
-ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }));
-ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }));
+await ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }));
+await ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }));
 
 await ragApplication.query('What is the net worth of Elon Musk today?')
 //Answer: The net worth of Elon Musk today is $258.7 billion.

diff --git a/examples/simple/src/main.ts b/examples/simple/src/main.ts
@@ -1,15 +1,16 @@
 import 'dotenv/config';
-import { RAGApplicationBuilder } from '@llm-tools/embedjs';
-import { OpenAi, OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
+import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
 import { WebLoader } from '@llm-tools/embedjs-loader-web';
 import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
 
-const llmApplication = await new RAGApplicationBuilder()
-    .setModel(new OpenAi({ modelName: 'gpt-4o' }))
+const ragApplication = await new RAGApplicationBuilder()
+    .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
     .setEmbeddingModel(new OpenAiEmbeddings())
     .setVectorDatabase(new HNSWDb())
     .build();
 
-await llmApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Tesla,_Inc.' }));
+await ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' }));
+await ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' }));
 
-console.log(await llmApplication.query('Who founded Tesla?'));
+await ragApplication.query('What is the net worth of Elon Musk today?');
diff --git a/loaders/embedjs-loader-confluence/CHANGELOG.md b/loaders/embedjs-loader-confluence/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-loader-confluence to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features

diff --git a/loaders/embedjs-loader-confluence/package.json b/loaders/embedjs-loader-confluence/package.json
@@ -1,10 +1,10 @@
 {
     "name": "@llm-tools/embedjs-loader-confluence",
-    "version": "0.1.17",
+    "version": "0.1.18",
     "description": "Confluence loader for embedjs",
     "dependencies": {
-        "@llm-tools/embedjs-interfaces": "0.1.17",
-        "@llm-tools/embedjs-loader-web": "0.1.17",
+        "@llm-tools/embedjs-interfaces": "0.1.18",
+        "@llm-tools/embedjs-loader-web": "0.1.18",
         "confluence.js": "^1.7.4",
         "debug": "^4.3.7",
         "md5": "^2.3.0"

diff --git a/loaders/embedjs-loader-csv/CHANGELOG.md b/loaders/embedjs-loader-csv/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.1.18 (2024-11-13)
+
+This was a version bump only for embedjs-loader-csv to align it with other projects, there were no code changes.
+
 ## 0.1.17 (2024-11-05)
 
 ### 🚀 Features