diff --git a/.gitignore b/.gitignore index 33db299a..46ab1ce6 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ acme.json temp/ storage/ lmdb/ +data.db #env .env \ No newline at end of file diff --git a/core/embedjs-interfaces/package.json b/core/embedjs-interfaces/package.json index 0952ddd3..5474b959 100644 --- a/core/embedjs-interfaces/package.json +++ b/core/embedjs-interfaces/package.json @@ -1,6 +1,6 @@ { "name": "@llm-tools/embedjs-interfaces", - "version": "0.1.21", + "version": "0.1.22", "description": "Interfaces for extending the embedjs ecosystem", "dependencies": { "@langchain/core": "^0.3.19", diff --git a/core/embedjs-utils/package.json b/core/embedjs-utils/package.json index 0ca9045c..5e1eb5a5 100644 --- a/core/embedjs-utils/package.json +++ b/core/embedjs-utils/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-utils", - "version": "0.1.21", + "version": "0.1.22", "description": "Useful util functions when extending the embedjs ecosystem", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21" + "@llm-tools/embedjs-interfaces": "0.1.22" }, "type": "module", "main": "./src/index.js", diff --git a/core/embedjs/package.json b/core/embedjs/package.json index 6f7499ec..aa2d7656 100644 --- a/core/embedjs/package.json +++ b/core/embedjs/package.json @@ -1,12 +1,12 @@ { "type": "module", "name": "@llm-tools/embedjs", - "version": "0.1.21", + "version": "0.1.22", "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7", "langchain": "^0.3.6", "md5": "^2.3.0", diff --git a/core/embedjs/src/core/rag-application.ts b/core/embedjs/src/core/rag-application.ts index 7f5763b4..de8c2e0c 100644 --- a/core/embedjs/src/core/rag-application.ts +++ b/core/embedjs/src/core/rag-application.ts @@ -306,6 +306,17 @@ export class RAGApplication { return this.vectorDatabase.getVectorCount(); } + /** + * The function `deleteConversation` deletes all entries related to a particular conversation from the database + * @param {string} conversationId - The `conversationId` that you want to delete. Pass 'default' to delete + * the default conversation thread that is created and maintained automatically + */ + public async deleteConversation(conversationId: string) { + if (this.store) { + await this.store.deleteConversation(conversationId); + } + } + /** * The function `deleteLoader` deletes embeddings from a loader after confirming the action. * @param {string} uniqueLoaderId - The `uniqueLoaderId` parameter is a string that represents the diff --git a/core/embedjs/src/store/memory-store.ts b/core/embedjs/src/store/memory-store.ts index a5cc6d1e..11c3f905 100644 --- a/core/embedjs/src/store/memory-store.ts +++ b/core/embedjs/src/store/memory-store.ts @@ -50,16 +50,22 @@ export class MemoryStore implements BaseStore { const loaderId = this.loaderCustomValues[key].loaderId; delete this.loaderList[key]; - this.loaderCustomValuesMap.set( - loaderId, - this.loaderCustomValuesMap.get(loaderId).filter((k) => k !== key), - ); + + if (this.loaderCustomValuesMap.has(loaderId)) { + this.loaderCustomValuesMap.set( + loaderId, + this.loaderCustomValuesMap.get(loaderId).filter((k) => k !== key), + ); + } } async deleteLoaderMetadataAndCustomValues(loaderId: string): Promise { - this.loaderCustomValuesMap.get(loaderId).forEach((key) => { - delete this.loaderCustomValues[key]; - }); + if (this.loaderCustomValuesMap.has(loaderId)) { + this.loaderCustomValuesMap.get(loaderId).forEach((key) => { + delete this.loaderCustomValues[key]; + }); + } + this.loaderCustomValuesMap.delete(loaderId); delete this.loaderList[loaderId]; } diff --git a/databases/embedjs-astra/package.json b/databases/embedjs-astra/package.json index a4ee78ac..4ec0d0f7 100644 --- a/databases/embedjs-astra/package.json +++ b/databases/embedjs-astra/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-astradb", - "version": "0.1.21", + "version": "0.1.22", "description": "Add AstraDB support to embedjs", "dependencies": { "@datastax/astra-db-ts": "^1.5.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/databases/embedjs-cosmos/package.json b/databases/embedjs-cosmos/package.json index a9906634..80a4e143 100644 --- a/databases/embedjs-cosmos/package.json +++ b/databases/embedjs-cosmos/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-cosmos", - "version": "0.1.21", + "version": "0.1.22", "description": "Add CosmosDB support to embedjs", "dependencies": { "@azure/cosmos": "^4.2.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/databases/embedjs-hnswlib/package.json b/databases/embedjs-hnswlib/package.json index c87c3e9d..1c7c3247 100644 --- a/databases/embedjs-hnswlib/package.json +++ b/databases/embedjs-hnswlib/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-hnswlib", - "version": "0.1.21", + "version": "0.1.22", "description": "Add HNSWLib support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "hnswlib-node": "^3.0.0" }, diff --git a/databases/embedjs-lancedb/package.json b/databases/embedjs-lancedb/package.json index 05c4db83..65140350 100644 --- a/databases/embedjs-lancedb/package.json +++ b/databases/embedjs-lancedb/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-lancedb", - "version": "0.1.21", + "version": "0.1.22", "description": "Add LanceDb support to embedjs", "dependencies": { "@lancedb/lancedb": "^0.13.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7" }, diff --git a/databases/embedjs-libsql/CHANGELOG.md b/databases/embedjs-libsql/CHANGELOG.md index 38b0cf3b..6c3d43de 100644 --- a/databases/embedjs-libsql/CHANGELOG.md +++ b/databases/embedjs-libsql/CHANGELOG.md @@ -1,5 +1,5 @@ -## 0.1.19 (2024-11-26) +## 0.1.22 (2024-11-26) ### 🚀 Features -- added sqlite vector database ([cd27bc5](https://github.com/llm-tools/embedJs/commit/cd27bc5)) +- added sqlite vector database ([cd27bc5](https://github.com/llm-tools/embedJs/commit/cd27bc5)) diff --git a/databases/embedjs-libsql/package.json b/databases/embedjs-libsql/package.json index c7ca0f2d..913b5092 100644 --- a/databases/embedjs-libsql/package.json +++ b/databases/embedjs-libsql/package.json @@ -1,10 +1,11 @@ { "name": "@llm-tools/embedjs-libsql", - "version": "0.1.21", + "version": "0.1.22", "description": "Add LibSQL support to embedjs", "dependencies": { "@libsql/client": "^0.14.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/databases/embedjs-libsql/src/index.ts b/databases/embedjs-libsql/src/index.ts index 78401e25..c2ab62ac 100644 --- a/databases/embedjs-libsql/src/index.ts +++ b/databases/embedjs-libsql/src/index.ts @@ -1 +1,2 @@ export * from './libsql-db.js'; +export * from './libsql-store.js'; diff --git a/databases/embedjs-libsql/src/libsql-db.ts b/databases/embedjs-libsql/src/libsql-db.ts index b08125f1..83243ac3 100644 --- a/databases/embedjs-libsql/src/libsql-db.ts +++ b/databases/embedjs-libsql/src/libsql-db.ts @@ -2,6 +2,7 @@ import createDebugMessages from 'debug'; import { Client, createClient } from '@libsql/client'; import { BaseVectorDatabase, ExtractChunkData, InsertChunkData } from '@llm-tools/embedjs-interfaces'; +import { truncateCenterString } from '@llm-tools/embedjs-utils'; export class LibSqlDb implements BaseVectorDatabase { private readonly debug = createDebugMessages('embedjs:vector:LibSqlDb'); @@ -16,46 +17,48 @@ export class LibSqlDb implements BaseVectorDatabase { } async init({ dimensions }: { dimensions: number }) { - await this.client.execute(`CREATE TABLE ${this.tableName} ( - id TEXT - pageContent TEXT, - uniqueLoaderId TEXT, - source TEXT, + await this.client.execute(`CREATE TABLE IF NOT EXISTS ${this.tableName} ( + id TEXT PRIMARY KEY, + pageContent TEXT UNIQUE, + uniqueLoaderId TEXT NOT NULL, + source TEXT NOT NULL, vector F32_BLOB(${dimensions}), metadata TEXT );`); } async insertChunks(chunks: InsertChunkData[]): Promise { - const values = chunks.map((chunk) => { - return `('${chunk.metadata.id}', '${chunk.pageContent}', '${chunk.metadata.uniqueLoaderId}', - '${chunk.metadata.source}', vector32('[${chunk.vector.join(',')}]'), - '${JSON.stringify(chunk.metadata)}')`; + const batch = chunks.map((chunk) => { + return { + sql: `INSERT OR IGNORE INTO ${this.tableName} (id, pageContent, uniqueLoaderId, source, vector, metadata) + VALUES (?, ?, ?, ?, vector32('[${chunk.vector.join(',')}]'), ?);`, + args: [ + chunk.metadata.id, + chunk.pageContent, + chunk.metadata.uniqueLoaderId, + chunk.metadata.source, + JSON.stringify(chunk.metadata), + ], + }; }); - const statement = `INSERT INTO ${this.tableName} (id, pageContent, uniqueLoaderId, source, vector, metadata) - VALUES - ${values.join(',')} - );`; - - this.debug(`Executing statement - ${statement}`); - const result = await this.client.execute(statement); - return result.rowsAffected; + this.debug(`Executing batch - ${truncateCenterString(JSON.stringify(batch), 1000)}`); + const result = await this.client.batch(batch, 'write'); + return result.reduce((a, b) => a + b.rowsAffected, 0); } async similaritySearch(query: number[], k: number): Promise { const statement = `SELECT id, pageContent, uniqueLoaderId, source, metadata, vector_distance_cos(vector, vector32('[${query.join(',')}]')) FROM ${this.tableName} - ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) - TOP ${k} - ASC;`; + ORDER BY vector_distance_cos(vector, vector32('[${query.join(',')}]')) ASC + LIMIT ${k};`; - this.debug(`Executing statement - ${statement}`); + this.debug(`Executing statement - ${truncateCenterString(statement, 700)}`); const results = await this.client.execute(statement); return results.rows.map((result) => { - const metadata = JSON.parse(result.metadata.toString()); + const metadata = JSON.parse(result.metadata.toString()); return { metadata, @@ -66,16 +69,21 @@ export class LibSqlDb implements BaseVectorDatabase { } async getVectorCount(): Promise { - return 0; + const statement = `SELECT count(id) as count FROM ${this.tableName};`; + + this.debug(`Executing statement - ${statement}`); + const results = await this.client.execute(statement); + + return Number.parseInt(results.rows[0].count.toString()); } async deleteKeys(uniqueLoaderId: string): Promise { - const statement = `DELETE FROM ${this.tableName} VALUES WHERE - uniqueLoaderId = '${uniqueLoaderId}';`; - - await this.client.execute(statement); + await this.client.execute(`DELETE FROM ${this.tableName} WHERE + uniqueLoaderId = '${uniqueLoaderId}';`); return true; } - async reset(): Promise {} + async reset(): Promise { + await this.client.execute(`DELETE FROM ${this.tableName};`); + } } diff --git a/databases/embedjs-libsql/src/libsql-store.ts b/databases/embedjs-libsql/src/libsql-store.ts new file mode 100644 index 00000000..1472a86f --- /dev/null +++ b/databases/embedjs-libsql/src/libsql-store.ts @@ -0,0 +1,232 @@ +import createDebugMessages from 'debug'; +import { Client, createClient } from '@libsql/client'; +import { BaseStore, Conversation, LoaderListEntry, Message } from '@llm-tools/embedjs-interfaces'; + +export class LibSqlStore implements BaseStore { + private readonly debug = createDebugMessages('embedjs:store:LibSqlStore'); + private readonly loadersCustomDataTableName: string; + private readonly conversationsTableName: string; + private readonly loadersTableName: string; + private readonly client: Client; + + constructor({ + path, + loadersTableName, + conversationsTableName, + loadersCustomDataTableName, + }: { + path: string; + loadersTableName?: string; + conversationsTableName?: string; + loadersCustomDataTableName?: string; + }) { + this.loadersCustomDataTableName = loadersCustomDataTableName ?? 'loaderCustomData'; + this.conversationsTableName = conversationsTableName ?? 'conversations'; + this.loadersTableName = loadersTableName ?? 'loaders'; + this.client = createClient({ + url: `file:${path}`, + }); + } + + async init(): Promise { + this.debug(`Creating table '${this.conversationsTableName}'`); + await this.client.execute(`CREATE TABLE IF NOT EXISTS ${this.conversationsTableName} ( + id TEXT PRIMARY KEY, + conversationId TEXT NOT NULL, + content TEXT NOT NULL, + timestamp TEXT NOT NULL, + actor TEXT NOT NULL, + sources TEXT + ); + + CREATE INDEX ${this.conversationsTableName}_index ON ${this.conversationsTableName} (conversationId);`); + this.debug(`Created table '${this.conversationsTableName}' and related indexes`); + + this.debug(`Creating table '${this.loadersTableName}'`); + await this.client.execute(`CREATE TABLE IF NOT EXISTS ${this.loadersTableName} ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + chunksProcessed INTEGER, + metadata TEXT + );`); + this.debug(`Created table '${this.loadersTableName}'`); + + this.debug(`Creating table '${this.loadersCustomDataTableName}'`); + await this.client.execute(`CREATE TABLE IF NOT EXISTS ${this.loadersCustomDataTableName} ( + key TEXT PRIMARY KEY, + loaderId TEXT NOT NULL, + value TEXT + ); + + CREATE INDEX ${this.loadersCustomDataTableName}_index ON ${this.loadersCustomDataTableName} (loaderId);`); + this.debug(`Created table '${this.loadersCustomDataTableName}' and related indexes`); + } + + async addLoaderMetadata(loaderId: string, value: LoaderListEntry): Promise { + await this.client.execute(`DELETE FROM ${this.loadersTableName} WHERE id = '${loaderId}';`); + + await this.client.execute({ + sql: `INSERT INTO ${this.loadersTableName} (id, type, chunksProcessed, metadata) + VALUES (?, ?, ?, ?)`, + args: [loaderId, value.type, value.chunksProcessed, JSON.stringify(value.loaderMetadata)], + }); + } + + async getLoaderMetadata(loaderId: string): Promise { + const results = await this.client.execute({ + sql: `SELECT type, chunksProcessed, metadata FROM ${this.loadersTableName} WHERE id = ?;`, + args: [loaderId], + }); + + const result = results.rows[0]; + const metadata = JSON.parse(result.metadata.toString()); + + return { + uniqueId: loaderId, + loaderMetadata: metadata, + chunksProcessed: Number.parseInt(result.chunksProcessed.toString()), + type: result.type.toString(), + }; + } + + async hasLoaderMetadata(loaderId: string): Promise { + const results = await this.client.execute({ + sql: `SELECT type, chunksProcessed, metadata FROM ${this.loadersTableName} WHERE id = ?;`, + args: [loaderId], + }); + + return results.rows.length > 0; + } + + async getAllLoaderMetadata(): Promise { + const results = await this.client.execute( + `SELECT id, type, chunksProcessed, metadata FROM ${this.loadersTableName};`, + ); + + return results.rows.map((result) => { + const metadata = JSON.parse(result.metadata.toString()); + + return { + uniqueId: result.id.toString(), + loaderMetadata: metadata, + chunksProcessed: Number.parseInt(result.chunksProcessed.toString()), + type: result.type.toString(), + }; + }); + } + + async loaderCustomSet>(loaderId: string, key: string, value: T): Promise { + await this.loaderCustomDelete(key); + + await this.client.execute({ + sql: `INSERT INTO ${this.loadersCustomDataTableName} (key, loaderId, value) + VALUES (?, ?, ?)`, + args: [key, loaderId, JSON.stringify(value)], + }); + } + + async loaderCustomGet>(key: string): Promise { + const results = await this.client.execute({ + sql: `SELECT value FROM ${this.loadersCustomDataTableName} WHERE key = ?;`, + args: [key], + }); + + return JSON.parse(results.rows[0].value.toString()); + } + + async loaderCustomHas(key: string): Promise { + const results = await this.client.execute({ + sql: `SELECT value FROM ${this.loadersCustomDataTableName} WHERE key = ?;`, + args: [key], + }); + + return results.rows.length > 0; + } + + async loaderCustomDelete(key: string): Promise { + await this.client.execute(`DELETE FROM ${this.loadersCustomDataTableName} WHERE key = '${key}';`); + } + + async deleteLoaderMetadataAndCustomValues(loaderId: string): Promise { + await this.client.execute(`DELETE FROM ${this.loadersTableName} WHERE id = '${loaderId}';`); + await this.client.execute(`DELETE FROM ${this.loadersCustomDataTableName} WHERE loaderId = '${loaderId}';`); + } + + async getConversation(conversationId: string): Promise { + const results = await this.client.execute({ + sql: `SELECT id, conversationId, content, timestamp, actor, sources FROM ${this.conversationsTableName} WHERE conversationId = ?;`, + args: [conversationId], + }); + + return { + conversationId, + entries: results.rows.map((result) => { + const timestamp = new Date(result.timestamp.toString()); + const actor = result.actor.toString(); + + if (actor === 'AI') { + return { + actor: 'AI', + id: result.id.toString(), + content: result.content.toString(), + sources: JSON.parse(result.sources.toString()), + timestamp, + }; + } else { + return { + id: result.id.toString(), + content: result.content.toString(), + actor: <'HUMAN' | 'SYSTEM'>actor, + timestamp, + }; + } + }), + }; + } + + async hasConversation(conversationId: string): Promise { + const results = await this.client.execute({ + sql: `SELECT id, conversationId, content, timestamp, actor FROM ${this.conversationsTableName} WHERE conversationId = ? LIMIT 1;`, + args: [conversationId], + }); + + return results.rows.length > 0; + } + + async deleteConversation(conversationId: string): Promise { + await this.client.execute( + `DELETE FROM ${this.conversationsTableName} WHERE conversationId = '${conversationId}';`, + ); + } + + async addEntryToConversation(conversationId: string, entry: Message): Promise { + if (entry.actor !== 'AI') { + await this.client.execute({ + sql: `INSERT OR IGNORE INTO ${this.conversationsTableName} (id, conversationId, content, timestamp, actor) + VALUES (?, ?, ?, ?, ?)`, + args: [entry.id, conversationId, entry.content, entry.timestamp, entry.actor], + }); + } else { + await this.client.execute({ + sql: `INSERT OR IGNORE INTO ${this.conversationsTableName} (id, conversationId, content, timestamp, actor, sources) + VALUES (?, ?, ?, ?, ?, ?)`, + args: [ + entry.id, + conversationId, + entry.content, + entry.timestamp, + entry.actor, + JSON.stringify(entry.sources), + ], + }); + } + } + + async clearConversations(): Promise { + await this.client.execute(`DELETE FROM ${this.conversationsTableName};`); + } + + async addConversation(_conversationId: string): Promise { + //There is nothing to be done for this in this libsql implementation + } +} diff --git a/databases/embedjs-lmdb/package.json b/databases/embedjs-lmdb/package.json index 4853963e..35945283 100644 --- a/databases/embedjs-lmdb/package.json +++ b/databases/embedjs-lmdb/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-lmdb", - "version": "0.1.21", + "version": "0.1.22", "description": "Add LMDB support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "lmdb": "^3.1.6" }, diff --git a/databases/embedjs-mongodb/package.json b/databases/embedjs-mongodb/package.json index e841010d..eada7230 100644 --- a/databases/embedjs-mongodb/package.json +++ b/databases/embedjs-mongodb/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-mongodb", - "version": "0.1.21", + "version": "0.1.22", "description": "Add MongoDB support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "mongodb": "^6.11.0" }, diff --git a/databases/embedjs-pinecone/package.json b/databases/embedjs-pinecone/package.json index 23f32756..10762677 100644 --- a/databases/embedjs-pinecone/package.json +++ b/databases/embedjs-pinecone/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-pinecone", - "version": "0.1.21", + "version": "0.1.22", "description": "Add Pinecone support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "@pinecone-database/pinecone": "^4.0.0", "debug": "^4.3.7" }, diff --git a/databases/embedjs-qdrant/package.json b/databases/embedjs-qdrant/package.json index fe45bb23..d140bde4 100644 --- a/databases/embedjs-qdrant/package.json +++ b/databases/embedjs-qdrant/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-qdrant", - "version": "0.1.21", + "version": "0.1.22", "description": "Add Qdrant support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "@qdrant/js-client-rest": "^1.12.0", "debug": "^4.3.7", "uuid": "^11.0.3" diff --git a/databases/embedjs-redis/package.json b/databases/embedjs-redis/package.json index 2feda20e..6ef73159 100644 --- a/databases/embedjs-redis/package.json +++ b/databases/embedjs-redis/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-redis", - "version": "0.1.21", + "version": "0.1.22", "description": "Add Redis support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "ioredis": "^5.4.1" }, "type": "module", diff --git a/databases/embedjs-weaviate/package.json b/databases/embedjs-weaviate/package.json index 1c12df68..f3fe7595 100644 --- a/databases/embedjs-weaviate/package.json +++ b/databases/embedjs-weaviate/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-weaviate", - "version": "0.1.21", + "version": "0.1.22", "description": "Add Weaviate support to embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7", "weaviate-ts-client": "^2.2.0" diff --git a/docs/api-reference/methods/add-loader.mdx b/docs/api-reference/methods/add-loader.mdx index 2cb46546..12cc9b79 100644 --- a/docs/api-reference/methods/add-loader.mdx +++ b/docs/api-reference/methods/add-loader.mdx @@ -6,7 +6,7 @@ The `addLoader()` method is used to load the data sources from different data so ### Parameters - + The data to embed, can be a URL, local file or raw content, depending on the data type.. You can find the full list of supported data sources [here](/components/data-sources/overview). diff --git a/docs/api-reference/methods/delete-conversation.mdx b/docs/api-reference/methods/delete-conversation.mdx new file mode 100644 index 00000000..b8e59517 --- /dev/null +++ b/docs/api-reference/methods/delete-conversation.mdx @@ -0,0 +1,37 @@ +--- +title: 🗑 deleteConversation +--- + +The `deleteConversation()` method allows you to delete a particular conversation thread from the database + +### Parameters + + + The conversationId that you want to delete. + If you want to delete the conversations stored when there were no conversation ids passed, pass in 'default' + + +### Usage + +```ts +import 'dotenv/config'; +import path from 'node:path'; +import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs'; +import { LibSqlDb, LibSqlStore } from '@llm-tools/embedjs-libsql'; +import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +const databasePath = path.resolve('./examples/libsql/data.db'); +const ragApplication = await new RAGApplicationBuilder() + .setStore(new LibSqlStore({ path: databasePath })) + .setVectorDatabase(new LibSqlDb({ path: databasePath })) + .setEmbeddingModel(new OpenAiEmbeddings()) + .setModel(SIMPLE_MODELS.OPENAI_GPT4_O) + .build(); + +await ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' })); +console.log(await ragApplication.query('Was Elon Musk the founder of Tesla originally?')); +console.log(await ragApplication.query('What is the net worth of Elon Musk today?')); + +await app.deleteConversation('default'); +``` diff --git a/docs/api-reference/methods/delete-loader.mdx b/docs/api-reference/methods/delete-loader.mdx index ccb4c8d0..483cb16a 100644 --- a/docs/api-reference/methods/delete-loader.mdx +++ b/docs/api-reference/methods/delete-loader.mdx @@ -6,7 +6,7 @@ The `deleteLoader()` method allows you to delete the data loaded by a loader pre ### Parameters - + The loader unique id that is generated when the loader is added to the app. diff --git a/docs/api-reference/methods/search.mdx b/docs/api-reference/methods/search.mdx index 3532b8a6..71abc7bd 100644 --- a/docs/api-reference/methods/search.mdx +++ b/docs/api-reference/methods/search.mdx @@ -7,7 +7,7 @@ Refer to the function signature below: ### Parameters - + Question diff --git a/docs/components/stores/libsql.mdx b/docs/components/stores/libsql.mdx new file mode 100644 index 00000000..c2d11bba --- /dev/null +++ b/docs/components/stores/libsql.mdx @@ -0,0 +1,24 @@ +--- +title: LibSQL +--- + +You can use [LibSQL](https://docs.turso.tech/introduction) a fork of SQLite to store values locally on disk. + +## Install LibSQL addon + +```bash +npm install @llm-tools/embedjs-libsql +``` + +## Usage + +```ts Partial example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { LibSqlStore } from '@llm-tools/embedjs-libsql'; + +const app = await new RAGApplicationBuilder() +.setStore(new LibSqlStore({ path: './data.db' })) +``` + + + diff --git a/docs/components/stores/overview.mdx b/docs/components/stores/overview.mdx index c63c3e58..0e05b5ef 100644 --- a/docs/components/stores/overview.mdx +++ b/docs/components/stores/overview.mdx @@ -12,6 +12,7 @@ The library handles the complexity of being able to swap the data persistance la + diff --git a/docs/components/vector-databases/astradb.mdx b/docs/components/vector-databases/astradb.mdx index 2258326e..4226132b 100644 --- a/docs/components/vector-databases/astradb.mdx +++ b/docs/components/vector-databases/astradb.mdx @@ -40,7 +40,7 @@ const app = await new RAGApplicationBuilder() .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/components/vector-databases/hnswlib.mdx b/docs/components/vector-databases/hnswlib.mdx index 2ecdf905..6e30b03a 100644 --- a/docs/components/vector-databases/hnswlib.mdx +++ b/docs/components/vector-databases/hnswlib.mdx @@ -28,7 +28,7 @@ const app = await new RAGApplicationBuilder() .setVectorDatabase(new HNSWDb()) .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/components/vector-databases/lancedb.mdx b/docs/components/vector-databases/lancedb.mdx index 3e068e54..1aae2551 100644 --- a/docs/components/vector-databases/lancedb.mdx +++ b/docs/components/vector-databases/lancedb.mdx @@ -34,7 +34,7 @@ const app = await new RAGApplicationBuilder() .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/components/vector-databases/libsql.mdx b/docs/components/vector-databases/libsql.mdx new file mode 100644 index 00000000..7690d543 --- /dev/null +++ b/docs/components/vector-databases/libsql.mdx @@ -0,0 +1,45 @@ +--- +title: LibSQL +--- + +LibSQL is an open source fork of SQLite and in most cases it is a drop in replacement for SQLite. It has native in-built support for vector similarity search. +Read more about LibSQL's support for AI [here](https://docs.turso.tech/features/ai-and-embeddings). +You can run LibSQL locally with EmbedJs and is available via the LibSQL extension for embedJs. + + + LibSql can also be used to store conversations and other data generated in embedJs. + This is done via the LibSqlStore available in the LibSQL extension for embedJs. Read more about it [here](/components/stores/libsql) + + + +## Install LibSQL addon + +```bash +npm install @llm-tools/embedjs-libsql +``` + +## Usage + +```ts Example +import { RAGApplicationBuilder } from '@llm-tools/embedjs'; +import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; +import { LibSqlDb } from '@llm-tools/embedjs-libsql'; + +// set OPENAI_API_KEY in your env +process.env.OPENAI_API_KEY = "sk-xxx"; + +const app = await new RAGApplicationBuilder() +.setVectorDatabase(new LibSqlDb({ path: './data.db' })) +.setEmbeddingModel(new OpenAiEmbeddings()) +.setModel(SIMPLE_MODELS.OPENAI_GPT4_O) +.build(); + + +//add data source and start query it +await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); +await app.query('Tell me about Elon Musk'); +``` + + + diff --git a/docs/components/vector-databases/mongodb.mdx b/docs/components/vector-databases/mongodb.mdx index 8681651a..15fb79ae 100644 --- a/docs/components/vector-databases/mongodb.mdx +++ b/docs/components/vector-databases/mongodb.mdx @@ -45,7 +45,7 @@ const app = await new RAGApplicationBuilder() })), .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/components/vector-databases/overview.mdx b/docs/components/vector-databases/overview.mdx index 862df84a..48c40be7 100644 --- a/docs/components/vector-databases/overview.mdx +++ b/docs/components/vector-databases/overview.mdx @@ -11,6 +11,7 @@ We've provided examples for each supported database below: + diff --git a/docs/components/vector-databases/pinecone.mdx b/docs/components/vector-databases/pinecone.mdx index 922043bc..167ed5f3 100644 --- a/docs/components/vector-databases/pinecone.mdx +++ b/docs/components/vector-databases/pinecone.mdx @@ -44,7 +44,7 @@ const app = await new RAGApplicationBuilder() .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/components/vector-databases/qdrant.mdx b/docs/components/vector-databases/qdrant.mdx index 05d3b1c7..33b32a8c 100644 --- a/docs/components/vector-databases/qdrant.mdx +++ b/docs/components/vector-databases/qdrant.mdx @@ -28,7 +28,7 @@ const app = await new RAGApplicationBuilder() .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/components/vector-databases/weaviate.mdx b/docs/components/vector-databases/weaviate.mdx index c4a6130f..bb9aad2d 100644 --- a/docs/components/vector-databases/weaviate.mdx +++ b/docs/components/vector-databases/weaviate.mdx @@ -28,7 +28,7 @@ const app = await new RAGApplicationBuilder() .build(); -//add data source and start query in +//add data source and start query it await app.addLoader(new WebLoader({ urlOrContent: 'https://www.forbes.com/profile/elon-musk' })); await app.query('Tell me about Elon Musk'); ``` diff --git a/docs/mint.json b/docs/mint.json index 1df234b5..56164fc9 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -96,6 +96,7 @@ { "group": "Built in", "pages": [ + "components/vector-databases/libsql", "components/vector-databases/lancedb", "components/vector-databases/mongodb", "components/vector-databases/pinecone", @@ -148,7 +149,12 @@ "components/stores/overview", { "group": "Built in", - "pages": ["components/stores/lmdb", "components/stores/mongodb", "components/stores/redis"] + "pages": [ + "components/stores/lmdb", + "components/stores/libsql", + "components/stores/mongodb", + "components/stores/redis" + ] } ] } @@ -174,6 +180,7 @@ "api-reference/methods/add-loader", "api-reference/methods/get-loaders", "api-reference/methods/delete-loader", + "api-reference/methods/delete-conversation", "api-reference/methods/reset" ] } diff --git a/examples/libsql/eslint.config.js b/examples/libsql/eslint.config.js new file mode 100644 index 00000000..e2a15f2a --- /dev/null +++ b/examples/libsql/eslint.config.js @@ -0,0 +1,3 @@ +import baseConfig from '../../eslint.config.js'; + +export default [...baseConfig]; diff --git a/examples/libsql/package.json b/examples/libsql/package.json new file mode 100644 index 00000000..988cb76a --- /dev/null +++ b/examples/libsql/package.json @@ -0,0 +1,11 @@ +{ + "name": "@llm-tools/embedjs-examples-libsql", + "version": "0.1.1", + "type": "module", + "dependencies": { + "dotenv": "^16.4.5" + }, + "scripts": { + "start": "nx run examples-libsql:serve" + } +} diff --git a/examples/libsql/project.json b/examples/libsql/project.json new file mode 100644 index 00000000..cc427f4f --- /dev/null +++ b/examples/libsql/project.json @@ -0,0 +1,57 @@ +{ + "name": "examples-libsql", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "examples/libsql/src", + "projectType": "application", + "tags": [], + "targets": { + "build": { + "executor": "@nx/esbuild:esbuild", + "outputs": ["{options.outputPath}"], + "defaultConfiguration": "development", + "options": { + "platform": "node", + "outputPath": "dist/examples/libsql", + "format": ["esm"], + "bundle": true, + "main": "examples/libsql/src/main.ts", + "tsConfig": "examples/libsql/tsconfig.app.json", + "generatePackageJson": false, + "esbuildOptions": { + "sourcemap": true, + "outExtension": { + ".js": ".js" + } + } + }, + "configurations": { + "development": {}, + "production": { + "esbuildOptions": { + "sourcemap": false, + "outExtension": { + ".js": ".js" + } + } + } + } + }, + "serve": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "dependsOn": ["build"], + "options": { + "buildTarget": "examples-libsql:build", + "runBuildTargetDependencies": true + }, + "configurations": { + "development": { + "buildTarget": "examples-libsql:build:development" + }, + "production": { + "buildTarget": "examples-libsql:build:production" + } + } + } + } +} diff --git a/examples/libsql/src/main.ts b/examples/libsql/src/main.ts new file mode 100644 index 00000000..21b046c5 --- /dev/null +++ b/examples/libsql/src/main.ts @@ -0,0 +1,26 @@ +import 'dotenv/config'; +import path from 'node:path'; +import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs'; +import { LibSqlDb, LibSqlStore } from '@llm-tools/embedjs-libsql'; +import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai'; +import { WebLoader } from '@llm-tools/embedjs-loader-web'; + +const databasePath = path.resolve('./examples/libsql/data.db'); +const ragApplication = await new RAGApplicationBuilder() + .setStore(new LibSqlStore({ path: databasePath })) + .setVectorDatabase(new LibSqlDb({ path: databasePath })) + .setEmbeddingModel(new OpenAiEmbeddings()) + .setModel(SIMPLE_MODELS.OPENAI_GPT4_O) + .build(); + +await ragApplication.deleteConversation('default'); +await ragApplication.addLoader(new WebLoader({ urlOrContent: 'https://en.wikipedia.org/wiki/Elon_Musk' })); +console.log(await ragApplication.query('Was Elon Musk the founder of Tesla originally?')); +console.log(await ragApplication.query('What is the net worth of Elon Musk today?')); +console.log('Embeddings count', await ragApplication.getEmbeddingsCount()); + +// await ragApplication.deleteLoader('WebLoader_1eab8dd1ffa92906f7fc839862871ca5'); +// console.log('Embeddings count', await ragApplication.getEmbeddingsCount()); + +// await ragApplication.reset(); +// console.log('Embeddings count', await ragApplication.getEmbeddingsCount()); diff --git a/examples/libsql/tsconfig.app.json b/examples/libsql/tsconfig.app.json new file mode 100644 index 00000000..5a8c35da --- /dev/null +++ b/examples/libsql/tsconfig.app.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "types": ["node"] + }, + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/examples/libsql/tsconfig.json b/examples/libsql/tsconfig.json new file mode 100644 index 00000000..c60cf5c2 --- /dev/null +++ b/examples/libsql/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + } + ], + "compilerOptions": { + "esModuleInterop": true, + "target": "ES2022", + "lib": ["ES2022", "ES2022.Object"], + "module": "NodeNext", + "moduleResolution": "nodenext" + } +} diff --git a/loaders/embedjs-loader-confluence/package.json b/loaders/embedjs-loader-confluence/package.json index b2dc576c..4aec537c 100644 --- a/loaders/embedjs-loader-confluence/package.json +++ b/loaders/embedjs-loader-confluence/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-confluence", - "version": "0.1.21", + "version": "0.1.22", "description": "Confluence loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-loader-web": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-loader-web": "0.1.22", "confluence.js": "^1.7.4", "debug": "^4.3.7", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-csv/package.json b/loaders/embedjs-loader-csv/package.json index f9c62c8f..3165c9a8 100644 --- a/loaders/embedjs-loader-csv/package.json +++ b/loaders/embedjs-loader-csv/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-csv", - "version": "0.1.21", + "version": "0.1.22", "description": "CSV loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "csv-parse": "^5.6.0", "debug": "^4.3.7", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-markdown/package.json b/loaders/embedjs-loader-markdown/package.json index 53246507..d0274355 100644 --- a/loaders/embedjs-loader-markdown/package.json +++ b/loaders/embedjs-loader-markdown/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-markdown", - "version": "0.1.21", + "version": "0.1.22", "description": "XML loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-loader-web": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-loader-web": "0.1.22", "debug": "^4.3.7", "md5": "^2.3.0", "micromark": "^4.0.1", diff --git a/loaders/embedjs-loader-msoffice/package.json b/loaders/embedjs-loader-msoffice/package.json index 3f50d1ff..ddec8455 100644 --- a/loaders/embedjs-loader-msoffice/package.json +++ b/loaders/embedjs-loader-msoffice/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-msoffice", - "version": "0.1.21", + "version": "0.1.22", "description": "Word, PPT and Excel loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" }, diff --git a/loaders/embedjs-loader-pdf/package.json b/loaders/embedjs-loader-pdf/package.json index 504ddb47..75caed91 100644 --- a/loaders/embedjs-loader-pdf/package.json +++ b/loaders/embedjs-loader-pdf/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-pdf", - "version": "0.1.21", + "version": "0.1.22", "description": "PDF loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" }, diff --git a/loaders/embedjs-loader-sitemap/package.json b/loaders/embedjs-loader-sitemap/package.json index 01c21243..dee5181e 100644 --- a/loaders/embedjs-loader-sitemap/package.json +++ b/loaders/embedjs-loader-sitemap/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-loader-sitemap", - "version": "0.1.21", + "version": "0.1.22", "description": "Sitemap recursive loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-loader-web": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-loader-web": "0.1.22", "debug": "^4.3.7", "md5": "^2.3.0", "sitemapper": "^3.2.18" diff --git a/loaders/embedjs-loader-web/package.json b/loaders/embedjs-loader-web/package.json index a1ab72fa..172d14a3 100644 --- a/loaders/embedjs-loader-web/package.json +++ b/loaders/embedjs-loader-web/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-web", - "version": "0.1.21", + "version": "0.1.22", "description": "Web page loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7", "html-to-text": "^9.0.5", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-xml/package.json b/loaders/embedjs-loader-xml/package.json index e51fff39..4a8e9a1d 100644 --- a/loaders/embedjs-loader-xml/package.json +++ b/loaders/embedjs-loader-xml/package.json @@ -1,9 +1,9 @@ { "name": "@llm-tools/embedjs-loader-xml", - "version": "0.1.21", + "version": "0.1.22", "description": "XML loader for embedjs", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "fast-xml-parser": "^4.5.0", "md5": "^2.3.0" diff --git a/loaders/embedjs-loader-youtube/package.json b/loaders/embedjs-loader-youtube/package.json index 6e422406..3c28c8cb 100644 --- a/loaders/embedjs-loader-youtube/package.json +++ b/loaders/embedjs-loader-youtube/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-loader-youtube", - "version": "0.1.21", + "version": "0.1.22", "description": "Youtube transcript and channel recursive loader for embedjs", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7", "md5": "^2.3.0", "usetube": "^2.2.7", diff --git a/models/embedjs-anthropic/package.json b/models/embedjs-anthropic/package.json index 58a7f571..98b8f06c 100644 --- a/models/embedjs-anthropic/package.json +++ b/models/embedjs-anthropic/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-anthropic", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of Anthropic models with embedjs", "dependencies": { "@langchain/anthropic": "^0.3.8", "@langchain/core": "^0.3.19", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-cohere/package.json b/models/embedjs-cohere/package.json index f40fe3a0..8fc7f680 100644 --- a/models/embedjs-cohere/package.json +++ b/models/embedjs-cohere/package.json @@ -1,10 +1,10 @@ { "name": "@llm-tools/embedjs-cohere", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of Cohere models with embedjs", "dependencies": { "@langchain/cohere": "^0.3.1", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "cohere-ai": "^7.14.0" }, "type": "module", diff --git a/models/embedjs-huggingface/package.json b/models/embedjs-huggingface/package.json index 2a3584bb..744cdf42 100644 --- a/models/embedjs-huggingface/package.json +++ b/models/embedjs-huggingface/package.json @@ -1,12 +1,12 @@ { "name": "@llm-tools/embedjs-huggingface", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of HuggingFace models with embedjs", "dependencies": { "@huggingface/inference": "^2.8.1", "@langchain/community": "^0.3.16", "@langchain/core": "^0.3.19", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-llama-cpp/package.json b/models/embedjs-llama-cpp/package.json index cd4f9dfc..5da109b5 100644 --- a/models/embedjs-llama-cpp/package.json +++ b/models/embedjs-llama-cpp/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-llama-cpp", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of Node-Llama-Cpp with embedjs", "dependencies": { "@langchain/community": "^0.3.16", "@langchain/core": "^0.3.19", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "node-llama-cpp": "^3.2.0" }, diff --git a/models/embedjs-mistral/package.json b/models/embedjs-mistral/package.json index 5ebc1dbf..5889520d 100644 --- a/models/embedjs-mistral/package.json +++ b/models/embedjs-mistral/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-mistral", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of Mistral models with embedjs", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/mistralai": "^0.2.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-ollama/package.json b/models/embedjs-ollama/package.json index 62dc2c65..f90ea8d1 100644 --- a/models/embedjs-ollama/package.json +++ b/models/embedjs-ollama/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-ollama", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of Ollama with embedjs", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/ollama": "^0.1.2", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-openai/package.json b/models/embedjs-openai/package.json index 7f8a6f40..bbd990ae 100644 --- a/models/embedjs-openai/package.json +++ b/models/embedjs-openai/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-openai", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of OpenAI models with embedjs", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/openai": "^0.3.14", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/models/embedjs-vertexai/package.json b/models/embedjs-vertexai/package.json index 22bc2d15..2f2d53e3 100644 --- a/models/embedjs-vertexai/package.json +++ b/models/embedjs-vertexai/package.json @@ -1,11 +1,11 @@ { "name": "@llm-tools/embedjs-vertexai", - "version": "0.1.21", + "version": "0.1.22", "description": "Enable usage of VertexAI models with embedjs", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/google-vertexai": "^0.1.3", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" }, "type": "module", diff --git a/package-lock.json b/package-lock.json index 5d16c568..f7cbb730 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,12 +48,12 @@ }, "core/embedjs": { "name": "@llm-tools/embedjs", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7", "langchain": "^0.3.6", "md5": "^2.3.0", @@ -68,7 +68,7 @@ }, "core/embedjs-interfaces": { "name": "@llm-tools/embedjs-interfaces", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.19", @@ -91,10 +91,10 @@ }, "core/embedjs-utils": { "name": "@llm-tools/embedjs-utils", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21" + "@llm-tools/embedjs-interfaces": "0.1.22" } }, "core/embedjs/node_modules/langchain": { @@ -189,91 +189,92 @@ }, "databases/embedjs-astra": { "name": "@llm-tools/embedjs-astradb", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@datastax/astra-db-ts": "^1.5.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, "databases/embedjs-cosmos": { "name": "@llm-tools/embedjs-cosmos", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@azure/cosmos": "^4.2.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, "databases/embedjs-hnswlib": { "name": "@llm-tools/embedjs-hnswlib", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "hnswlib-node": "^3.0.0" } }, "databases/embedjs-lancedb": { "name": "@llm-tools/embedjs-lancedb", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@lancedb/lancedb": "^0.13.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7" } }, "databases/embedjs-libsql": { "name": "@llm-tools/embedjs-libsql", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@libsql/client": "^0.14.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7" } }, "databases/embedjs-lmdb": { "name": "@llm-tools/embedjs-lmdb", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "lmdb": "^3.1.6" } }, "databases/embedjs-mongodb": { "name": "@llm-tools/embedjs-mongodb", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "mongodb": "^6.11.0" } }, "databases/embedjs-pinecone": { "name": "@llm-tools/embedjs-pinecone", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "@pinecone-database/pinecone": "^4.0.0", "debug": "^4.3.7" } }, "databases/embedjs-qdrant": { "name": "@llm-tools/embedjs-qdrant", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "@qdrant/js-client-rest": "^1.12.0", "debug": "^4.3.7", "uuid": "^11.0.3" @@ -293,19 +294,19 @@ }, "databases/embedjs-redis": { "name": "@llm-tools/embedjs-redis", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "ioredis": "^5.4.1" } }, "databases/embedjs-weaviate": { "name": "@llm-tools/embedjs-weaviate", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "compute-cosine-similarity": "^1.1.0", "debug": "^4.3.7", "weaviate-ts-client": "^2.2.0" @@ -313,11 +314,11 @@ }, "loaders/embedjs-loader-confluence": { "name": "@llm-tools/embedjs-loader-confluence", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-loader-web": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-loader-web": "0.1.22", "confluence.js": "^1.7.4", "debug": "^4.3.7", "md5": "^2.3.0" @@ -325,11 +326,11 @@ }, "loaders/embedjs-loader-csv": { "name": "@llm-tools/embedjs-loader-csv", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "csv-parse": "^5.6.0", "debug": "^4.3.7", "md5": "^2.3.0" @@ -337,11 +338,11 @@ }, "loaders/embedjs-loader-markdown": { "name": "@llm-tools/embedjs-loader-markdown", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-loader-web": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-loader-web": "0.1.22", "debug": "^4.3.7", "md5": "^2.3.0", "micromark": "^4.0.1", @@ -351,35 +352,35 @@ }, "loaders/embedjs-loader-msoffice": { "name": "@llm-tools/embedjs-loader-msoffice", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" } }, "loaders/embedjs-loader-pdf": { "name": "@llm-tools/embedjs-loader-pdf", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "md5": "^2.3.0", "office-text-extractor": "^3.0.3" } }, "loaders/embedjs-loader-sitemap": { "name": "@llm-tools/embedjs-loader-sitemap", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-loader-web": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-loader-web": "0.1.22", "debug": "^4.3.7", "md5": "^2.3.0", "sitemapper": "^3.2.18" @@ -387,12 +388,12 @@ }, "loaders/embedjs-loader-web": { "name": "@llm-tools/embedjs-loader-web", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7", "html-to-text": "^9.0.5", "md5": "^2.3.0" @@ -403,10 +404,10 @@ }, "loaders/embedjs-loader-xml": { "name": "@llm-tools/embedjs-loader-xml", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "fast-xml-parser": "^4.5.0", "md5": "^2.3.0" @@ -414,12 +415,12 @@ }, "loaders/embedjs-loader-youtube": { "name": "@llm-tools/embedjs-loader-youtube", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/textsplitters": "^0.1.0", - "@llm-tools/embedjs-interfaces": "0.1.21", - "@llm-tools/embedjs-utils": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", + "@llm-tools/embedjs-utils": "0.1.22", "debug": "^4.3.7", "md5": "^2.3.0", "usetube": "^2.2.7", @@ -431,34 +432,34 @@ }, "models/embedjs-anthropic": { "name": "@llm-tools/embedjs-anthropic", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/anthropic": "^0.3.8", "@langchain/core": "^0.3.19", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, "models/embedjs-cohere": { "name": "@llm-tools/embedjs-cohere", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/cohere": "^0.3.1", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "cohere-ai": "^7.14.0" } }, "models/embedjs-huggingface": { "name": "@llm-tools/embedjs-huggingface", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@huggingface/inference": "^2.8.1", "@langchain/community": "^0.3.16", "@langchain/core": "^0.3.19", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, @@ -973,12 +974,12 @@ }, "models/embedjs-llama-cpp": { "name": "@llm-tools/embedjs-llama-cpp", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/community": "^0.3.16", "@langchain/core": "^0.3.19", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7", "node-llama-cpp": "^3.2.0" } @@ -1494,45 +1495,45 @@ }, "models/embedjs-mistral": { "name": "@llm-tools/embedjs-mistral", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/mistralai": "^0.2.0", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, "models/embedjs-ollama": { "name": "@llm-tools/embedjs-ollama", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/ollama": "^0.1.2", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, "models/embedjs-openai": { "name": "@llm-tools/embedjs-openai", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/openai": "^0.3.14", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, "models/embedjs-vertexai": { "name": "@llm-tools/embedjs-vertexai", - "version": "0.1.21", + "version": "0.1.22", "license": "Apache-2.0", "dependencies": { "@langchain/core": "^0.3.19", "@langchain/google-vertexai": "^0.1.3", - "@llm-tools/embedjs-interfaces": "0.1.21", + "@llm-tools/embedjs-interfaces": "0.1.22", "debug": "^4.3.7" } }, diff --git a/scripts/publish-via-nx.js b/scripts/publish-via-nx.js index 764fdf25..273f604f 100644 --- a/scripts/publish-via-nx.js +++ b/scripts/publish-via-nx.js @@ -61,7 +61,7 @@ async function updatePackageVersion(pkgName, version, versionMap, dryRun) { async function createRelease(dryRun, version, makeGitCommit) { console.log('Running nx release'); - const { /*workspaceVersion,*/ projectsVersionData } = await releaseVersion({ + const { workspaceVersion, projectsVersionData } = await releaseVersion({ gitTag: false, gitCommit: false, specifier: version,