From deb269415725b81bac97f891c8b779f91532fec9 Mon Sep 17 00:00:00 2001 From: Adhityan K V Date: Fri, 26 Apr 2024 05:55:36 +0200 Subject: [PATCH] Support for Anthropic added --- README.md | 69 ++++++++++++------ package-lock.json | 120 +++++++++++++++++++++++++++----- package.json | 12 ++-- src/core/rag-application.ts | 2 +- src/index.ts | 2 + src/models/anthropic-model.ts | 53 ++++++++++++++ src/models/huggingface-model.ts | 3 +- src/models/mistral-model.ts | 3 +- src/models/openai-model.ts | 5 +- 9 files changed, 219 insertions(+), 50 deletions(-) create mode 100644 src/models/anthropic-model.ts diff --git a/README.md b/README.md index 4a6db539..063eb255 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,10 @@ The author(s) are looking to add core maintainers for this opensource project. R - [How to request more loaders](#more-loaders-coming-soon) - [LLMs](#llms) - [OpenAI](#openai) + - [Azure OpenAI](#azure-openai) - [Mistral](#mistral) - [Hugging Face](#hugging-face) - - [Azure OpenAI](#azure-openai) + - [Anthropic](#anthropic) - [Bring your own LLMs](#use-custom-llm-model) - [Request support for new LLMs](#more-llms-coming-soon) - [Embedding Models](#embedding-models) @@ -358,6 +359,36 @@ const ragApplication = await new RAGApplicationBuilder() **Note:** GPT 3.5 Turbo is used as the default model if you do not specifiy one. +## Azure OpenAI + +In order to be able to use an OpenAI model on Azure, it first needs to be deployed. Please refer to [Azure OpenAI documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/) on how to deploy a model on Azure. To run this library, you will need to deploy two models - + +- text-embedding-ada +- GPT-3.5-turbo (or the 4 series) + +Once these models are deployed, using Azure OpenAI instead of the regular OpenAI is easy to do. Just follow these steps - + +- Remove the `OPENAI_API_KEY` environment variable if you have set it already. + +- Set the following environment variables - + +```bash +# Set this to `azure` +OPENAI_API_TYPE=azure +# The API version you want to use +AZURE_OPENAI_API_VERSION=2023-05-15 +# The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. +export AZURE_OPENAI_BASE_PATH=https://your-resource-name.openai.azure.com/openai/deployments +# The API key1 or key2 for your Azure OpenAI resource +export AZURE_OPENAI_API_KEY= +# The deployment name you used for your embedding model +AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=text-embedding-ada-002 +# The deployment name you used for your llm +AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-35-turbo +``` + +You can all set and can now run the Azure OpenAI LLMs using the [`OpenAi` model](#openai) steps detailed above. + ## Mistral To use Mirstal's models, you will need to get an API Key from Mistral. You can do this from their [console](https://console.mistral.ai/user/api-keys/). Once you have obtained a key, set Mistral as your LLM of choice - @@ -397,35 +428,29 @@ const ragApplication = await new RAGApplicationBuilder() To use these 'not-free' models via HuggingFace, you need to subscribe to their [Pro plan](https://huggingface.co/pricing) or create a custom [inference endpoint](https://ui.endpoints.huggingface.co/). It is possible to self host these models for free and run them locally via Ollama - support for which is coming soon. -## Azure OpenAI +## Anthropic -In order to be able to use an OpenAI model on Azure, it first needs to be deployed. Please refer to [Azure OpenAI documentation](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/) on how to deploy a model on Azure. To run this library, you will need to deploy two models - +To use Anthropic's Claude models, you will need to get an API Key from Anthropic. You can do this from their [console](https://console.anthropic.com/settings/keys). Once you obtain a key, set it in the environment variable, like so - -- text-embedding-ada -- GPT-3.5-turbo (or the 4 series) +```bash +ANTHROPIC_API_KEY="" +``` -Once these models are deployed, using Azure OpenAI instead of the regular OpenAI is easy to do. Just follow these steps - +Once this is done, it is relatively easy to use Anthropic's Claude in your RAG application. Simply set Anthropic as your LLM of choice - -- Remove the `OPENAI_API_KEY` environment variable if you have set it already. +```TS +const ragApplication = await new RAGApplicationBuilder() +.setModel(new Anthropic()) +``` -- Set the following environment variables - +By default, the `claude-3-sonnet-20240229` model from Anthropic is used. If you want to use a different Anthropic model, you can specify it via the optional parameter to the Anthropic constructor, like so - -```bash -# Set this to `azure` -OPENAI_API_TYPE=azure -# The API version you want to use -AZURE_OPENAI_API_VERSION=2023-05-15 -# The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource. -export AZURE_OPENAI_BASE_PATH=https://your-resource-name.openai.azure.com/openai/deployments -# The API key1 or key2 for your Azure OpenAI resource -export AZURE_OPENAI_API_KEY= -# The deployment name you used for your embedding model -AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME=text-embedding-ada-002 -# The deployment name you used for your llm -AZURE_OPENAI_API_DEPLOYMENT_NAME=gpt-35-turbo +```TS +const ragApplication = await new RAGApplicationBuilder() +.setModel(new Anthropic({ modelName: "..." })) ``` -You can now run the Azure OpenAI LLMs using the [`OpenAi` model](#openai) detailed above. +You can read more about the various models provided by Anthropic [here](https://docs.anthropic.com/claude/docs/models-overview). ## Use custom LLM model diff --git a/package-lock.json b/package-lock.json index a765bd34..1b20d8b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,25 +1,26 @@ { "name": "@llm-tools/embedjs", - "version": "0.0.68", + "version": "0.0.71", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@llm-tools/embedjs", - "version": "0.0.68", + "version": "0.0.71", "license": "Apache-2.0", "dependencies": { "@huggingface/inference": "^2.6.7", + "@langchain/anthropic": "^0.1.16", "@langchain/cohere": "^0.0.8", - "@langchain/community": "^0.0.51", - "@langchain/core": "^0.1.59", + "@langchain/community": "^0.0.52", + "@langchain/core": "^0.1.60", "@langchain/mistralai": "^0.0.19", "@langchain/openai": "^0.0.28", "axios": "^1.6.8", "confluence.js": "^1.7.4", "debug": "^4.3.4", "html-to-text": "^9.0.5", - "langchain": "^0.1.35", + "langchain": "^0.1.36", "md5": "^2.3.0", "pdf-parse-fork": "^1.2.0", "sitemapper": "^3.1.8", @@ -441,6 +442,44 @@ "win32" ] }, + "node_modules/@langchain/anthropic": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.1.16.tgz", + "integrity": "sha512-vCbwkZ3pkMSKf67fBgNlslvuW9f3EZGBbO8Ic2etgX3xFl6L0WuMtfS26P1FCDpRwaKuC1BrCj2aLAeMzMq/Fg==", + "dependencies": { + "@anthropic-ai/sdk": "^0.20.1", + "@langchain/core": "~0.1.56", + "fast-xml-parser": "^4.3.5", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/anthropic/node_modules/@anthropic-ai/sdk": { + "version": "0.20.7", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.20.7.tgz", + "integrity": "sha512-uyc+3WGLpe8ur6mSIKSab7P9JdBerTdmqb7popc/yROYLLCW/Ykyw4ZfjmN/cLmxjnAKnv5YUngzbPM0BJuGjg==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + } + }, + "node_modules/@langchain/anthropic/node_modules/@types/node": { + "version": "18.19.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.31.tgz", + "integrity": "sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/@langchain/cohere": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/@langchain/cohere/-/cohere-0.0.8.tgz", @@ -454,11 +493,11 @@ } }, "node_modules/@langchain/community": { - "version": "0.0.51", - "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.0.51.tgz", - "integrity": "sha512-TInG0nDCGIwg2iwHJkv1OYGVR2fDMYdpKRuuLlQoevvcHcCRIVlQS5d0MnAx0bWGUbVdkKpshrH8Ltpxt6fewA==", + "version": "0.0.52", + "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.0.52.tgz", + "integrity": "sha512-L+IMAAaLNP7++4HhdvuVJegc8bdw8WP77Jvp98YcySFZTZWH1yasSQSlFn3jgBk+3xLBsudpTZuttKTrZ/TtVQ==", "dependencies": { - "@langchain/core": "~0.1.56", + "@langchain/core": "~0.1.60", "@langchain/openai": "~0.0.28", "expr-eval": "^2.0.2", "flat": "^5.0.2", @@ -844,9 +883,9 @@ } }, "node_modules/@langchain/core": { - "version": "0.1.59", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.1.59.tgz", - "integrity": "sha512-cdLi/hH/gj1CIWVcwkOxM+QYpz5kYIBxpuRkoye3OwHry9Qa9djwlsGxpUmrv5pIbPkYd7GK9L+xTQqyYIAp6A==", + "version": "0.1.60", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.1.60.tgz", + "integrity": "sha512-3EJW4ir0tFe17AakpXCgO9flSoDjFELpSQs2w/CMZ5FBlHYxo3ODgVQAZvlHy97khEVgcnvlL3EDhPE7IdNibA==", "dependencies": { "ansi-styles": "^5.0.0", "camelcase": "6", @@ -895,6 +934,18 @@ "node": ">=18" } }, + "node_modules/@langchain/textsplitters": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.0.0.tgz", + "integrity": "sha512-3hPesWomnmVeYMppEGYbyv0v/sRUugUdlFBNn9m1ueJYHAIKbvCErkWxNUH3guyKKYgJVrkvZoQxcd9faucSaw==", + "dependencies": { + "@langchain/core": "~0.1", + "js-tiktoken": "^1.0.11" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@lmdb/lmdb-darwin-arm64": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-3.0.6.tgz", @@ -2909,6 +2960,27 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.6.tgz", + "integrity": "sha512-M2SovcRxD4+vC493Uc2GZVcZaj66CCJhWurC4viynVSTvrpErCShNcDz1lAho6n9REQKvL/ll4A4/fw6Y9z8nw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -3713,9 +3785,9 @@ "integrity": "sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ==" }, "node_modules/js-tiktoken": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.8.tgz", - "integrity": "sha512-r7XK3E9/I+SOrbAGqb39pyO/rHAS1diAOSRAvaaLfHgXjkUSK9AiSd+r84Vn2f/GvXJYRAxKj8NHrUvqlaH5qg==", + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.11.tgz", + "integrity": "sha512-PajXFLq2vx7/8jllQZ43vzNpAai/0MOVdJjW/UrNyJorNQRTjHrqdGJG/mjHVy7h9M6dW6CaG43eNLMYFkTh6w==", "dependencies": { "base64-js": "^1.5.1" } @@ -3783,14 +3855,15 @@ } }, "node_modules/langchain": { - "version": "0.1.35", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.1.35.tgz", - "integrity": "sha512-wTaTNzaAPb/umcoWxFupc0QGsEJvPB6pRE8Ecy8v4CCsn/a5O6QpfRylvJKXXWoxGG8BI4L3G+HjsXCDUNxVrQ==", + "version": "0.1.36", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.1.36.tgz", + "integrity": "sha512-NTbnCL/jKWIeEI//Nm1oG8nhW3vkYWvEMr1MPotmTThTfeKfO87eV/OAzAyh6Ruy6GFs/qofRgQZGIe6XvXTNQ==", "dependencies": { "@anthropic-ai/sdk": "^0.9.1", "@langchain/community": "~0.0.47", - "@langchain/core": "~0.1.56", + "@langchain/core": "~0.1.60", "@langchain/openai": "~0.0.28", + "@langchain/textsplitters": "~0.0.0", "binary-extensions": "^2.2.0", "js-tiktoken": "^1.0.7", "js-yaml": "^4.1.0", @@ -3819,6 +3892,7 @@ "@gomomento/sdk-web": "^1.51.1", "@google-ai/generativelanguage": "^0.2.1", "@google-cloud/storage": "^6.10.1 || ^7.7.0", + "@mendable/firecrawl-js": "^0.0.13", "@notionhq/client": "^2.2.10", "@pinecone-database/pinecone": "*", "@supabase/supabase-js": "^2.10.0", @@ -3891,6 +3965,9 @@ "@google-cloud/storage": { "optional": true }, + "@mendable/firecrawl-js": { + "optional": true + }, "@notionhq/client": { "optional": true }, @@ -5566,6 +5643,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/package.json b/package.json index 9fdf8336..f768f9db 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@llm-tools/embedjs", - "version": "0.0.68", + "version": "0.0.71", "description": "A NodeJS RAG framework to easily work with LLMs and custom datasets", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -24,6 +24,9 @@ "llm", "gpt", "openai", + "anthropic", + "claude", + "qdrant", "chatgpt", "hugging-face", "mistral", @@ -49,16 +52,17 @@ "homepage": "https://github.com/llm-tools/embedjs#readme", "dependencies": { "@huggingface/inference": "^2.6.7", + "@langchain/anthropic": "^0.1.16", "@langchain/cohere": "^0.0.8", - "@langchain/community": "^0.0.51", - "@langchain/core": "^0.1.59", + "@langchain/community": "^0.0.52", + "@langchain/core": "^0.1.60", "@langchain/mistralai": "^0.0.19", "@langchain/openai": "^0.0.28", "axios": "^1.6.8", "confluence.js": "^1.7.4", "debug": "^4.3.4", "html-to-text": "^9.0.5", - "langchain": "^0.1.35", + "langchain": "^0.1.36", "md5": "^2.3.0", "pdf-parse-fork": "^1.2.0", "sitemapper": "^3.1.8", diff --git a/src/core/rag-application.ts b/src/core/rag-application.ts index 0d54c254..2dbd8a35 100644 --- a/src/core/rag-application.ts +++ b/src/core/rag-application.ts @@ -67,7 +67,7 @@ export class RAGApplication { await this.addLoader(loader); } } - this.debug('Initial loaders added'); + this.debug('Initialized pre-loaders'); } private async batchLoadEmbeddings(loaderUniqueId: string, formattedChunks: Chunk[]) { diff --git a/src/index.ts b/src/index.ts index 92e380fb..dfa92643 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ import { OpenAi3LargeEmbeddings } from './embeddings/openai-3large-embeddings.js import { OpenAi3SmallEmbeddings } from './embeddings/openai-3small-embeddings.js'; import { Mistral } from './models/mistral-model.js'; import { HuggingFace } from './models/huggingface-model.js'; +import { Anthropic } from './models/anthropic-model.js'; export { RAGApplication, @@ -48,4 +49,5 @@ export { OpenAi3SmallEmbeddings, Mistral, HuggingFace, + Anthropic, }; diff --git a/src/models/anthropic-model.ts b/src/models/anthropic-model.ts new file mode 100644 index 00000000..5dd0ae2f --- /dev/null +++ b/src/models/anthropic-model.ts @@ -0,0 +1,53 @@ +import createDebugMessages from 'debug'; +import { ChatAnthropic } from '@langchain/anthropic'; +import { HumanMessage, AIMessage, SystemMessage } from '@langchain/core/messages'; + +import { BaseModel } from '../interfaces/base-model.js'; +import { Chunk, ConversationHistory } from '../global/types.js'; + +export class Anthropic extends BaseModel { + private readonly debug = createDebugMessages('embedjs:model:Anthropic'); + private readonly modelName: string; + private model: ChatAnthropic; + + constructor(params?: { temperature?: number; modelName?: string }) { + super(params?.temperature); + this.modelName = params?.modelName ?? 'claude-3-sonnet-20240229'; + } + + override async init(): Promise { + this.model = new ChatAnthropic({ temperature: this.temperature, model: this.modelName }); + } + + override async runQuery( + system: string, + userQuery: string, + supportingContext: Chunk[], + pastConversations: ConversationHistory[], + ): Promise { + const pastMessages: (AIMessage | SystemMessage | HumanMessage)[] = [ + new SystemMessage( + `${system}. Supporting context: ${supportingContext.map((s) => s.pageContent).join('; ')}`, + ), + ]; + + pastMessages.push.apply( + pastConversations.map((c) => { + if (c.sender === 'AI') + return new AIMessage({ + content: c.message, + }); + + return new HumanMessage({ + content: c.message, + }); + }), + ); + pastMessages.push(new HumanMessage(`${userQuery}?`)); + + this.debug('Executing anthropic model with prompt -', userQuery); + const result = await this.model.invoke(pastMessages); + this.debug('Anthropic response -', result); + return result.content.toString(); + } +} diff --git a/src/models/huggingface-model.ts b/src/models/huggingface-model.ts index 4b2bb1c8..f6b6f6b6 100644 --- a/src/models/huggingface-model.ts +++ b/src/models/huggingface-model.ts @@ -53,7 +53,8 @@ export class HuggingFace extends BaseModel { const finalPrompt = pastMessages.join('\n'); // this.debug('Final prompt being sent to HF - ', finalPrompt); this.debug(`Executing hugging face '${this.model.model}' model with prompt -`, userQuery); - const result = await this.model.invoke(finalPrompt, {}); + const result = await this.model.invoke(finalPrompt); + this.debug('Hugging response -', result); return result; } } diff --git a/src/models/mistral-model.ts b/src/models/mistral-model.ts index 54318c97..9658ebdb 100644 --- a/src/models/mistral-model.ts +++ b/src/models/mistral-model.ts @@ -19,7 +19,7 @@ export class Mistral extends BaseModel { modelName?: string; }) { super(temperature); - this.model = new ChatMistralAI({ apiKey: accessToken, modelName: modelName ?? 'mistral-medium' }); + this.model = new ChatMistralAI({ apiKey: accessToken, model: modelName ?? 'mistral-medium' }); } override async runQuery( @@ -49,6 +49,7 @@ export class Mistral extends BaseModel { this.debug('Executing mistral model with prompt -', userQuery); const result = await this.model.invoke(pastMessages); + this.debug('Mistral response -', result); return result.content.toString(); } } diff --git a/src/models/openai-model.ts b/src/models/openai-model.ts index 5e9702c4..1e7d5200 100644 --- a/src/models/openai-model.ts +++ b/src/models/openai-model.ts @@ -16,7 +16,7 @@ export class OpenAi extends BaseModel { } override async init(): Promise { - this.model = new ChatOpenAI({ temperature: this.temperature, modelName: this.modelName }); + this.model = new ChatOpenAI({ temperature: this.temperature, model: this.modelName }); } override async runQuery( @@ -45,7 +45,8 @@ export class OpenAi extends BaseModel { pastMessages.push(new HumanMessage(`${userQuery}?`)); this.debug('Executing openai model with prompt -', userQuery); - const result = await this.model.invoke(pastMessages, {}); + const result = await this.model.invoke(pastMessages); + this.debug('OpenAI response -', result); return result.content.toString(); } }