From 8e2392af2badf960cf3baf06ded589353a638a7f Mon Sep 17 00:00:00 2001 From: Brandon Ruffridge Date: Thu, 21 Nov 2024 12:15:39 -0500 Subject: [PATCH 1/2] added support for azure openai assistants api. fixed thread auto naming uncaught exception if first user message is an image upload. added back support for vision, and code_interpreter based on the upload file's type. fixed padding on loading message. Removed submit button loading animation. fixed getActiveThread validAssistant call to use the full assistant name with version. --- package-lock.json | 8 +- package.json | 2 +- src/assistant/bidara.js | 1 + src/components/AssistantDeepChat.svelte | 76 ++++++++--- src/utils/openaiUtils.js | 160 ++++++++++++++---------- src/utils/threadUtils.js | 15 ++- 6 files changed, 173 insertions(+), 89 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5c92218..b7fe1ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "bidara-deep-chat-svelte", "version": "1.0.0", "dependencies": { - "deep-chat-dev": "^9.0.179", + "deep-chat-dev": "^9.0.202", "jspdf": "^2.5.1", "sirv-cli": "^2.0.0", "svelte-agnostic-draggable": "^0.2.0", @@ -1135,9 +1135,9 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" }, "node_modules/deep-chat-dev": { - "version": "9.0.179", - "resolved": "https://registry.npmjs.org/deep-chat-dev/-/deep-chat-dev-9.0.179.tgz", - "integrity": "sha512-IlQJloUFgpdi4zGa0ah8wv/9nalfPu9gV/4azrkSqtDwo4blwrOs+EfMtHpCFkjRGz77ZTuJXbg2S9xCBjwZxg==", + "version": "9.0.202", + "resolved": "https://registry.npmjs.org/deep-chat-dev/-/deep-chat-dev-9.0.202.tgz", + "integrity": "sha512-1SFTSkllIeCuprAGz0YbbJq9fW6MC+Y9MIjDzrpIlH1YdFbTV1tQIuOKlny+opsYf2fQsXPljevWsEghva3yog==", "dependencies": { "@microsoft/fetch-event-source": "^2.0.1", "remarkable": "^2.0.1", diff --git a/package.json b/package.json index 8c8f5e8..d980dd6 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "svelte": "^4.2.18" }, "dependencies": { - "deep-chat-dev": "^9.0.179", + "deep-chat-dev": "^9.0.202", "jspdf": "^2.5.1", "sirv-cli": "^2.0.0", "svelte-agnostic-draggable": "^0.2.0", diff --git a/src/assistant/bidara.js b/src/assistant/bidara.js index 7dce7ac..c4298a7 100644 --- a/src/assistant/bidara.js +++ b/src/assistant/bidara.js @@ -18,6 +18,7 @@ export const ADVISORY = "**Do not share any sensitive information** in your conv export const GREETING = "How can I assist you today?"; export const MODEL = "gpt-4o-2024-08-06"; +//export const MODEL = "gpt-4o"; // Azure Model Deployment Name const now = new Date(); const formattedDate = now.toLocaleDateString(); diff --git a/src/components/AssistantDeepChat.svelte b/src/components/AssistantDeepChat.svelte index da491b4..154eb30 100644 --- a/src/components/AssistantDeepChat.svelte +++ b/src/components/AssistantDeepChat.svelte @@ -66,7 +66,15 @@ if (thread.length === 0 || thread.length === asst.history + 1) { const maxCharLen = 50; - const words = message.message.text.split(/\s+/); + let words = message.message.text?.split(/\s+/); + if(!words) { + // first user message is a file upload. + words = message.message.files?.[0]?.name?.split(/\s+/); + } + if(!words) { + // first user message is an image upload. + words = message.message.files?.[0]?.ref?.name?.split(/\s+/); + } let name = ""; @@ -127,6 +135,20 @@ imagesToProcess.push(imageFile); } + function fileTools(fileNames) { + let fileTool = "code_interpreter"; + if (Array.isArray(fileNames) && fileNames.length > 0) { + let fileType = threadUtils.getFileTypeByName(fileNames[0]); + if (fileType == "image") { + fileTool = "images"; + } + else if (["txt","powerpoint","pdf","word"].indexOf(fileType) !== -1) { + fileTool = "file_search"; + } + } + return fileTool; + } + async function handleFuncCalling(functionDetails) { let context = { lastMessageId, @@ -196,9 +218,16 @@ async function requestInterceptor(request) { if (newFileUploads.length > 0) { - newFileIds = request.body.attachments.map(attachment => attachment.file_id); - - handleFileUploads(newFileIds, newFileUploads); + if(request.body.attachments) { + newFileIds = request.body.attachments.map(attachment => attachment.file_id); + handleFileUploads(newFileIds, newFileUploads); + } + else if(Array.isArray(request.body?.content) && request.body?.content.length > 0) { + //image uploaded + newFileIds = request.body.content.map(attachment => attachment.image_file?.file_id); + newFileIds = newFileIds.filter(item => item !== undefined); // an undefined array entry is created if an image is uploaded alongside a text message. This line removes that undefined array entry. + handleFileUploads(newFileIds, newFileUploads); + } } return request; @@ -232,17 +261,19 @@ ' } + }, + loading: { + container: { + default: { + backgroundColor: "var(--chat-background-color)", + } + }, + svg: { + content: '' + } } }} auxiliaryStyle={` diff --git a/src/utils/openaiUtils.js b/src/utils/openaiUtils.js index bbb6ced..15c255c 100644 --- a/src/utils/openaiUtils.js +++ b/src/utils/openaiUtils.js @@ -3,7 +3,71 @@ import { getActiveThread, getFileByFileId } from "./threadUtils"; import { ASSISTANT_OPTIONS } from "../assistant"; import { ENV } from "process.env"; +// todo: test all functions with fetch calls to make sure they still work after the header refactor. (test with public openai, not azure.) +/* +createAssistant +updateAssistant +getFileContent +getFileInfo +uploadFile - works +getChatCompletion - works +getAssistantId - works +getNewThreadId - works +getThreadMessages - works +cancelThreadRun - works +validThread - works +validAssistant - works +validAPIKey -works +getDalleImageGeneration - works +*/ + let openaiKey = null; +let apiEndpoint = 'https://api.openai.com/v1'; +let apiVersion = ''; +let apiVersionMultipleParams = ''; + +let azureOpenAI = false; + +function getAPIHeaders(type = 'default') { + let retHeader = ''; + + if (azureOpenAI) { + if (type == 'default' || type == 'wooaib') { + retHeader = { + 'api-key': openaiKey, + 'Content-Type': 'application/json' + }; + } + else if (type == 'woct') { + retHeader = { + 'api-key': openaiKey + }; + } + } + else { + // OpenAI Headers + if (type == 'default') { + retHeader = { + 'Authorization': 'Bearer ' + openaiKey, + 'Content-Type': 'application/json', + 'OpenAI-Beta': 'assistants=v2' + }; + } + else if (type == 'woct') { + retHeader = { + 'Authorization': 'Bearer ' + openaiKey, + 'OpenAI-Beta': 'assistants=v2' + }; + } + else if (type == 'wooaib') { + retHeader = { + 'Authorization': 'Bearer ' + openaiKey, + 'Content-Type': 'application/json' + }; + } + } + return retHeader; +} function getAssistantConfigFromName(asstName) { const asst = ASSISTANT_OPTIONS.find((opt) => opt.name === asstName); @@ -23,13 +87,9 @@ export async function validAssistant(id, asstName) { if (!openaiKey) { throw new Error('openai key not set. cannot validate assistant.'); } - const response = await fetch("https://api.openai.com/v1/assistants/" + id, { + const response = await fetch(apiEndpoint + "/assistants/" + id + apiVersion, { method: "GET", - headers: { - Authorization: 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }, + headers: getAPIHeaders(), body: null }); @@ -50,13 +110,9 @@ export async function updateAssistant(id, config) { if (!openaiKey) { throw new Error('openai key not set. cannot update assistant.'); } - const response = await fetch("https://api.openai.com/v1/assistants/" + id, { + const response = await fetch(apiEndpoint + "/assistants/" + id + apiVersion, { method: "POST", - headers: { - Authorization: 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }, + headers: getAPIHeaders(), body: JSON.stringify(config) }); @@ -72,13 +128,9 @@ export async function createAssistant(config) { if (!openaiKey) { throw new Error('openai key not set. cannot update assistant.'); } - const response = await fetch("https://api.openai.com/v1/assistants", { + const response = await fetch(apiEndpoint + "/assistants" + apiVersion, { method: "POST", - headers: { - Authorization: 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }, + headers: getAPIHeaders(), body: JSON.stringify(config) }); @@ -94,13 +146,9 @@ export async function getAssistantId(asstName, asstVersion, asstConfig) { throw new Error('openai key not set. cannot search for bidara assistant.'); } // get assistants - const response = await fetch("https://api.openai.com/v1/assistants?limit=50", { + const response = await fetch(apiEndpoint + "/assistants?limit=50" + apiVersionMultipleParams, { method: "GET", - headers: { - Authorization: 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }, + headers: getAPIHeaders(), body: null }); @@ -132,9 +180,10 @@ export async function getAssistantId(asstName, asstVersion, asstConfig) { } export async function validApiKey(key) { - const response = await fetch("https://api.openai.com/v1/models", { + openaiKey = key; + const response = await fetch(apiEndpoint + "/models", { method: "GET", - headers: { Authorization: 'Bearer ' + key, 'Content-Type': 'application/json' }, + headers: getAPIHeaders(), body: null }); @@ -223,13 +272,9 @@ export async function validThread(thread_id) { } try { - const response = await fetch(`https://api.openai.com/v1/threads/${thread_id}`, { + const response = await fetch(apiEndpoint + `/threads/${thread_id}` + apiVersion, { method: "GET", - headers: { - Authorization: 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }, + headers: getAPIHeaders(), body: null }); @@ -268,13 +313,9 @@ export async function getNewThreadId() { const body = { metadata } - const response = await fetch("https://api.openai.com/v1/threads", { + const response = await fetch(apiEndpoint + "/threads" + apiVersion, { method: "POST", - headers: { - Authorization: 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }, + headers: getAPIHeaders(), body: JSON.stringify(body) }); @@ -312,8 +353,8 @@ export async function getDalleImageGeneration(prompt, image_size = null, image_q return null; } - const requestURL = "https://api.openai.com/v1/images/generations"; - + const requestURL = apiEndpoint + "/images/generations"; + //todo: update headers for Azure. const request = { method: "POST", headers: { @@ -346,7 +387,7 @@ export async function getDalleImageGeneration(prompt, image_size = null, image_q } export async function cancelThreadRun(threadId, runId) { - const url = `https://api.openai.com/v1/threads/${threadId}/runs/${runId}/cancel`; + const url = apiEndpoint + `/threads/${threadId}/runs/${runId}/cancel` + apiVersion; if (!openaiKey) { throw new Error('openai key not set. cannot validate thread.'); @@ -357,10 +398,7 @@ export async function cancelThreadRun(threadId, runId) { } const method = 'POST'; - const headers = { - 'Authorization': 'Bearer ' + openaiKey, - 'OpenAI-Beta': 'assistants=v2' - }; + const headers = getAPIHeaders('woct'); const request = { method, @@ -377,7 +415,7 @@ export async function cancelThreadRun(threadId, runId) { } export async function getThreadMessages(threadId, limit) { - const url = `https://api.openai.com/v1/threads/${threadId}/messages?limit=${limit}`; + const url = apiEndpoint + `/threads/${threadId}/messages?limit=${limit}` + apiVersionMultipleParams; if (!openaiKey) { throw new Error('openai key not set. cannot validate thread.'); @@ -388,11 +426,7 @@ export async function getThreadMessages(threadId, limit) { } const method = 'GET'; - const headers = { - 'Authorization': 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - 'OpenAI-Beta': 'assistants=v2' - }; + const headers = getAPIHeaders(); const request = { method, @@ -411,17 +445,15 @@ export async function getThreadMessages(threadId, limit) { } export async function getFileContent(fileId) { - const url = `https://api.openai.com/v1/files/${fileId}/content`; + //todo update for Azure + const url = `${apiEndpoint}/files/${fileId}/content`; if (!openaiKey) { throw new Error('openai key not set. cannot validate thread.'); } const method = 'GET'; - const headers = { - 'Authorization': 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - }; + const headers = getAPIHeaders('wooaib'); const request = { method, @@ -460,7 +492,8 @@ export async function getFileSrc(fileId) { } export async function getFileInfo(fileId) { - const url = `https://api.openai.com/v1/files/${fileId}`; + //todo update for Azure + const url = `${apiEndpoint}/files/${fileId}`; if (!openaiKey) { throw new Error('openai key not set. cannot validate thread.'); @@ -493,12 +526,9 @@ export async function getChatCompletion(model, messages, tokenLimit) { throw new Error('openai key not set. cannot validate thread.'); } - const url = `https://api.openai.com/v1/chat/completions`; + const url = apiEndpoint + `/chat/completions`; const method = 'POST'; - const headers = { - 'Authorization': 'Bearer ' + openaiKey, - 'Content-Type': 'application/json', - }; + const headers = getAPIHeaders('wooaib'); const body = JSON.stringify({ "model": model, "messages": messages, @@ -535,7 +565,9 @@ export async function uploadFile(b64Data, fileName, type) { form.append('purpose', 'assistants'); form.append('file', file); - const url = `https://api.openai.com/v1/files`; + //todo update for Azure. + + const url = apiEndpoint + `/files`; const method = 'POST'; const headers = { 'Authorization': 'Bearer ' + openaiKey, @@ -570,7 +602,7 @@ export async function getImageDescription(base64, prompt) { prompt = "Give a detailed but concise description of the image. If there are any engineering, biological, or mechanical processes present, include how they're present." } - const model = "gpt-4o-2024-05-13" + const model = "gpt-4o-2024-05-13" //todo: update for azure. const messages = [ { "role": "user", diff --git a/src/utils/threadUtils.js b/src/utils/threadUtils.js index e9729ae..6f86142 100644 --- a/src/utils/threadUtils.js +++ b/src/utils/threadUtils.js @@ -51,7 +51,7 @@ export async function getActiveThread(defaultAsst) { await setThreadAsst(thread, asst); thread.asst = asst; - } else if (asst && !(await validAssistant(asst.id, asst.name))) { // asst doesn't exist, or is invalid + } else if (asst && !(await validAssistant(asst.id, asst.name+'v'+asst.version))) { // asst doesn't exist, or is invalid asst = await getNewAsst(asst, defaultAsst); await setThreadAsst(thread, asst); thread.asst = asst; @@ -226,16 +226,21 @@ export function getFileTypeByName(fileName) { "xlsx": "excel", "pdf": "pdf", "txt": "txt", + "pptx": "powerpoint", + "docx": "word", + "doc": "word", "png": "image", "jpg": "image", "jpeg": "image", + "webp": "image", + "gif": "image" } if (!fileName) { return ""; } - const extensionMatches = /^.*\.(csv|xlsx|pdf|txt|png|jpg|jpeg)$/gm.exec(fileName); + const extensionMatches = /^.*\.(csv|xlsx|pdf|txt|pptx|docx|doc|png|jpg|jpeg|webp|gif)$/gm.exec(fileName); // first is whole match, second is capture group. Only one capture group can appear. if (!extensionMatches || extensionMatches.length !== 2) { @@ -316,16 +321,16 @@ function handleAnnotations(content, storedFiles, newFiles, annotatedFiles) { const annotations = msg.text.annotations; annotations.forEach((annotation) => { - const fileId = annotation.file_path.file_id; + const fileId = annotation.file_citation.file_id; const replacement = annotation.text; const storedFile = storedFiles.get(fileId); const newFile = newFiles.get(fileId); if (storedFile) { - msgText = msgText.replaceAll(replacement, storedFile.src); + msgText = msgText.replaceAll(replacement, storedFile.src ? storedFile.src : ""); } else if (newFile) { - msgText = msgText.replaceAll(replacement, newFile.src); + msgText = msgText.replaceAll(replacement, newFile.src ? newFile.src : ""); } else { msgText = msgText.replaceAll(replacement, "[ Deleted File ]") From 917a0fd29be62ee8c2fad6518415c2d6725cad47 Mon Sep 17 00:00:00 2001 From: Brandon Ruffridge Date: Wed, 18 Dec 2024 14:28:09 -0500 Subject: [PATCH 2/2] updated rollup and svelte packages. --- package-lock.json | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 08b73cc..916503f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,12 +21,12 @@ "@rollup/plugin-terser": "^0.4.4", "highlight.js": "^11.9.0", "npm-run-all": "^4.1.5", - "rollup": "^4.18.0", + "rollup": "^4.28.1", "rollup-plugin-baked-env": "^1.0.1", "rollup-plugin-css-only": "^4.5.2", "rollup-plugin-livereload": "^2.0.0", "rollup-plugin-svelte": "^7.2.2", - "svelte": "^4.2.18" + "svelte": "^4.2.19" } }, "node_modules/@ampproject/remapping": { diff --git a/package.json b/package.json index f5c731c..9f4ebe8 100644 --- a/package.json +++ b/package.json @@ -7,12 +7,12 @@ "@rollup/plugin-terser": "^0.4.4", "highlight.js": "^11.9.0", "npm-run-all": "^4.1.5", - "rollup": "^4.18.0", + "rollup": "^4.28.1", "rollup-plugin-baked-env": "^1.0.1", "rollup-plugin-css-only": "^4.5.2", "rollup-plugin-livereload": "^2.0.0", "rollup-plugin-svelte": "^7.2.2", - "svelte": "^4.2.18" + "svelte": "^4.2.19" }, "dependencies": { "deep-chat-dev": "^9.0.202",