From 76b1155336c7b030924ad6b18091663b648f00d8 Mon Sep 17 00:00:00 2001 From: H0llyW00dzZ Date: Mon, 26 Feb 2024 05:27:40 +0700 Subject: [PATCH] [Cherry Pick] [Head Repo] Add Image Viewer (#286) * feat: add image viewer * feat: add image viewer * add alt attribute to image viewer * fix: prompt-toast obscuring the image viewer --------- Co-authored-by: TheRamU <43772379+TheRamU@users.noreply.github.com> --- app/client/platforms/google.ts | 7 +- app/components/chat.module.scss | 63 ++++++++++++++- app/components/chat.tsx | 133 ++++++++++++++++++++++---------- app/components/markdown.tsx | 18 +++++ app/utils.ts | 81 ++++++++++--------- 5 files changed, 218 insertions(+), 84 deletions(-) diff --git a/app/client/platforms/google.ts b/app/client/platforms/google.ts index 9cfd320f272..9d9e9762bd7 100644 --- a/app/client/platforms/google.ts +++ b/app/client/platforms/google.ts @@ -165,9 +165,6 @@ export class GeminiProApi implements LLMApi { chatConfig.useMaxTokens, ); - // if (visionModel && messages.length > 1) { - // options.onError?.(new Error("Multiturn chat is not enabled for models/gemini-pro-vision")); - // } const modelConfig = { ...useAppConfig.getState().modelConfig, ...useChatStore.getState().currentSession().mask.modelConfig, @@ -217,9 +214,7 @@ export class GeminiProApi implements LLMApi { const controller = new AbortController(); options.onController?.(controller); try { - let googleChatPath = visionModel - ? Google.VisionChatPath - : Google.ChatPath; + let googleChatPath = multimodal ? Google.VisionChatPath : Google.ChatPath; let chatPath = this.path(googleChatPath); // let baseUrl = accessStore.googleUrl; diff --git a/app/components/chat.module.scss b/app/components/chat.module.scss index 5a182665a6c..d1772438df9 100644 --- a/app/components/chat.module.scss +++ b/app/components/chat.module.scss @@ -8,7 +8,7 @@ } .attach-image { - cursor: default; + cursor: pointer; width: 64px; height: 64px; border: rgba($color: #888, $alpha: 0.2) 1px solid; @@ -39,6 +39,12 @@ border-radius: 5px; float: right; background-color: var(--white); + opacity: 0.8; + transition: all ease 0.3s; + } + + .delete-image:hover { + opacity: 1; } } @@ -422,6 +428,10 @@ transition: all ease 0.3s; } +.chat-message-item img { + cursor: pointer; +} + .chat-message-item-image { width: 100%; margin-top: 10px; @@ -624,4 +634,55 @@ .chat-input-send { bottom: 30px; } +} + +@keyframes slide-in { + from { + transform: translateY(10px); + opacity: 0; + } + + to { + transform: translateY(0); + opacity: 1; + } +} + +.image-box { + background-color: rgba($color: #000000, $alpha: 0.5); + position: fixed; + width: 100vw; + height: 100vh; + top: 0; + left: 0; + z-index: 999; +} + +.image-box>img { + animation: slide-in ease 0.3s; + width: 100%; + height: 100%; + object-fit: contain; +} + +.image-box-close-button { + cursor: pointer; + box-sizing: border-box; + position: fixed; + top: 20px; + right: 20px; + display: flex; + justify-content: center; + align-items: center; + width: 40px; + height: 40px; + border-radius: 10px; + border: rgba($color: #888, $alpha: 0.2) 1px solid; + background-color: var(--white); + transition: all ease 0.3s; +} + +.image-box-close-button:hover { + border-color: var(--primary); + filter: brightness(0.9); } \ No newline at end of file diff --git a/app/components/chat.tsx b/app/components/chat.tsx index e84279cd07f..916d8a94539 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -42,6 +42,7 @@ import ChatGptIcon from "../icons/chatgpt.png"; import EyeOnIcon from "../icons/eye.svg"; import EyeOffIcon from "../icons/eye-off.svg"; import { debounce, escapeRegExp } from "lodash"; +import CloseIcon from "../icons/close.svg"; import { ChatMessage, @@ -838,12 +839,38 @@ function useDebouncedEffect(effect: () => void, deps: any[], delay: number) { export function DeleteImageButton(props: { deleteImage: () => void }) { return ( -
+
{ + e.preventDefault(); + e.stopPropagation(); + props.deleteImage(); + }} + >
); } +export function ImageBox(props: { + showImageBox: boolean; + data: { src: string; alt: string }; + closeImageBox: () => void; +}) { + return ( +
+ {props.data.alt} +
+ +
+
+ ); +} + function _Chat() { type RenderMessage = ChatMessage & { preview?: boolean }; @@ -866,6 +893,8 @@ function _Chat() { const isApp = getClientConfig()?.isApp; const [attachImages, setAttachImages] = useState([]); const [uploading, setUploading] = useState(false); + const [showImageBox, setShowImageBox] = useState(false); + const [imageBoxData, setImageBoxData] = useState({ src: "", alt: "" }); // prompt hints const promptStore = usePromptStore(); @@ -1456,48 +1485,58 @@ function _Chat() { }, [session.id]); async function uploadImage() { - const images: string[] = []; - images.push(...attachImages); - - images.push( - ...(await new Promise((res, rej) => { - const fileInput = document.createElement("input"); - fileInput.type = "file"; - fileInput.accept = - "image/png, image/jpeg, image/webp, image/heic, image/heif"; - fileInput.multiple = true; - fileInput.onchange = (event: any) => { - setUploading(true); - const files = event.target.files; - const imagesData: string[] = []; - for (let i = 0; i < files.length; i++) { - const file = event.target.files[i]; - compressImage(file, 256 * 1024) - .then((dataUrl) => { - imagesData.push(dataUrl); - if ( - imagesData.length === 3 || - imagesData.length === files.length - ) { - setUploading(false); - res(imagesData); - } - }) - .catch((e) => { + const maxImages = 3; + if (uploading) return; + new Promise((res, rej) => { + const fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.accept = + "image/png, image/jpeg, image/webp, image/heic, image/heif"; + fileInput.multiple = true; + fileInput.onchange = (event: any) => { + setUploading(true); + const files = event.target.files; + const imagesData: string[] = []; + for (let i = 0; i < files.length; i++) { + const file = event.target.files[i]; + compressImage(file, 256 * 1024) + .then((dataUrl) => { + imagesData.push(dataUrl); + if ( + imagesData.length + attachImages.length >= maxImages || + imagesData.length === files.length + ) { setUploading(false); - rej(e); - }); - } - }; - fileInput.click(); - })), - ); + res(imagesData); + } + }) + .catch((e) => { + rej(e); + }); + } + }; + fileInput.click(); + }) + .then((imagesData) => { + const images: string[] = []; + images.push(...attachImages); + images.push(...imagesData); + setAttachImages(images); + const imagesLength = images.length; + if (imagesLength > maxImages) { + images.splice(maxImages, imagesLength - maxImages); + } + setAttachImages(images); + }) + .catch(() => { + setUploading(false); + }); + } - const imagesLength = images.length; - if (imagesLength > 3) { - images.splice(3, imagesLength - 3); - } - setAttachImages(images); + function openImageBox(src: string, alt?: string) { + alt = alt ?? ""; + setImageBoxData({ src, alt }); + setShowImageBox(true); } // this now better @@ -1589,7 +1628,11 @@ function _Chat() { setShowModal={setShowPromptModal} />
- + setShowImageBox(false)} + >
= messages.length - 6} + openImageBox={openImageBox} /> {getMessageImages(message).length == 1 && ( // this fix when uploading @@ -1753,6 +1797,9 @@ function _Chat() { className={styles["chat-message-item-image"]} src={getMessageImages(message)[0]} alt="" + onClick={() => + openImageBox(getMessageImages(message)[0]) + } /> )} {getMessageImages(message).length > 1 && ( @@ -1774,6 +1821,7 @@ function _Chat() { src={image} alt="" layout="responsive" + onClick={() => openImageBox(image)} /> ); })} @@ -1852,6 +1900,7 @@ function _Chat() { key={index} className={styles["attach-image"]} style={{ backgroundImage: `url("${image}")` }} + onClick={() => openImageBox(image)} >
; defaultShow?: boolean; + openImageBox?: (src: string, alt: string) => void; } & React.DOMAttributes, ) { const mdRef = useRef(null); + const { parentRef, openImageBox } = props; + + useEffect(() => { + if (!parentRef || !openImageBox) { + return; + } + const imgs = mdRef.current?.querySelectorAll("img"); + if (imgs) { + imgs.forEach((img) => { + const src = img.getAttribute("src"); + const alt = img.getAttribute("alt") ?? ""; + if (src) { + img.onclick = () => openImageBox(src, alt); + } + }); + } + }, [mdRef, parentRef, openImageBox]); return (
{ return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = (readerEvent: any) => { - const image = new Image(); - image.onload = () => { - let canvas = document.createElement("canvas"); + fetch(URL.createObjectURL(file)) + .then((response) => response.blob()) + .then((blob) => createImageBitmap(blob)) + .then((imageBitmap) => { + let canvas = new OffscreenCanvas(imageBitmap.width, imageBitmap.height); let ctx = canvas.getContext("2d"); - let width = image.width; - let height = image.height; + let width = imageBitmap.width; + let height = imageBitmap.height; let quality = 0.9; - let dataUrl; - - do { - canvas.width = width; - canvas.height = height; - ctx?.clearRect(0, 0, canvas.width, canvas.height); - ctx?.drawImage(image, 0, 0, width, height); - dataUrl = canvas.toDataURL("image/jpeg", quality); - - if (dataUrl.length < maxSize) break; - - if (quality > 0.5) { - // Prioritize quality reduction - quality -= 0.1; - } else { - // Then reduce the size - width *= 0.9; - height *= 0.9; - } - } while (dataUrl.length > maxSize); - - resolve(dataUrl); - }; - image.onerror = reject; - image.src = readerEvent.target.result; - }; - reader.onerror = reject; - reader.readAsDataURL(file); + + const checkSizeAndPostMessage = () => { + canvas + .convertToBlob({ type: "image/jpeg", quality: quality }) + .then((blob) => { + const reader = new FileReader(); + reader.onloadend = function () { + const base64data = reader.result; + if (typeof base64data !== "string") { + reject("Invalid base64 data"); + return; + } + if (base64data.length < maxSize) { + resolve(base64data); + return; + } + if (quality > 0.5) { + // Prioritize quality reduction + quality -= 0.1; + } else { + // Then reduce the size + width *= 0.9; + height *= 0.9; + } + canvas.width = width; + canvas.height = height; + + ctx?.drawImage(imageBitmap, 0, 0, width, height); + checkSizeAndPostMessage(); + }; + reader.readAsDataURL(blob); + }); + }; + ctx?.drawImage(imageBitmap, 0, 0, width, height); + checkSizeAndPostMessage(); + }) + .catch((error) => { + throw error; + }); }); }