feat: support other image file types

app/client/fetch/file.ts

@@ -1,24 +1,19 @@
 import { URLDetailContent } from "./url";
 import { FileWrap } from "../../utils/file";
+import {
+  ALLOWED_IMAGE_EXTENSIONS,
+  IMAGE_TYPES,
+  ImageType,
+} from "@/app/constant";
 
 export async function getDetailContentFromFile(
   file: FileWrap,
 ): Promise<URLDetailContent> {
-  switch (file.extension) {
-    case "pdf": {
-      return await getPDFFileDetail(file);
-    }
-    case "txt": {
-      return await getTextFileDetail(file);
-    }
-    case "jpg":
-    case "jpeg": {
-      return await getImageFileDetail(file);
-    }
-    default: {
-      throw new Error("Not supported file type");
-    }
-  }
+  if (file.extension === "pdf") return await getPDFFileDetail(file);
+  if (file.extension === "txt") return await getTextFileDetail(file);
+  if (ALLOWED_IMAGE_EXTENSIONS.includes(file.extension))
+    return await getImageFileDetail(file);
+  throw new Error("Not supported file type");
 }
 
 async function getPDFFileDetail(file: FileWrap): Promise<URLDetailContent> {
@@ -70,3 +65,6 @@ async function getImageFileDetail(file: FileWrap) {
   console.log(data);
   return data as URLDetailContent;
 }
+
+export const isImageFileType = (type: string) =>
+  IMAGE_TYPES.includes(type as ImageType);

app/client/fetch/url.ts

@@ -1,12 +1,16 @@
+import { DocumentType, ImageType } from "@/app/constant";
+
 export type Embedding = {
   text: string;
   embedding: number[];
 };
 
+export type UrlDetailType = DocumentType | ImageType;
+
 export type URLDetail = {
   url: string;
   size: number;
-  type: "text/html" | "application/pdf" | "text/plain" | "image/jpeg";
+  type: UrlDetailType;
   embeddings?: Embedding[];
 };
 

app/components/chat/chat-input.tsx

@@ -1,4 +1,7 @@
-import { getDetailContentFromFile } from "@/app/client/fetch/file";
+import {
+  getDetailContentFromFile,
+  isImageFileType,
+} from "@/app/client/fetch/file";
 import { URLDetail, URLDetailContent, isURL } from "@/app/client/fetch/url";
 import { Button } from "@/app/components/ui/button";
 import { Textarea } from "@/app/components/ui/textarea";
@@ -119,13 +122,13 @@ export default function ChatInput(props: ChatInputProps) {
     action: () => Promise<void>,
   ): Promise<void> => {
     let tempUrl: string;
-    if (file.type === "image/jpeg") {
+    if (isImageFileType(file.type)) {
       tempUrl = URL.createObjectURL(file);
       setTemporaryBlobUrl(tempUrl);
     }
 
     return action().finally(() => {
-      if (file.type === "image/jpeg") {
+      if (isImageFileType(file.type)) {
         URL.revokeObjectURL(tempUrl);
         setTemporaryBlobUrl(undefined);
       }
@@ -136,7 +139,7 @@ export default function ChatInput(props: ChatInputProps) {
     try {
       await manageTemporaryBlobUrl(fileInput.file, async () => {
         const fileDetail = await getDetailContentFromFile(fileInput);
-        if (fileInput.file.type === "image/jpeg") {
+        if (isImageFileType(fileInput.file.type)) {
           setImageFile(fileDetail);
         } else {
           callLLM({ fileDetail });

app/components/chat/chat.tsx

@@ -24,6 +24,7 @@ import { ChatAction } from "./chat-action";
 import ChatHeader from "./chat-header";
 import ChatInput from "./chat-input";
 import { ClearContextDivider } from "./clear-context-divider";
+import { isImageFileType } from "@/app/client/fetch/file";
 
 const Markdown = dynamic(
   async () => (await import("../ui/markdown")).Markdown,
@@ -94,7 +95,7 @@ export function Chat() {
   const renderMessages = useMemo(() => {
     const getFrontendMessages = (messages: ChatMessage[]) => {
       return messages.map((message) => {
-        if (!message.urlDetail || message.urlDetail.type === "image/jpeg")
+        if (!message.urlDetail || isImageFileType(message.urlDetail.type))
           return message;
         const urlTypePrefix = getUrlTypePrefix(message.urlDetail.type);
         const sizeInKB = Math.round(message.urlDetail.size / 1024);
@@ -256,13 +257,14 @@ export function Chat() {
                               : "bg-muted",
                           )}
                         >
-                          {message.urlDetail?.type === "image/jpeg" && (
-                            <img
-                              src={message.urlDetail.url}
-                              alt="Message image"
-                              className="object-contain w-full h-52 rounded-lg mb-2"
-                            />
-                          )}
+                          {message.urlDetail?.type &&
+                            isImageFileType(message.urlDetail.type) && (
+                              <img
+                                src={message.urlDetail.url}
+                                alt="Message image"
+                                className="object-contain w-full h-52 rounded-lg mb-2"
+                              />
+                            )}
                           <Markdown
                             content={message.content}
                             loading={

app/constant.ts

@@ -16,10 +16,27 @@ export const REQUEST_TIMEOUT_MS = 60000;
 export const CHAT_PAGE_SIZE = 15;
 export const MAX_RENDER_MSG_COUNT = 45;
 
-export const ALLOWED_IMAGE_EXTENSIONS = ["jpeg", "jpg"];
+export const ALLOWED_IMAGE_EXTENSIONS = ["jpeg", "jpg", "png", "gif", "webp"];
 export const ALLOWED_TEXT_EXTENSIONS = ["pdf", "txt"];
 export const ALLOWED_DOCUMENT_EXTENSIONS = [
   ...ALLOWED_TEXT_EXTENSIONS,
   ...ALLOWED_IMAGE_EXTENSIONS,
 ];
 export const DOCUMENT_FILE_SIZE_LIMIT = 1024 * 1024 * 10; // 10 MB
+
+export const DOCUMENT_TYPES = [
+  "text/html",
+  "application/pdf",
+  "text/plain",
+] as const;
+
+export type DocumentType = (typeof DOCUMENT_TYPES)[number];
+
+export const IMAGE_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+] as const;
+
+export type ImageType = (typeof IMAGE_TYPES)[number];

app/locales/en.ts

@@ -137,7 +137,7 @@ const en = {
   },
 
   Welcome: {
-    Title: "LlamaIndex Chat",
+    Title: "Chat LlamaIndex",
     SubTitle: "Create chat bots that know your data",
     Quote:
       "“This tool has saved me countless hours of work and helped me apply AI features to my work faster than ever before.”",

app/store/session.ts

@@ -16,6 +16,7 @@ import {
 } from "../client/platforms/llm";
 import { prettyObject } from "../utils/format";
 import { Bot } from "./bot";
+import { isImageFileType } from "../client/fetch/file";
 
 export type ChatMessage = {
   role: MessageRole;
@@ -157,7 +158,7 @@ export async function callSession(
 
   let embeddings: Embedding[] | undefined;
   let message;
-  if (userMessage.urlDetail && userMessage.urlDetail.type !== "image/jpeg") {
+  if (userMessage.urlDetail && !isImageFileType(userMessage.urlDetail.type)) {
     // if the user sends document, let the LLM summarize the content of the URL and just use the document's embeddings
     message = "Summarize the given context briefly in 200 words or less";
     embeddings = userMessage.urlDetail?.embeddings;
@@ -168,7 +169,10 @@ export async function callSession(
       .flatMap((message: ChatMessage) => message.urlDetail?.embeddings)
       .filter((m) => m !== undefined) as Embedding[];
     embeddings = embeddings.length > 0 ? embeddings : undefined;
-    if (userMessage.urlDetail?.type === "image/jpeg") {
+    if (
+      userMessage.urlDetail?.type &&
+      isImageFileType(userMessage.urlDetail?.type)
+    ) {
       message = [
         {
           type: "text",