livekit · s-hamdananwar · Jul 27, 2024 · Jul 27, 2024
diff --git a/packages/core/src/components/chat.ts b/packages/core/src/components/chat.ts
@@ -9,6 +9,14 @@ export interface ChatMessage {
   id: string;
   timestamp: number;
   message: string;
+  image?: Blob;
+  imagePacketProperties?: ImagePacketProperties;
+}
+
+export interface ImagePacketProperties {
+  packetIndex: number;
+  totalPacketCount: number;
+  packetImageData: string;
 }
 
 /** @public */
@@ -44,11 +52,26 @@ const encode = (message: ChatMessage) => encoder.encode(JSON.stringify(message))
 
 const decode = (message: Uint8Array) => JSON.parse(decoder.decode(message)) as ReceivedChatMessage;
 
+const getImageBlob = (imagePackets: string[] | null[]) => {
+  const completeImageData = imagePackets.join('');
+  const byteCharacters = atob(completeImageData);
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+  const byteArray = new Uint8Array(byteNumbers);
+  const blob = new Blob([byteArray], { type: 'image/jpeg' });
+  return blob;
+};
+
 export function setupChat(room: Room, options?: ChatOptions) {
   const onDestroyObservable = new Subject<void>();
 
   const { messageDecoder, messageEncoder, channelTopic, updateChannelTopic } = options ?? {};
 
+  // Used to map an image to its packets using id
+  const chatImagesMap: { [key: string]: string[] | null[] } = {};
+
   const topic = channelTopic ?? DataTopic.CHAT;
 
   const updateTopic = updateChannelTopic ?? DataTopic.CHAT_UPDATE;
@@ -95,7 +118,43 @@ export function setupChat(room: Room, options?: ChatOptions) {
 
         return [...acc];
       }
-      return [...acc, value];
+      // handle image messages
+      if (value.imagePacketProperties) {
+        const imageId = value.id;
+        const totalPacketCount = value.imagePacketProperties.totalPacketCount;
+        const packetIndex = value.imagePacketProperties.packetIndex;
+        if (!chatImagesMap[imageId]) {
+          chatImagesMap[imageId] = new Array(totalPacketCount).fill(null);
+        }
+
+        if (packetIndex >= 0 && packetIndex < totalPacketCount) {
+          chatImagesMap[imageId][packetIndex] = value.imagePacketProperties.packetImageData;
+        } else {
+          console.error(
+            `Index ${packetIndex} is out of bounds, Array size with size ${totalPacketCount}`,
+          );
+        }
+
+        // received final image packet, can send a message with image to users
+        if (packetIndex == totalPacketCount - 1) {
+          const packets = chatImagesMap[imageId];
+          if (packets != null && packets.every((element) => element !== null)) {
+            const imageBlob = getImageBlob(packets);
+            value.image = imageBlob;
+            delete chatImagesMap[imageId];
+            return [...acc, value];
+          } else {
+            console.error(`Didn't receive every image packet in order, discarding image message`);
+            delete chatImagesMap[imageId];
+            return [...acc];
+          }
+        } else {
+          // ignore messages by not sending them to the user until final image packet arrives
+          return [...acc];
+        }
+      } else {
+        return [...acc, value];
+      }
     }, []),
     takeUntil(onDestroyObservable),
   );
@@ -147,12 +206,55 @@ export function setupChat(room: Room, options?: ChatOptions) {
     }
   };
 
+  const sendImagePacket = async (
+    message: string,
+    messageId: string,
+    packetIndex: number,
+    totalPacketCount: number,
+    packetImageData: string,
+  ) => {
+    const timestamp = Date.now();
+    const imageProp: ImagePacketProperties = {
+      packetIndex: packetIndex,
+      totalPacketCount: totalPacketCount,
+      packetImageData: packetImageData,
+    };
+    const chatMessage: ChatMessage = {
+      id: messageId,
+      message,
+      timestamp,
+      imagePacketProperties: imageProp,
+    };
+    const encodedMsg = finalMessageEncoder(chatMessage);
+    isSending$.next(true);
+    try {
+      await sendMessage(room.localParticipant, encodedMsg, {
+        topic: topic,
+        reliable: true,
+      });
+      messageSubject.next({
+        payload: encodedMsg,
+        topic: topic,
+        from: room.localParticipant,
+      });
+      return chatMessage;
+    } finally {
+      isSending$.next(false);
+    }
+  };
+
   function destroy() {
     onDestroyObservable.next();
     onDestroyObservable.complete();
     topicSubjectMap.delete(room);
   }
   room.once(RoomEvent.Disconnected, destroy);
 
-  return { messageObservable: messagesObservable, isSendingObservable: isSending$, send, update };
+  return {
+    messageObservable: messagesObservable,
+    isSendingObservable: isSending$,
+    send,
+    sendImagePacket,
+    update,
+  };
 }
diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md
@@ -123,6 +123,12 @@ export interface ChatMessage {
     // (undocumented)
     id: string;
     // (undocumented)
+    image?: Blob;
+    // Warning: (ae-forgotten-export) The symbol "ImagePacketProperties" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    imagePacketProperties?: ImagePacketProperties;
+    // (undocumented)
     message: string;
     // (undocumented)
     timestamp: number;
@@ -230,6 +236,11 @@ export interface FeatureFlags {
     autoSubscription?: boolean;
 }
 
+// Warning: (ae-internal-missing-underscore) The name "FileAttachIcon" should be prefixed with an underscore because the declaration is marked as @internal
+//
+// @internal (undocumented)
+export const FileAttachIcon: (props: SVGProps<SVGSVGElement>) => React_2.JSX.Element;
+
 // @public
 export function FocusLayout({ trackRef, ...htmlProps }: FocusLayoutProps): React_2.JSX.Element;
 
@@ -695,6 +706,7 @@ export function useChat(options?: ChatOptions): {
     update: (message: string, messageId: string) => Promise<ChatMessage>;
     chatMessages: ReceivedChatMessage[];
     isSending: boolean;
+    sendImagePacket: (message: string, messageId: string, packetIndex: number, totalPacketCount: number, packetImageData: string) => Promise<ChatMessage>;
 };
 
 // @public

diff --git a/packages/react/src/assets/icons/FileAttachIcon.tsx b/packages/react/src/assets/icons/FileAttachIcon.tsx
@@ -0,0 +1,22 @@
+/**
+ * WARNING: This file was auto-generated by svgr. Do not edit.
+ */
+import * as React from 'react';
+import type { SVGProps } from 'react';
+/**
+ * @internal
+ */
+const SvgFileAttachIcon = (props: SVGProps<SVGSVGElement>) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    xmlSpace="preserve"
+    width={16}
+    height={16}
+    fill="#fff"
+    viewBox="0 0 500 500"
+    {...props}
+  >
+    <path d="M459.1 37.48c-49.999-49.973-131.045-49.973-181.018 0L21.594 295.355c-8.309 8.354-8.272 21.861.081 30.17 8.354 8.309 21.861 8.272 30.17-.081L308.293 67.609c33.271-33.271 87.308-33.271 120.641.045 33.328 33.328 33.328 87.359 0 120.67L164.912 453.755c-20.787 20.772-54.563 20.772-75.396-.046-20.826-20.826-20.826-54.593.006-75.425l218.731-218.731.02-.022c8.332-8.309 21.816-8.303 30.139.02 8.33 8.33 8.33 21.831 0 30.161l-105.58 105.602c-8.33 8.332-8.329 21.84.003 30.17 8.332 8.33 21.84 8.329 30.17-.003l105.579-105.6c24.991-24.991 24.991-65.507-.002-90.499-24.993-24.993-65.508-24.993-90.501 0l-.04.044L59.352 348.115c-37.494 37.494-37.494 98.276 0 135.77 37.498 37.471 98.272 37.471 135.764.006L459.14 218.458c49.956-49.93 49.956-130.982-.04-180.978z" />
+  </svg>
+);
+export default SvgFileAttachIcon;
diff --git a/packages/react/src/assets/icons/index.ts b/packages/react/src/assets/icons/index.ts
@@ -3,6 +3,7 @@ export { default as CameraIcon } from './CameraIcon';
 export { default as ChatCloseIcon } from './ChatCloseIcon';
 export { default as ChatIcon } from './ChatIcon';
 export { default as Chevron } from './Chevron';
+export { default as FileAttachIcon } from './FileAttachIcon';
 export { default as FocusToggleIcon } from './FocusToggleIcon';
 export { default as GearIcon } from './GearIcon';
 export { default as LeaveIcon } from './LeaveIcon';

diff --git a/packages/react/src/components/ChatEntry.tsx b/packages/react/src/components/ChatEntry.tsx
@@ -46,6 +46,10 @@ export const ChatEntry: (
     const hasBeenEdited = !!entry.editTimestamp;
     const time = new Date(entry.timestamp);
     const locale = navigator ? navigator.language : 'en-US';
+    let imgSrc = '';
+    if (entry.image) {
+      imgSrc = URL.createObjectURL(entry.image);
+    }
 
     return (
       <li
@@ -71,8 +75,12 @@ export const ChatEntry: (
             )}
           </span>
         )}
-
-        <span className="lk-message-body">{formattedMessage}</span>
+        <span>
+          <div className="lk-message-body">
+            {formattedMessage}
+            {entry.image && <img className="lk-image-body" src={imgSrc}></img>}
+          </div>
+        </span>
       </li>
     );
   },

diff --git a/packages/react/src/hooks/useChat.ts b/packages/react/src/hooks/useChat.ts
@@ -28,5 +28,11 @@ export function useChat(options?: ChatOptions) {
   const isSending = useObservableState(setup.isSendingObservable, false);
   const chatMessages = useObservableState<ReceivedChatMessage[]>(setup.messageObservable, []);
 
-  return { send: setup.send, update: setup.update, chatMessages, isSending };
+  return {
+    send: setup.send,
+    update: setup.update,
+    chatMessages,
+    isSending,
+    sendImagePacket: setup.sendImagePacket,
+  };
 }
diff --git a/packages/react/src/prefabs/Chat.tsx b/packages/react/src/prefabs/Chat.tsx
@@ -6,7 +6,7 @@ import type { MessageFormatter } from '../components/ChatEntry';
 import { ChatEntry } from '../components/ChatEntry';
 import { useChat } from '../hooks/useChat';
 import { ChatToggle } from '../components';
-import { ChatCloseIcon } from '../assets/icons';
+import { FileAttachIcon, ChatCloseIcon } from '../assets/icons';
 
 /** @public */
 export interface ChatProps extends React.HTMLAttributes<HTMLDivElement>, ChatOptions {
@@ -33,20 +33,38 @@ export function Chat({
   ...props
 }: ChatProps) {
   const inputRef = React.useRef<HTMLInputElement>(null);
+  const thumbnailRef = React.useRef<HTMLInputElement>(null);
   const ulRef = React.useRef<HTMLUListElement>(null);
+  const [imagePackets, setImagePackets] = React.useState<string[]>([]);
+  const [selectedImage, setSelectedImage] = React.useState<string | null>(null);
 
   const chatOptions: ChatOptions = React.useMemo(() => {
     return { messageDecoder, messageEncoder, channelTopic };
   }, [messageDecoder, messageEncoder, channelTopic]);
 
-  const { send, chatMessages, isSending } = useChat(chatOptions);
+  const { send, chatMessages, isSending, sendImagePacket } = useChat(chatOptions);
 
   const layoutContext = useMaybeLayoutContext();
   const lastReadMsgAt = React.useRef<ChatMessage['timestamp']>(0);
 
   async function handleSubmit(event: React.FormEvent) {
     event.preventDefault();
-    if (inputRef.current && inputRef.current.value.trim() !== '') {
+    if (thumbnailRef.current && imagePackets.length > 0) {
+      if (sendImagePacket) {
+        const message = inputRef.current ? inputRef.current.value : '';
+        const id = crypto.randomUUID();
+        for (let i = 0; i < imagePackets.length; i++) {
+          await sendImagePacket(message, id, i, imagePackets.length, imagePackets[i]);
+        }
+        if (inputRef.current) {
+          inputRef.current.value = '';
+          inputRef.current.focus();
+        }
+        thumbnailRef.current.value = '';
+        setImagePackets([]);
+        setSelectedImage(null);
+      }
+    } else if (inputRef.current && inputRef.current.value.trim() !== '') {
       if (send) {
         await send(inputRef.current.value);
         inputRef.current.value = '';
@@ -55,6 +73,50 @@ export function Chat({
     }
   }
 
+  function handleImageUploadClicked(event: React.MouseEvent<HTMLButtonElement, MouseEvent>): void {
+    event.preventDefault();
+    if (!thumbnailRef || !thumbnailRef.current) return;
+
+    thumbnailRef.current.click();
+  }
+
+  const handleFileRead = async (event: React.ChangeEvent<HTMLInputElement>) => {
+    event.preventDefault();
+    if (event.target.files != null) {
+      const file = event.target.files[0];
+      if (file.size > 3000000) {
+        throw Error('Image file is larger than 3 MB, please select a different image');
+      } else {
+        const fileData = await getFileData(file);
+        packetizeFileData(String(fileData));
+      }
+    }
+  };
+
+  const packetizeFileData = (fileData: string) => {
+    const maxPacketSize = 50000; // LiveKit doesn't support message length higher than ~6.5 kb
+    const packets = [];
+    for (let i = 0; i < fileData.length; i += maxPacketSize) {
+      packets.push(fileData.slice(i, i + maxPacketSize));
+    }
+    setImagePackets(packets);
+    inputRef.current?.focus();
+  };
+
+  const getFileData = (file: File) => {
+    return new Promise((resolve, reject) => {
+      const fileReader = new FileReader();
+      fileReader.readAsDataURL(file);
+      fileReader.onload = () => {
+        setSelectedImage(fileReader.result as string);
+        resolve(String(fileReader.result).split(',')[1]);
+      };
+      fileReader.onerror = (error) => {
+        reject(error);
+      };
+    });
+  };
+
   React.useEffect(() => {
     if (ulRef) {
       ulRef.current?.scrollTo({ top: ulRef.current.scrollHeight });
@@ -119,7 +181,25 @@ export function Chat({
               );
             })}
       </ul>
+      {selectedImage && (
+        <div className="lk-thumbnail">
+          <img src={selectedImage} alt="Selected image" />
+          <button
+            className="lk-delete-button"
+            onClick={() => {
+              if (thumbnailRef.current) thumbnailRef.current.value = '';
+              setImagePackets([]);
+              setSelectedImage(null);
+            }}
+          >
+            <ChatCloseIcon />
+          </button>
+        </div>
+      )}
       <form className="lk-chat-form" onSubmit={handleSubmit}>
+        <button type="button" onClick={handleImageUploadClicked} className="lk-file-attach-button">
+          <FileAttachIcon />
+        </button>
         <input
           className="lk-form-control lk-chat-form-input"
           disabled={isSending}
@@ -134,6 +214,15 @@ export function Chat({
           Send
         </button>
       </form>
+      <input
+        disabled={isSending}
+        hidden
+        ref={thumbnailRef}
+        type="file"
+        accept="image/*"
+        onChange={(e) => handleFileRead(e)}
+        placeholder="upload image..."
+      />
     </div>
   );
 }