From 01cc6b222b7bbaa3df39021fcb5f6cec54d60da0 Mon Sep 17 00:00:00 2001 From: Hamdan Date: Sat, 27 Jul 2024 14:57:10 -0500 Subject: [PATCH 1/2] added image attach feature to chat component --- packages/core/src/components/chat.ts | 106 +++++++++++++++++- .../react/src/assets/icons/FileAttachIcon.tsx | 22 ++++ packages/react/src/assets/icons/index.ts | 1 + packages/react/src/components/ChatEntry.tsx | 12 +- packages/react/src/hooks/useChat.ts | 8 +- packages/react/src/prefabs/Chat.tsx | 95 +++++++++++++++- .../styles/assets/icons/file-attach-icon.svg | 18 +++ .../scss/components/controls/_button.scss | 40 +++++++ packages/styles/scss/prefabs/chat.scss | 43 +++++++ 9 files changed, 337 insertions(+), 8 deletions(-) create mode 100644 packages/react/src/assets/icons/FileAttachIcon.tsx create mode 100644 packages/styles/assets/icons/file-attach-icon.svg diff --git a/packages/core/src/components/chat.ts b/packages/core/src/components/chat.ts index 088b6e932..269641540 100644 --- a/packages/core/src/components/chat.ts +++ b/packages/core/src/components/chat.ts @@ -9,6 +9,14 @@ export interface ChatMessage { id: string; timestamp: number; message: string; + image?: Blob; + imagePacketProperties?: ImagePacketProperties; +} + +export interface ImagePacketProperties { + packetIndex: number; + totalPacketCount: number; + packetImageData: string; } /** @public */ @@ -44,11 +52,26 @@ const encode = (message: ChatMessage) => encoder.encode(JSON.stringify(message)) const decode = (message: Uint8Array) => JSON.parse(decoder.decode(message)) as ReceivedChatMessage; +const getImageBlob = (imagePackets: string[] | null[]) => { + const completeImageData = imagePackets.join(''); + const byteCharacters = atob(completeImageData); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + const blob = new Blob([byteArray], { type: 'image/jpeg' }); + return blob; +}; + export function setupChat(room: Room, options?: ChatOptions) { const onDestroyObservable = new Subject(); const { messageDecoder, messageEncoder, channelTopic, updateChannelTopic } = options ?? {}; + // Used to map an image to its packets using id + const chatImagesMap: { [key: string]: string[] | null[] } = {}; + const topic = channelTopic ?? DataTopic.CHAT; const updateTopic = updateChannelTopic ?? DataTopic.CHAT_UPDATE; @@ -95,7 +118,43 @@ export function setupChat(room: Room, options?: ChatOptions) { return [...acc]; } - return [...acc, value]; + // handle image messages + if (value.imagePacketProperties) { + const imageId = value.id; + const totalPacketCount = value.imagePacketProperties.totalPacketCount; + const packetIndex = value.imagePacketProperties.packetIndex; + if (!chatImagesMap[imageId]) { + chatImagesMap[imageId] = new Array(totalPacketCount).fill(null); + } + + if (packetIndex >= 0 && packetIndex < totalPacketCount) { + chatImagesMap[imageId][packetIndex] = value.imagePacketProperties.packetImageData; + } else { + console.error( + `Index ${packetIndex} is out of bounds, Array size with size ${totalPacketCount}`, + ); + } + + // received final image packet, can send a message with image to users + if (packetIndex == totalPacketCount - 1) { + const packets = chatImagesMap[imageId]; + if (packets != null && packets.every((element) => element !== null)) { + const imageBlob = getImageBlob(packets); + value.image = imageBlob; + delete chatImagesMap[imageId]; + return [...acc, value]; + } else { + console.error(`Didn't receive every image packet in order, discarding image message`); + delete chatImagesMap[imageId]; + return [...acc]; + } + } else { + // ignore messages by not sending them to the user until final image packet arrives + return [...acc]; + } + } else { + return [...acc, value]; + } }, []), takeUntil(onDestroyObservable), ); @@ -147,6 +206,43 @@ export function setupChat(room: Room, options?: ChatOptions) { } }; + const sendImagePacket = async ( + message: string, + messageId: string, + packetIndex: number, + totalPacketCount: number, + packetImageData: string, + ) => { + const timestamp = Date.now(); + const imageProp: ImagePacketProperties = { + packetIndex: packetIndex, + totalPacketCount: totalPacketCount, + packetImageData: packetImageData, + }; + const chatMessage: ChatMessage = { + id: messageId, + message, + timestamp, + imagePacketProperties: imageProp, + }; + const encodedMsg = finalMessageEncoder(chatMessage); + isSending$.next(true); + try { + await sendMessage(room.localParticipant, encodedMsg, { + topic: topic, + reliable: true, + }); + messageSubject.next({ + payload: encodedMsg, + topic: topic, + from: room.localParticipant, + }); + return chatMessage; + } finally { + isSending$.next(false); + } + }; + function destroy() { onDestroyObservable.next(); onDestroyObservable.complete(); @@ -154,5 +250,11 @@ export function setupChat(room: Room, options?: ChatOptions) { } room.once(RoomEvent.Disconnected, destroy); - return { messageObservable: messagesObservable, isSendingObservable: isSending$, send, update }; + return { + messageObservable: messagesObservable, + isSendingObservable: isSending$, + send, + sendImagePacket, + update, + }; } diff --git a/packages/react/src/assets/icons/FileAttachIcon.tsx b/packages/react/src/assets/icons/FileAttachIcon.tsx new file mode 100644 index 000000000..f502dbf33 --- /dev/null +++ b/packages/react/src/assets/icons/FileAttachIcon.tsx @@ -0,0 +1,22 @@ +/** + * WARNING: This file was auto-generated by svgr. Do not edit. + */ +import * as React from 'react'; +import type { SVGProps } from 'react'; +/** + * @internal + */ +const SvgFileAttachIcon = (props: SVGProps) => ( + + + +); +export default SvgFileAttachIcon; diff --git a/packages/react/src/assets/icons/index.ts b/packages/react/src/assets/icons/index.ts index 7425b1e9a..032f00698 100644 --- a/packages/react/src/assets/icons/index.ts +++ b/packages/react/src/assets/icons/index.ts @@ -3,6 +3,7 @@ export { default as CameraIcon } from './CameraIcon'; export { default as ChatCloseIcon } from './ChatCloseIcon'; export { default as ChatIcon } from './ChatIcon'; export { default as Chevron } from './Chevron'; +export { default as FileAttachIcon } from './FileAttachIcon'; export { default as FocusToggleIcon } from './FocusToggleIcon'; export { default as GearIcon } from './GearIcon'; export { default as LeaveIcon } from './LeaveIcon'; diff --git a/packages/react/src/components/ChatEntry.tsx b/packages/react/src/components/ChatEntry.tsx index 90382a10a..711da03d8 100644 --- a/packages/react/src/components/ChatEntry.tsx +++ b/packages/react/src/components/ChatEntry.tsx @@ -46,6 +46,10 @@ export const ChatEntry: ( const hasBeenEdited = !!entry.editTimestamp; const time = new Date(entry.timestamp); const locale = navigator ? navigator.language : 'en-US'; + let imgSrc = ''; + if (entry.image) { + imgSrc = URL.createObjectURL(entry.image); + } return (
  • )} - - {formattedMessage} + +
    + {formattedMessage} + {entry.image && } +
    +
    • ); }, diff --git a/packages/react/src/hooks/useChat.ts b/packages/react/src/hooks/useChat.ts index 52f251ad8..dfc3c4bb2 100644 --- a/packages/react/src/hooks/useChat.ts +++ b/packages/react/src/hooks/useChat.ts @@ -28,5 +28,11 @@ export function useChat(options?: ChatOptions) { const isSending = useObservableState(setup.isSendingObservable, false); const chatMessages = useObservableState(setup.messageObservable, []); - return { send: setup.send, update: setup.update, chatMessages, isSending }; + return { + send: setup.send, + update: setup.update, + chatMessages, + isSending, + sendImagePacket: setup.sendImagePacket, + }; } diff --git a/packages/react/src/prefabs/Chat.tsx b/packages/react/src/prefabs/Chat.tsx index 879a32e36..4dd615fc6 100644 --- a/packages/react/src/prefabs/Chat.tsx +++ b/packages/react/src/prefabs/Chat.tsx @@ -6,7 +6,7 @@ import type { MessageFormatter } from '../components/ChatEntry'; import { ChatEntry } from '../components/ChatEntry'; import { useChat } from '../hooks/useChat'; import { ChatToggle } from '../components'; -import { ChatCloseIcon } from '../assets/icons'; +import { FileAttachIcon, ChatCloseIcon } from '../assets/icons'; /** @public */ export interface ChatProps extends React.HTMLAttributes, ChatOptions { @@ -33,20 +33,38 @@ export function Chat({ ...props }: ChatProps) { const inputRef = React.useRef(null); + const thumbnailRef = React.useRef(null); const ulRef = React.useRef(null); + const [imagePackets, setImagePackets] = React.useState([]); + const [selectedImage, setSelectedImage] = React.useState(null); const chatOptions: ChatOptions = React.useMemo(() => { return { messageDecoder, messageEncoder, channelTopic }; }, [messageDecoder, messageEncoder, channelTopic]); - const { send, chatMessages, isSending } = useChat(chatOptions); + const { send, chatMessages, isSending, sendImagePacket } = useChat(chatOptions); const layoutContext = useMaybeLayoutContext(); const lastReadMsgAt = React.useRef(0); async function handleSubmit(event: React.FormEvent) { event.preventDefault(); - if (inputRef.current && inputRef.current.value.trim() !== '') { + if (thumbnailRef.current && imagePackets.length > 0) { + if (sendImagePacket) { + const message = inputRef.current ? inputRef.current.value : ''; + const id = crypto.randomUUID(); + for (let i = 0; i < imagePackets.length; i++) { + await sendImagePacket(message, id, i, imagePackets.length, imagePackets[i]); + } + if (inputRef.current) { + inputRef.current.value = ''; + inputRef.current.focus(); + } + thumbnailRef.current.value = ''; + setImagePackets([]); + setSelectedImage(null); + } + } else if (inputRef.current && inputRef.current.value.trim() !== '') { if (send) { await send(inputRef.current.value); inputRef.current.value = ''; @@ -55,6 +73,50 @@ export function Chat({ } } + function handleImageUploadClicked(event: React.MouseEvent): void { + event.preventDefault(); + if (!thumbnailRef || !thumbnailRef.current) return; + + thumbnailRef.current.click(); + } + + const handleFileRead = async (event: React.ChangeEvent) => { + event.preventDefault(); + if (event.target.files != null) { + const file = event.target.files[0]; + if (file.size > 3000000) { + throw Error('Image file is larger than 3 MB, please select a different image'); + } else { + const fileData = await getFileData(file); + packetizeFileData(String(fileData)); + } + } + }; + + const packetizeFileData = (fileData: string) => { + const maxPacketSize = 50000; // LiveKit doesn't support message length higher than ~6.5 kb + const packets = []; + for (let i = 0; i < fileData.length; i += maxPacketSize) { + packets.push(fileData.slice(i, i + maxPacketSize)); + } + setImagePackets(packets); + inputRef.current?.focus(); + }; + + const getFileData = (file: File) => { + return new Promise((resolve, reject) => { + const fileReader = new FileReader(); + fileReader.readAsDataURL(file); + fileReader.onload = () => { + setSelectedImage(fileReader.result as string); + resolve(String(fileReader.result).split(',')[1]); + }; + fileReader.onerror = (error) => { + reject(error); + }; + }); + }; + React.useEffect(() => { if (ulRef) { ulRef.current?.scrollTo({ top: ulRef.current.scrollHeight }); @@ -119,7 +181,25 @@ export function Chat({ ); })} + {selectedImage && ( +
    + Selected image + +
    + )}
    +
    + handleFileRead(e)} + placeholder="upload image..." + /> ); } diff --git a/packages/styles/assets/icons/file-attach-icon.svg b/packages/styles/assets/icons/file-attach-icon.svg new file mode 100644 index 000000000..9dca2c4c0 --- /dev/null +++ b/packages/styles/assets/icons/file-attach-icon.svg @@ -0,0 +1,18 @@ + + + + + + + + + \ No newline at end of file diff --git a/packages/styles/scss/components/controls/_button.scss b/packages/styles/scss/components/controls/_button.scss index 2b4ce1a74..568b043af 100644 --- a/packages/styles/scss/components/controls/_button.scss +++ b/packages/styles/scss/components/controls/_button.scss @@ -38,6 +38,46 @@ } } +.file-attach-button { + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.5rem; + // padding: 0.625rem 1rem; + color: var(--control-fg); + background-image: none; + background-color: var(--control-bg); + border: 0; + border-radius: var(--border-radius); + cursor: pointer; + white-space: nowrap; + + &:not(:disabled):hover { + background-color: var(--control-hover-bg); + } + + &[aria-pressed='true'] { + background-color: var(--control-active-bg); + + &:hover { + background-color: var(--control-active-hover-bg); + } + } + + /* apply accent color (blue) to screen share button, when screen sharing is active */ + &[data-source='screen_share'][data-enabled='true'] { + background-color: var(--accent-bg); + &:hover { + background-color: var(--accent2); + } + } + + &:disabled { + opacity: 0.5; + } +} + .button-group { display: inline-flex; align-items: stretch; diff --git a/packages/styles/scss/prefabs/chat.scss b/packages/styles/scss/prefabs/chat.scss index 109e4a83c..ad93fb6bc 100644 --- a/packages/styles/scss/prefabs/chat.scss +++ b/packages/styles/scss/prefabs/chat.scss @@ -75,6 +75,16 @@ max-width: calc(100% - 32px); // leave space for edit button } + .image-body { + display: block; + border-radius: 10px; + margin-top: 0.75rem; + margin-bottom: 0.75rem; + width: fit-content; + max-width: calc(100% - 32px); + overflow: auto; + } + &[data-message-origin='local'] { .message-body { background-color: var(--bg5); @@ -109,6 +119,39 @@ width: 100%; } +.thumbnail { + position: relative; + display: inline-block; + width: 50px; + height: 50px; + margin: 10px; +} + +.thumbnail img { + width: 100%; + height: 100%; + object-fit: cover; + border-radius: 25%; +} + +.delete-button { + width: 12px; + height: 12px; + position: absolute; + top: -5px; + right: -5px; + background: red; + color: white; + border: none; + border-radius: 25%; + cursor: pointer; + padding: 0; + font-size: 10px; + display: flex; + align-items: center; + justify-content: center; +} + @media (max-width: 600px) { .chat { position: fixed; From 196ead69463e969fc7b5b579a840f1cefe10133a Mon Sep 17 00:00:00 2001 From: Hamdan Date: Sat, 27 Jul 2024 16:16:14 -0500 Subject: [PATCH 2/2] upadted api change --- packages/react/etc/components-react.api.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index c6deb3194..84c804973 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -123,6 +123,12 @@ export interface ChatMessage { // (undocumented) id: string; // (undocumented) + image?: Blob; + // Warning: (ae-forgotten-export) The symbol "ImagePacketProperties" needs to be exported by the entry point index.d.ts + // + // (undocumented) + imagePacketProperties?: ImagePacketProperties; + // (undocumented) message: string; // (undocumented) timestamp: number; @@ -230,6 +236,11 @@ export interface FeatureFlags { autoSubscription?: boolean; } +// Warning: (ae-internal-missing-underscore) The name "FileAttachIcon" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal (undocumented) +export const FileAttachIcon: (props: SVGProps) => React_2.JSX.Element; + // @public export function FocusLayout({ trackRef, ...htmlProps }: FocusLayoutProps): React_2.JSX.Element; @@ -695,6 +706,7 @@ export function useChat(options?: ChatOptions): { update: (message: string, messageId: string) => Promise; chatMessages: ReceivedChatMessage[]; isSending: boolean; + sendImagePacket: (message: string, messageId: string, packetIndex: number, totalPacketCount: number, packetImageData: string) => Promise; }; // @public