From 0151fd863223fcbe80984a2f6c1bc85ada239123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=A6Ltorio?= Date: Mon, 7 Oct 2024 10:56:35 +0200 Subject: [PATCH] Refactor Flluentui v9 display --- tokenCount/index.html | 2 +- tokenCount/src/App.tsx | 11 ++- tokenCount/src/components/TextAnalyzer.tsx | 80 ++++++++++++++++------ 3 files changed, 66 insertions(+), 27 deletions(-) diff --git a/tokenCount/index.html b/tokenCount/index.html index 3a353d9..d183d77 100644 --- a/tokenCount/index.html +++ b/tokenCount/index.html @@ -3,7 +3,7 @@ - Llama 3.1 Token Counter + Llama 3.1 Tokenizer and Counter
diff --git a/tokenCount/src/App.tsx b/tokenCount/src/App.tsx index 350828a..440e8ff 100644 --- a/tokenCount/src/App.tsx +++ b/tokenCount/src/App.tsx @@ -10,9 +10,8 @@ import { const useStyles = makeStyles({ card: { - margin: "auto", - width: "720px", - maxWidth: "100%", + width: "80vw", + maxWidth: "1024px", }, footer: { }, @@ -21,17 +20,17 @@ function App() { const styles = useStyles(); return ( <> - + - LLama 3.1 Token counter + LLama 3.1 Tokenizer and counter } /> - Copyright 2024 Ronan Le Meillat + Copyright 2024 Ronan Le Meillat diff --git a/tokenCount/src/components/TextAnalyzer.tsx b/tokenCount/src/components/TextAnalyzer.tsx index ff14e49..1304828 100644 --- a/tokenCount/src/components/TextAnalyzer.tsx +++ b/tokenCount/src/components/TextAnalyzer.tsx @@ -1,5 +1,12 @@ import React, { useState, useId, useRef } from "react"; -import { Textarea, Button, makeStyles, CardPreview } from "@fluentui/react-components"; +import { + Textarea, + Button, + makeStyles, + CardPreview, + Field, + Text, +} from "@fluentui/react-components"; import { Buffer } from "buffer"; import { SentencePieceProcessor, @@ -12,23 +19,36 @@ globalThis.Buffer = Buffer; const useStyles = makeStyles({ root: { + // width: "100%", + // margin: "auto", }, + field: {}, textarea: { + margin: "1em", + display: "block", minHeight: "300px", - margin: "10px", }, - button: { - margin: "auto", - width: "30%", + textareain: { + width: "100%", + maxHeight: 'unset', + minHeight: '300px', + maxWidth: 'unset', + }, + result: { + margin: "1em", }, + button: {}, }); function TextAnalyzer() { const styles = useStyles(); const textAnalyserId = useId(); const textAreaRef = useRef(null); + const resultRef = useRef(null); + const [isVisible, setIsVisible] = useState(false); const [text, setText] = useState(""); - const [tokenCount, setWordCount] = useState(0); + const [tokenCount, setTokenCount] = useState(0); + const [tokens, setTokens] = useState(""); const handleDrop = (event: React.DragEvent) => { event.preventDefault(); @@ -55,26 +75,46 @@ function TextAnalyzer() { processor.loadFromB64StringModel(llama_3_1_tokeniser_b64).then(() => { const cleanedText = cleanText(text); const tokens = processor.encodeIds(cleanedText); + const decoded = processor.encodePieces(cleanedText); const count = tokens.length; - setWordCount(count); + setTokenCount(count); + setTokens(decoded.join(" ")); }); }; + const toggleVisibility = () => { + setIsVisible(!isVisible); + }; + return ( - + + + + {tokenCount > 0 && ( +
+ {tokenCount} tokens + :
{tokens} +
+ )} + ); }