Refactor Flluentui v9 display

sctg-development · Oct 7, 2024 · 0151fd8 · 0151fd8
1 parent cf680e2
commit 0151fd8
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 27 deletions.
diff --git a/tokenCount/index.html b/tokenCount/index.html
@@ -3,7 +3,7 @@
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Llama 3.1 Token Counter</title>
+    <title>Llama 3.1 Tokenizer and Counter</title>
   </head>
   <body>
     <div id="root"></div>

diff --git a/tokenCount/src/App.tsx b/tokenCount/src/App.tsx
@@ -10,9 +10,8 @@ import {
 
 const useStyles = makeStyles({
   card: {
-    margin: "auto",
-    width: "720px",
-    maxWidth: "100%",
+    width: "80vw",
+    maxWidth: "1024px",
   },
   footer: {
   },
@@ -21,17 +20,17 @@ function App() {
   const styles = useStyles();
   return (
     <>
-      <Card className={styles.card}>
+      <Card className={styles.card} size="large">
         <CardHeader
           header={
             <Body1>
-              <b>LLama 3.1 Token counter</b>
+              <b>LLama 3.1 Tokenizer and counter</b>
             </Body1>
           }
         />
         <TextAnalyzer />
         <CardFooter className={styles.footer}>
-          <b>Copyright 2024 Ronan Le Meillat</b>
+          <b>Copyright 2024 <a href="https://github.com/sctg-development/sentencepiece-js">Ronan Le Meillat</a></b>
         </CardFooter>
       </Card>
     </>

diff --git a/tokenCount/src/components/TextAnalyzer.tsx b/tokenCount/src/components/TextAnalyzer.tsx
@@ -1,5 +1,12 @@
 import React, { useState, useId, useRef } from "react";
-import { Textarea, Button, makeStyles, CardPreview } from "@fluentui/react-components";
+import {
+  Textarea,
+  Button,
+  makeStyles,
+  CardPreview,
+  Field,
+  Text,
+} from "@fluentui/react-components";
 import { Buffer } from "buffer";
 import {
   SentencePieceProcessor,
@@ -12,23 +19,36 @@ globalThis.Buffer = Buffer;
 
 const useStyles = makeStyles({
   root: {
+    // width: "100%",
+    // margin: "auto",
   },
+  field: {},
   textarea: {
+    margin: "1em",
+    display: "block",
     minHeight: "300px",
-    margin: "10px",
   },
-  button: {
-    margin: "auto",
-    width: "30%",
+  textareain: {
+    width: "100%",
+    maxHeight: 'unset',
+    minHeight: '300px',
+    maxWidth: 'unset',
+  },
+  result: {
+    margin: "1em",
   },
+  button: {},
 });
 
 function TextAnalyzer() {
   const styles = useStyles();
   const textAnalyserId = useId();
   const textAreaRef = useRef<HTMLTextAreaElement>(null);
+  const resultRef = useRef<HTMLSpanElement>(null);
+  const [isVisible, setIsVisible] = useState(false);
   const [text, setText] = useState("");
-  const [tokenCount, setWordCount] = useState(0);
+  const [tokenCount, setTokenCount] = useState(0);
+  const [tokens, setTokens] = useState<string>("");
 
   const handleDrop = (event: React.DragEvent<HTMLTextAreaElement>) => {
     event.preventDefault();
@@ -55,26 +75,46 @@ function TextAnalyzer() {
     processor.loadFromB64StringModel(llama_3_1_tokeniser_b64).then(() => {
       const cleanedText = cleanText(text);
       const tokens = processor.encodeIds(cleanedText);
+      const decoded = processor.encodePieces(cleanedText);
       const count = tokens.length;
-      setWordCount(count);
+      setTokenCount(count);
+      setTokens(decoded.join(" "));
     });
   };
 
+  const toggleVisibility = () => {
+    setIsVisible(!isVisible);
+  };
+
   return (
     <CardPreview className={styles.root}>
-      <Textarea
-        className={styles.textarea}
-        appearance="outline"
-        id={textAnalyserId}
-        ref={textAreaRef}
-        placeholder="Enter text or drop a text file"
-        onDrop={handleDrop}
-        onChange={(event) => setText(event.target.value)}
-      />
-      <Button appearance="primary" onClick={calculateTokenCount} className={styles.button}>
-        Token count
-      </Button>
-      {tokenCount > 0 && <div>{tokenCount} tokens</div>}
+      <Field size="small" className={styles.field}>
+        <Textarea
+          className={styles.textarea}
+          textarea={{ className: styles.textareain }}
+          appearance="outline"
+          id={textAnalyserId}
+          ref={textAreaRef}
+          placeholder="Enter text or drop a text file"
+          onDrop={handleDrop}
+          onChange={(event) => setText(event.target.value)}
+        ></Textarea>
+      </Field>
+      <Text style={{width:"unset"}} className={styles.result} block>
+        <Button
+          appearance="primary"
+          onClick={calculateTokenCount}
+          className={styles.button}
+        >
+          Token count
+        </Button>
+        {tokenCount > 0 && (
+          <div>
+            {tokenCount} <a onClick={toggleVisibility} style={{ cursor: 'pointer' }}>tokens</a> 
+            <span ref={resultRef} style={{ display: isVisible ? 'inline' : 'none' }}>:<br/>{tokens}</span>
+          </div>
+        )}
+      </Text>
     </CardPreview>
   );
 }