remove gpt-tokenizer

Signed-off-by: Mengxin Liu <[email protected]>
oilbeater · Sep 10, 2024 · a8d8405 · a8d8405
1 parent 161e5bb
commit a8d8405
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 19 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -21,7 +21,6 @@
     "wrangler": "^3.60.3"
   },
   "dependencies": {
-    "gpt-tokenizer": "^2.2.1",
     "hono": "^4.5.5"
   }
 }
diff --git a/src/index.ts b/src/index.ts
@@ -1,5 +1,4 @@
 import { Hono, Context } from 'hono'
-import { encodeChat } from 'gpt-tokenizer'
 
 type Bindings = {
   MALACCA: AnalyticsEngineDataset,
@@ -40,7 +39,7 @@ async function handleChat(c: Context) {
     return c.text('Internal Server Error', 500)
   }
   const decoder = new TextDecoder('utf-8');
-  let prompt_tokens = encodeChat(body['messages']).length;
+  let prompt_tokens = 0;
   let completion_tokens = 0;
 
   if (response.status == 200) {