fix: too small token limits on claude 3 models

tmokmss · Jun 25, 2024 · 71a7867 · 71a7867
1 parent 66bc3b6
commit 71a7867
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 14 deletions.
diff --git a/dist/index.js b/dist/index.js
diff --git a/src/bot.ts b/src/bot.ts
@@ -64,11 +64,7 @@ export class Bot {
                 // eslint-disable-next-line camelcase
                 anthropic_version: 'bedrock-2023-05-31',
                 // eslint-disable-next-line camelcase
-                max_tokens: 4000,
-                // eslint-disable-next-line camelcase
-                top_p: 0.9,
-                // eslint-disable-next-line camelcase
-                top_k: 250,
+                max_tokens: 4096,
                 temperature: 0,
                 messages: [
                   {

diff --git a/src/limits.ts b/src/limits.ts
@@ -11,8 +11,9 @@ export class TokenLimits {
       this.maxTokens = 100_000
       this.responseTokens = 3000
     } else {
-      this.maxTokens = 4000
-      this.responseTokens = 1000
+      // The latest models usually have this level of limits.
+      this.maxTokens = 200_000
+      this.responseTokens = 4096
     }
     // provide some margin for the request tokens
     this.requestTokens = this.maxTokens - this.responseTokens - 100