From 40d0d6a24985b8ed308f47924afe1b5f438751cf Mon Sep 17 00:00:00 2001
From: debuggerone <83404568+debuggerone@users.noreply.github.com>
Date: Wed, 11 Sep 2024 00:43:24 +0200
Subject: [PATCH] Update token_counter.py

added a comment about tiktoken doesn't support gpt4o-mini
---
 src/core/token_counter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/core/token_counter.py b/src/core/token_counter.py
index beed4b3..4eb8cad 100644
--- a/src/core/token_counter.py
+++ b/src/core/token_counter.py
@@ -2,8 +2,8 @@
 
 
 class TokenCounter:
-    def __init__(self, model="gpt-3.5-turbo"):
-        self.encoder = tiktoken.get_encoding("cl100k_base")
+    def __init__(self, model="gpt-3.5-turbo"): # tiktoken does not provide direct gpt4o-mini support atm
+        self.encoder = tiktoken.get_encoding("cl100k_base") # so we have to improvise
 
     def count_tokens(self, messages):
         total_tokens = 0