Merge pull request #10 from Stevenic/main

resolve merge conflict

src/core/token_counter.py

@@ -17,7 +17,8 @@ def __init__(self, model="gpt-3.5-turbo"):
 
         Args:
             model (str): The model name to use for token encoding.
-                        Note: tiktoken does not directly support "gpt-4o-mini" at the moment.
+                        Note: tiktoken does not directly support "gpt-4o-mini" at the moment,
+                        so "cl100k_base" is used as a workaround.
         """
         self.encoder = tiktoken.get_encoding("cl100k_base")
 
@@ -35,3 +36,4 @@ def count_tokens(self, messages):
         for message in messages:
             total_tokens += len(self.encoder.encode(message["content"]))
         return total_tokens
+