Skip to content

Commit

Permalink
Merge pull request #4 from Stevenic/main
Browse files Browse the repository at this point in the history
update fork
  • Loading branch information
debuggerone authored Sep 10, 2024
2 parents 0fdbd00 + 44c095a commit 971936d
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions src/core/token_counter.py
Original file line number Diff line number Diff line change
@@ -2,8 +2,8 @@


class TokenCounter:
def __init__(self, model="gpt-3.5-turbo"):
self.encoder = tiktoken.get_encoding("cl100k_base")
def __init__(self, model="gpt-3.5-turbo"): # tiktoken does not provide direct gpt4o-mini support atm
self.encoder = tiktoken.get_encoding("cl100k_base") # so we have to improvise

def count_tokens(self, messages):
total_tokens = 0

0 comments on commit 971936d

Please sign in to comment.