fix exl bug: sometimes piece_to_id not populated, but get_piece_to_id…

…() still works
lapp0 · Oct 4, 2024 · 86d1d10 · 86d1d10
1 parent 8bd29f5
commit 86d1d10
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/outlines/models/exllamav2.py b/outlines/models/exllamav2.py
@@ -220,9 +220,8 @@ def token_generator() -> Iterator[str]:
         return token_generator()
 
 
-# Taken from https://github.com/lapp0/exllamav2/pull/1/files#diff-26f303de07c10aad998e33d3df52581643673a598162cc4b35ef051f52d7c60b
 def patch_tokenizer(tokenizer):
-    tokenizer.vocabulary = tokenizer.piece_to_id
+    tokenizer.vocabulary = tokenizer.piece_to_id_dict()
     tokenizer.special_tokens = set(tokenizer.extended_piece_to_id)
     tokenizer.convert_token_to_string = lambda t: t
     return tokenizer