Merge pull request #263 from aurelio-labs/james/encoder-disallowed-sp…

…ecial fix: James/encoder disallowed special
aurelio-labs · Apr 28, 2024 · 29762f0 · 29762f0
2 parents eb9ebe3 + cc1430d
commit 29762f0
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "semantic-router"
-version = "0.0.36"
+version = "0.0.37"
 description = "Super fast semantic router for AI decision making"
 authors = [
     "James Briggs <[email protected]>",

diff --git a/semantic_router/__init__.py b/semantic_router/__init__.py
@@ -4,4 +4,4 @@
 
 __all__ = ["RouteLayer", "HybridRouteLayer", "Route", "LayerConfig"]
 
-__version__ = "0.0.36"
+__version__ = "0.0.37"
diff --git a/semantic_router/encoders/openai.py b/semantic_router/encoders/openai.py
@@ -114,7 +114,8 @@ def __call__(self, docs: List[str], truncate: bool = True) -> List[List[float]]:
         return embeddings
 
     def _truncate(self, text: str) -> str:
-        tokens = self._token_encoder.encode(text)
+        # we use encode_ordinary as faster equivalent to encode(text, disallowed_special=())
+        tokens = self._token_encoder.encode_ordinary(text)
         if len(tokens) > self.token_limit:
             logger.warning(
                 f"Document exceeds token limit: {len(tokens)} > {self.token_limit}"