Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed Issue: #1977 #2181

Merged
merged 5 commits into from
Dec 9, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion bertopic/representation/_cohere.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from scipy.sparse import csr_matrix
from typing import Mapping, List, Tuple, Union, Callable
from bertopic.representation._base import BaseRepresentation
from bertopic.representation._utils import truncate_document
from bertopic.representation._utils import truncate_document, validate_truncate_document_parameters


DEFAULT_PROMPT = """
Expand Down Expand Up @@ -126,6 +126,8 @@ def __init__(
self.tokenizer = tokenizer
self.prompts_ = []

_ = validate_truncate_document_parameters(self.tokenizer, self.doc_length)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you returning the _ for a specific reason? Looking at the validate_truncate_document_parameters, there doesn't seem anything that is returned.


def extract_topics(
self,
topic_model,
Expand Down
4 changes: 3 additions & 1 deletion bertopic/representation/_langchain.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from typing import Callable, Mapping, List, Tuple, Union

from bertopic.representation._base import BaseRepresentation
from bertopic.representation._utils import truncate_document
from bertopic.representation._utils import truncate_document, validate_truncate_document_parameters

DEFAULT_PROMPT = "What are these documents about? Please give a single label."

Expand Down Expand Up @@ -148,6 +148,8 @@ def __init__(
self.diversity = diversity
self.doc_length = doc_length
self.tokenizer = tokenizer
_ = validate_truncate_document_parameters(self.tokenizer, self.doc_length)


def extract_topics(
self,
Expand Down
3 changes: 2 additions & 1 deletion bertopic/representation/_llamacpp.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from llama_cpp import Llama
from typing import Mapping, List, Tuple, Any, Union, Callable
from bertopic.representation._base import BaseRepresentation
from bertopic.representation._utils import truncate_document
from bertopic.representation._utils import truncate_document, validate_truncate_document_parameters


DEFAULT_PROMPT = """
Expand Down Expand Up @@ -118,6 +118,7 @@ def __init__(
self.tokenizer = tokenizer

self.prompts_ = []
_ = validate_truncate_document_parameters(self.tokenizer, self.doc_length)

def extract_topics(
self,
Expand Down
3 changes: 3 additions & 0 deletions bertopic/representation/_openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from bertopic.representation._utils import (
retry_with_exponential_backoff,
truncate_document,
validate_truncate_document_parameters
)


Expand Down Expand Up @@ -180,6 +181,8 @@ def __init__(
if not self.generator_kwargs.get("stop") and not chat:
self.generator_kwargs["stop"] = "\n"

_ = validate_truncate_document_parameters(self.tokenizer, self.doc_length)

def extract_topics(
self,
topic_model,
Expand Down
3 changes: 2 additions & 1 deletion bertopic/representation/_textgeneration.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from transformers.pipelines.base import Pipeline
from typing import Mapping, List, Tuple, Any, Union, Callable
from bertopic.representation._base import BaseRepresentation
from bertopic.representation._utils import truncate_document
from bertopic.representation._utils import truncate_document, validate_truncate_document_parameters


DEFAULT_PROMPT = """
Expand Down Expand Up @@ -114,6 +114,7 @@ def __init__(
self.tokenizer = tokenizer

self.prompts_ = []
_ = validate_truncate_document_parameters(self.tokenizer, self.doc_length)

def extract_topics(
self,
Expand Down
12 changes: 11 additions & 1 deletion bertopic/representation/_utils.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import random
import time
from typing import Union


def truncate_document(topic_model, doc_length, tokenizer, document: str):
def truncate_document(topic_model, doc_length: Union[int, None], tokenizer: Union[str, callable], document: str) -> str:
"""Truncate a document to a certain length.

If you want to add a custom tokenizer, then it will need to have a `decode` and
Expand Down Expand Up @@ -57,6 +58,15 @@ def decode(self, doc_chunks):
return truncated_document
return document

def validate_truncate_document_parameters(tokenizer, doc_length) -> Union[None, ValueError]:
"""validates parameters that are used in the function `truncate_document`"""
if tokenizer is None and doc_length is not None:
raise ValueError(
"Please select from one of the valid options for the `tokenizer` parameter: \n"
"{'char', 'whitespace', 'vectorizer'} \n"
"If `tokenizer` is of type callable ensure it has methods to encode and decode a document \n"
)
Comment on lines +64 to +69
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we also include a check for the opposite? That someone does use a tokenizer but not a doc_length?



def retry_with_exponential_backoff(
func,
Expand Down