diff --git a/toolkit/dataloader_mixins.py b/toolkit/dataloader_mixins.py index 80f00cf0..cdea9a6d 100644 --- a/toolkit/dataloader_mixins.py +++ b/toolkit/dataloader_mixins.py @@ -121,7 +121,7 @@ def get_caption_item(self: 'AiToolkitDataset', index): break if os.path.exists(prompt_path): - with open(prompt_path, 'r', encoding='utf-8') as f: + with open(prompt_path, 'r', encoding='utf-8', errors='ignore') as f: prompt = f.read() # check if is json if prompt_path.endswith('.json'): @@ -301,7 +301,7 @@ def load_caption(self: 'FileItemDTO', caption_dict: Union[dict, None]): short_caption = None if os.path.exists(prompt_path): - with open(prompt_path, 'r', encoding='utf-8') as f: + with open(prompt_path, 'r', encoding='utf-8', errors='ignore') as f: prompt = f.read() short_caption = None if prompt_path.endswith('.json'): @@ -1127,7 +1127,7 @@ def __init__(self: 'FileItemDTO', *args, **kwargs): caption_path = file_path_no_ext + '.json' if not os.path.exists(caption_path): raise Exception(f"Error: caption file not found for poi: {caption_path}") - with open(caption_path, 'r', encoding='utf-8') as f: + with open(caption_path, 'r', encoding='utf-8', errors='ignore') as f: json_data = json.load(f) if 'poi' not in json_data: print(f"Warning: poi not found in caption file: {caption_path}")