Bugfix: a case that files' encodings can not be detected by chardet (#61

)
aigc-apps · Jun 12, 2024 · b492801 · b492801
1 parent a707e6d
commit b492801
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/pai_rag/integrations/readers/pai_csv_reader.py b/src/pai_rag/integrations/readers/pai_csv_reader.py
@@ -141,7 +141,7 @@ def load_data(
             with fs.open(file) as f:
                 encoding = chardet.detect(f.read(100000))["encoding"]
                 f.seek(0)
-                if "GB" in encoding.upper():
+                if encoding is not None and "GB" in encoding.upper():
                     self._pandas_config["encoding"] = "GB18030"
                 try:
                     df = pd.read_csv(f, **self._pandas_config)
@@ -155,7 +155,7 @@ def load_data(
             with open(file, "rb") as f:
                 encoding = chardet.detect(f.read(100000))["encoding"]
                 f.seek(0)
-                if "GB" in encoding.upper():
+                if encoding is not None and "GB" in encoding.upper():
                     self._pandas_config["encoding"] = "GB18030"
                 try:
                     df = pd.read_csv(file, **self._pandas_config)