Fix json compatible libraries (#3003)

fix json compatible libraries
huggingface · Jul 30, 2024 · df29e85 · df29e85
1 parent c4bc5e7
commit df29e85
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/services/worker/src/worker/job_runners/dataset/compatible_libraries.py b/services/worker/src/worker/job_runners/dataset/compatible_libraries.py
@@ -412,7 +412,7 @@ def get_compatible_libraries_for_json(
         library = "pandas"
         function = "pd.read_json"
         for loading_code in loading_codes:
-            first_file = next(iter(loading_code["arguments"]["splits"].values()))
+            first_file = f"datasets/{dataset}/" + next(iter(loading_code["arguments"]["splits"].values()))
             if ".jsonl" in first_file or HfFileSystem(token=hf_token).open(first_file, "r").read(1) != "[":
                 args = ", lines=True"
                 loading_code["arguments"]["lines"] = True
@@ -696,7 +696,7 @@ def fmt_code(
             args = f"{args}, separator='\\t'"
 
     elif builder_name == "json":
-        first_file = next(iter(loading_codes[0]["arguments"]["splits"].values()))
+        first_file = f"datasets/{dataset}/" + next(iter(loading_codes[0]["arguments"]["splits"].values()))
         is_json_lines = ".jsonl" in first_file or HfFileSystem(token=hf_token).open(first_file, "r").read(1) != "["
 
         if is_json_lines: