force download when previous file is obsolete

huggingface · Oct 5, 2023 · 2547434 · 2547434
1 parent 6e19231
commit 2547434
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 0 deletions.
diff --git a/libs/libapi/src/libapi/duckdb.py b/libs/libapi/src/libapi/duckdb.py
@@ -83,6 +83,7 @@ def download_index_file(
         local_dir_use_symlinks=False,
         token=hf_token,
         cache_dir=cache_folder,
+        force_download=True,
     )
 
 

diff --git a/services/worker/src/worker/job_runners/split/descriptive_statistics.py b/services/worker/src/worker/job_runners/split/descriptive_statistics.py
@@ -443,6 +443,7 @@ def compute_descriptive_statistics_response(
             local_dir_use_symlinks=False,
             token=hf_token,
             cache_dir=local_parquet_directory,
+            force_download=True,
         )
 
     local_parquet_glob_path = Path(local_parquet_directory) / config / f"{split}/*.parquet"

diff --git a/services/worker/src/worker/job_runners/split/duckdb_index.py b/services/worker/src/worker/job_runners/split/duckdb_index.py
@@ -171,6 +171,7 @@ def compute_index_rows(
             local_dir_use_symlinks=False,
             token=hf_token,
             cache_dir=duckdb_index_file_directory,
+            force_download=True,
         )
 
     all_split_parquets = f"{duckdb_index_file_directory}/{config}/{split_directory}/*.parquet"