Scan instead of load

equinor · Dec 5, 2024 · dd87b9c · dd87b9c
1 parent a26e843
commit dd87b9c
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/src/ert/storage/migration/to9.py b/src/ert/storage/migration/to9.py
@@ -32,10 +32,15 @@ def _migrate_response_configs_wrt_finalized_keys(path: Path) -> None:
 
                         for real_dir in real_dirs:
                             if (real_dir / f"{response_type}.parquet").exists():
-                                df = polars.read_parquet(
+                                df = polars.scan_parquet(
                                     real_dir / f"{response_type}.parquet"
                                 )
-                                response_keys = df["response_key"].unique().to_list()
+                                response_keys = (
+                                    df.select("response_key")
+                                    .unique()
+                                    .collect()
+                                    .to_list()
+                                )
                                 config["has_finalized_keys"] = True
                                 config["keys"] = sorted(response_keys)
                                 break