Merge branch 'main' into dataset-git-version

huggingface · Oct 17, 2023 · f4d5d1b · f4d5d1b
2 parents 1367a2c + 3160cdd
commit f4d5d1b
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/services/worker/src/worker/job_runners/split/descriptive_statistics.py b/services/worker/src/worker/job_runners/split/descriptive_statistics.py
@@ -481,7 +481,13 @@ def compute_descriptive_statistics_response(
     logging.info(f"Original number of threads={n_threads}")
     con.sql("SET threads TO 8;")
     n_threads = con.sql("SELECT current_setting('threads')").fetchall()[0][0]
-    logging.info(f"Number of threads={n_threads}")
+    logging.info(f"Current number of threads={n_threads}")
+
+    max_memory = con.sql("SELECT current_setting('max_memory');").fetchall()[0][0]
+    logging.info(f"Original {max_memory=}")
+    con.sql("SET max_memory TO '28gb';")
+    max_memory = con.sql("SELECT current_setting('max_memory');").fetchall()[0][0]
+    logging.info(f"Current {max_memory=}")
 
     logging.info("Loading data into in-memory table. ")
     create_table_command = CREATE_TABLE_COMMAND.format(