test

rapidsai · Sep 18, 2024 · 91b03c6 · 91b03c6
1 parent bc0fc18
commit 91b03c6
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 17 deletions.
diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh
@@ -33,7 +33,7 @@ bash python/cudf/cudf/pandas/scripts/run-pandas-tests.sh \
   -m "not slow" \
   --max-worker-restart=3 \
   --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-pandas.xml" \
-  --dist loadfile \
+  --dist worksteal \
   --report-log=${PANDAS_TESTS_BRANCH}.json 2>&1
 
 SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-${RAPIDS_FULL_VERSION}-results.json

diff --git a/python/cudf/cudf/pandas/scripts/summarize-test-results.py b/python/cudf/cudf/pandas/scripts/summarize-test-results.py
@@ -60,23 +60,46 @@ def get_per_module_results(log_file_name):
                 per_module_results[module_name]["total"] += 1
                 per_module_results[module_name][outcome] += 1
 
+    directory = os.path.dirname(log_file_name)
+    pattern = os.path.join(directory, "function_call_counts_worker_*.json")
+    matching_files = glob.glob(pattern)
+    function_call_counts = {}
+    for file in matching_files:
+        with open(file) as f:
+            function_call_count = json.load(f)
+        if not function_call_counts:
+            function_call_counts.update(function_call_count)
+        else:
+            for key, value in function_call_count.items():
+                function_call_counts[key]["_slow_function_call"] += value.get(
+                    "_slow_function_call", 0
+                )
+                function_call_counts[key]["_fast_function_call"] += value.get(
+                    "_fast_function_call", 0
+                )
+            # per_module_results[key]["_slow_function_call"] = (
+            #     per_module_results[key].get("_slow_function_call", 0)
+            #     + function_call_counts.get("_slow_function_call", 0)
+            # )
+            # per_module_results[key]["_fast_function_call"] = (
+            #     per_module_results[key].get("_fast_function_call", 0)
+            #     + function_call_counts.get("_fast_function_call", 0)
+            # )
     for key, value in per_module_results.items():
-        processed_name = key.replace("/", "__") + "_*_metrics.json"
-        # Assuming the directory is the same as the module name's directory
-        directory = os.path.dirname(log_file_name)
-        pattern = os.path.join(directory, processed_name)
-        matching_files = glob.glob(pattern)
-        for file in matching_files:
-            with open(file) as f:
-                function_call_counts = json.load(f)
-            per_module_results[key]["_slow_function_call"] = (
-                per_module_results[key].get("_slow_function_call", 0)
-                + function_call_counts.get("_slow_function_call", 0)
-            )
-            per_module_results[key]["_fast_function_call"] = (
-                per_module_results[key].get("_fast_function_call", 0)
-                + function_call_counts.get("_fast_function_call", 0)
-            )
+        # processed_name = key.replace("/", "__") + "_*_metrics.json"
+        # # Assuming the directory is the same as the module name's directory
+        # directory = os.path.dirname(log_file_name)
+        # pattern = os.path.join(directory, processed_name)
+        # matching_files = glob.glob(pattern)
+        # for file in matching_files:
+        #     with open(file) as f:
+        #         function_call_counts = json.load(f)
+        per_module_results[key]["_slow_function_call"] = function_call_counts[
+            key
+        ].get("_slow_function_call", 0)
+        per_module_results[key]["_fast_function_call"] = function_call_counts[
+            key
+        ].get("_fast_function_call", 0)
     return per_module_results