decrease memory footprint of drop_duplicates

VibhuJawa · Nov 21, 2023 · 64ec881 · 64ec881
1 parent 6069f3c
commit 64ec881
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/python/cugraph/cugraph/structure/symmetrize.py b/python/cugraph/cugraph/structure/symmetrize.py
@@ -300,5 +300,5 @@ def _memory_efficient_drop_duplicates(ddf, vertex_col_name, num_workers):
     """
     # drop duplicates has a 5x+ overhead
     ddf = ddf.reset_index(drop=True).repartition(npartitions=num_workers * 2)
-    ddf = ddf.drop_duplicates(subset=[*vertex_col_name], ignore_index=True)
+    ddf = ddf.drop_duplicates(subset=[*vertex_col_name], ignore_index=True, split_out=num_workers*2)
     return ddf