Add more accurate RSS high water mark measurement for Linux

core/dbt/cli/requires.py

@@ -32,6 +32,7 @@
 from dbt.plugins import set_up_plugin_manager
 from dbt.profiler import profiler
 from dbt.tracking import active_user, initialize_from_flags, track_run
+from dbt.utils import try_get_max_rss_kb
 from dbt.version import installed as installed_version
 from dbt_common.clients.system import get_env
 from dbt_common.context import get_invocation_context, set_invocation_context
@@ -148,14 +149,17 @@ def wrapper(*args, **kwargs):
                 import resource
 
                 rusage = resource.getrusage(resource.RUSAGE_SELF)
+                max_rss = try_get_max_rss_kb()
+                if max_rss is None:
+                    max_rss = rusage.ru_maxrss
                 fire_event(
                     ResourceReport(
                         command_name=ctx.command.name,
                         command_success=success,
                         command_wall_clock_time=time.perf_counter() - start_func,
                         process_user_time=rusage.ru_utime,
                         process_kernel_time=rusage.ru_stime,
-                        process_mem_max_rss=rusage.ru_maxrss,
+                        process_mem_max_rss=max_rss,
                         process_in_blocks=rusage.ru_inblock,
                         process_out_blocks=rusage.ru_oublock,
                     ),

core/dbt/utils.py

@@ -386,3 +386,21 @@ def strtobool(val: str) -> bool:
         return False
     else:
         raise ValueError("invalid truth value %r" % (val,))
+
+
+def try_get_max_rss_kb() -> Optional[int]:
+    """Attempts to get the high water mark for this process's memory use via
+    the most reliable and accurate mechanism available through the host OS.
+    Currently only implemented for Linux."""
+    try:
+        # On Linux, the most reliable documented mechanism for getting the RSS
+        # high-water-mark comes from the line confusingly labeled VmHWM in the
+        # /proc/self/status virtual file.
+        with open("/proc/self/status") as f:
+            for line in f:
+                if line.startswith("VmHWM:"):
+                    return int(str.split(line)[1])
+    except Exception:
+        pass
+
+    return None