Allow runbms skipping emitting heapsize modifiers for some configs

docs/src/changelog.md

@@ -12,6 +12,11 @@
 
 ### Security
 
+## [`v0.4.5` (2024-05-23)](https://github.com/anupli/running-ng/releases/tag/v0.4.5)
+### Added
+#### Modifiers
+- `NoImplicitHeapsizeModifier`
+
 ## [`v0.4.4` (2023-11-23)](https://github.com/anupli/running-ng/releases/tag/v0.4.4)
 ### Fixed
 #### Benchmark Suites

docs/src/references/modifier.md

@@ -96,3 +96,14 @@ If a companion program also exist for the benchmark suite you use, this companio
 `val`: a single string with [shell-like syntax](https://docs.python.org/3/library/shlex.html#shlex.split).
 Multiple arguments are space separated.
 Environment variables will be expanded.
+
+## `NoImplicitHeapsizeModifier` (preview ⚠️)
+`runbms` specific.
+
+### Description
+Normally `runbms` will iterate through a set of heap sizes, either specific multiples of the minheap of each benchmark via `-s`, or spreading the multiples across 1~`heap_range` (using `N` and optionally `ns`).
+
+This modifier prevents `runbms` from applying the heap sizes for certain configs, which is useful, for example, for running `NoGC` or `EpsilonGC`.
+
+### Keys
+No argument is allowed.

src/running/__version__.py

@@ -1,2 +1,2 @@
-VERSION = (0, 4, 4)
+VERSION = (0, 4, 5)
 __VERSION__ = ".".join(map(str, VERSION))

src/running/command/runbms.py

@@ -20,6 +20,7 @@
     get_logged_in_users,
     config_index_to_chr,
     config_str_encode,
+    dont_emit_heapsize_modifier,
 )
 import socket
 from datetime import datetime
@@ -453,6 +454,42 @@ def run(args):
                 p.set_runbms_dir(runbms_dir)
                 p.set_log_dir(log_dir)
 
+        configs_no_heapsize = [
+            c for c in configs if dont_emit_heapsize_modifier(configuration, c)
+        ]
+        configs_with_heapsize = [
+            c for c in configs if not dont_emit_heapsize_modifier(configuration, c)
+        ]
+
+        # Simple case
+        if not slice and N is None:
+            # run all configs without specifying heap size
+            run_one_hfac(
+                invocations,
+                None,  # not specifying heap size
+                suites,
+                benchmarks,
+                configs,
+                Path(runbms_dir),
+                log_dir,
+            )
+            # early return
+            return True
+
+        # In all other cases, we will first run configs that don't want
+        # implicit heapsize modifiers
+        logging.info("Running all configs with NoImplicitHeapSizeModifier set")
+        run_one_hfac(
+            invocations,
+            None,  # not specifying heap size
+            suites,
+            benchmarks,
+            configs_no_heapsize,
+            Path(runbms_dir),
+            log_dir,
+        )
+
+        # Helper function for running benchmarks using multiple heap factors
         def run_hfacs(hfacs):
             logging.info(
                 "hfacs: {}".format(", ".join([hfac_str(hfac) for hfac in hfacs]))
@@ -463,35 +500,31 @@ def run_hfacs(hfacs):
                     hfac,
                     suites,
                     benchmarks,
-                    configs,
+                    configs_with_heapsize,
                     Path(runbms_dir),
                     log_dir,
                 )
                 print()
 
+        # Helper function for using the heap factor spreading algorithm
         def run_N_ns(N, ns):
             hfacs = get_hfacs(heap_range, spread_factor, N, ns)
             run_hfacs(hfacs)
 
-        if slice:
+        if slice:  # specified -s, we respect that first
+            if N is not None:
+                logging.warning(
+                    "You specified both N={} and -s {}, N is ignored.".format(
+                        N,
+                        ",".join([str(s) for s in slice]),
+                    )
+                )
             run_hfacs(slice)
-            return True
-
-        if N is None:
-            run_one_hfac(
-                invocations,
-                None,
-                suites,
-                benchmarks,
-                configs,
-                Path(runbms_dir),
-                log_dir,
-            )
-            return True
-
-        if len(ns) == 0:
-            fillin(run_N_ns, round(math.log2(N)))
         else:
-            run_N_ns(N, ns)
+            assert N is not None
+            if len(ns) == 0:
+                fillin(run_N_ns, round(math.log2(N)))
+            else:
+                run_N_ns(N, ns)
 
         return True

src/running/config/runbms_example.yml

@@ -1,38 +1,17 @@
 includes:
   - "$RUNNING_NG_PACKAGE_DATA/base/runbms.yml"
 
-overrides:
-  "suites.dacapo2006.timing_iteration": 5
-  "suites.dacapobach.timing_iteration": 5
-  "suites.dacapo2006.callback": "probe.Dacapo2006Callback"
-  "suites.dacapobach.callback": "probe.DacapoBachCallback"
+benchmarks:
+  dacapochopin:
+    - fop
 
 modifiers:
-  probes_cp:
-    type: JVMClasspath
-    val: "/home/zixianc/MMTk-Dev/evaluation/probes /home/zixianc/MMTk-Dev/evaluation/probes/probes.jar"
-  probes_native:
+  heap4g:
     type: JVMArg
-    val: "-Djava.library.path=/home/zixianc/MMTk-Dev/evaluation/probes"
-  probes_rustmmtk:
-    type: JVMArg
-    val: "-Dprobes=RustMMTk"
-  value_opts_example:
-    type: EnvVar
-    var: "FOOBAR{0}"
-    val: "BUZZ{1}"
-
-runtimes:
-  jdk-mmtk:
-    type: OpenJDK
-    release: 11
-    home: "/home/zixianc/MMTk-Dev/evaluation/build/jdk-mmtk/jdk"
-
-benchmarks:
-  dacapo2006:
-    - eclipse
-  dacapobach:
-    - fop
+    val: -Xms4G -Xmx4G
+  no_heapsize:
+    type: NoImplicitHeapsizeModifier
 
 configs:
-  - "jdk-mmtk|ms|s|c2|ss|tph|probes_cp|probes_rustmmtk|probes_native|value_opts_example-42-43"
+  - "temurin-21|dacapochopin_jdk21"
+  - "temurin-21|dacapochopin_jdk21|epsilon|no_heapsize|heap4g"

src/running/config/runbms_example_probes.yml

@@ -0,0 +1,38 @@
+includes:
+  - "$RUNNING_NG_PACKAGE_DATA/base/runbms.yml"
+
+overrides:
+  "suites.dacapo2006.timing_iteration": 5
+  "suites.dacapobach.timing_iteration": 5
+  "suites.dacapo2006.callback": "probe.Dacapo2006Callback"
+  "suites.dacapobach.callback": "probe.DacapoBachCallback"
+
+modifiers:
+  probes_cp:
+    type: JVMClasspath
+    val: "/home/zixianc/MMTk-Dev/evaluation/probes /home/zixianc/MMTk-Dev/evaluation/probes/probes.jar"
+  probes_native:
+    type: JVMArg
+    val: "-Djava.library.path=/home/zixianc/MMTk-Dev/evaluation/probes"
+  probes_rustmmtk:
+    type: JVMArg
+    val: "-Dprobes=RustMMTk"
+  value_opts_example:
+    type: EnvVar
+    var: "FOOBAR{0}"
+    val: "BUZZ{1}"
+
+runtimes:
+  jdk-mmtk:
+    type: OpenJDK
+    release: 11
+    home: "/home/zixianc/MMTk-Dev/evaluation/build/jdk-mmtk/jdk"
+
+benchmarks:
+  dacapo2006:
+    - eclipse
+  dacapobach:
+    - fop
+
+configs:
+  - "jdk-mmtk|ms|s|c2|ss|tph|probes_cp|probes_rustmmtk|probes_native|value_opts_example-42-43"

src/running/modifier.py

@@ -180,3 +180,12 @@ def __init__(self, value_opts=None, **kwargs):
 
     def __str__(self) -> str:
         return "{} JuliaArg {}".format(super().__str__(), self.val)
+
+
+@register(Modifier)
+class NoImplicitHeapsizeModifier(Modifier):
+    def __init__(self, value_opts=None, **kwargs):
+        super().__init__(value_opts, **kwargs)
+
+    def __str__(self) -> str:
+        return "{} NoImplicitHeapsizeModifier".format(super().__str__())

src/running/util.py

@@ -70,6 +70,16 @@ def parse_config_str(
     return runtime, mods
 
 
+def dont_emit_heapsize_modifier(configuration: "Configuration", c: str) -> bool:
+    mods = parse_modifier_strs(configuration, c.split("|")[1:])
+    from running.modifier import NoImplicitHeapsizeModifier
+
+    for mod in mods:
+        if isinstance(mod, NoImplicitHeapsizeModifier):
+            return True
+    return False
+
+
 def config_str_encode(c: str) -> str:
     return ".".join([x.strip() for x in c.split("|")])
 

tests/test_modifier.py

@@ -2,6 +2,7 @@
 from running.modifier import *
 from running.config import Configuration
 from running.runtime import OpenJDK
+from running.util import dont_emit_heapsize_modifier
 
 
 def test_jvm_arg():
@@ -163,3 +164,23 @@ def test_envvar():
     )
     assert "$HOME" not in jb.to_string(openjdk)
     assert "$DAHKDLHDIWHEIUWHEIWEHIJHDJKAGDKJADGUQDGIQUWDGI" in jb.to_string(openjdk)
+
+
+def test_no_implicit_heapsize_modifier():
+    c = Configuration(
+        {
+            "modifiers": {
+                "no_hfac": {"type": "NoImplicitHeapsizeModifier"},
+            },
+            "runtimes": {
+                "jdk8": {
+                    "type": "OpenJDK",
+                    "release": 8,
+                    "home": "/usr/lib/jvm/temurin-8-jdk-amd64",
+                }
+            },
+        }
+    )
+    c.resolve_class()
+    assert not dont_emit_heapsize_modifier(c, "jdk8|")
+    assert dont_emit_heapsize_modifier(c, "jdk8|no_hfac")