CORE-207: JMH benchmarking

README.md

@@ -119,3 +119,38 @@ In order to debug in Intellij:
 5. Run the local firecloud docker with `./config/docker-rsync-local-orch.sh` from the root directory
 6. In Intellij, choose your debug configuration and run 'debug'.
 
+
+
+## Benchmarking
+
+Benchmarks reside in the [benchmarks](benchmarks) subdirectory,
+and are accessible via an `sbt` subproject named `bench`.
+
+Benchmarks are powered by [JMH](https://github.com/openjdk/jmh)
+and integrated into sbt using the [sbt-jmh plugin](https://github.com/sbt/sbt-jmh).
+
+To execute benchmarks:
+```
+sbt bench/Jmh/run
+```
+
+To see options for running benchmarks:
+```
+sbt "bench/Jmh/run -h"
+```
+Example of specifying options when running benchmarks (this example sets 3 iterations,
+2 warmup iterations, and one fork):
+```
+sbt "bench/Jmh/run -i 3 -wi 2 -f 1"
+```
+
+Running benchmarks takes a while, especially when specifying more iterations/warmups/forks.
+Of course, using more iterations/warmups/forks is also more accurate. Benchmark output is fairly verbose;
+look for the final summary that will look something like:
+```
+[info] Benchmark                                    Mode  Cnt         Score           Error  Units
+[info] TsvFormatterBenchmark.tsvSafeStringNoTab    thrpt    3  85746770.866 ± 112486692.134  ops/s
+[info] TsvFormatterBenchmark.tsvSafeStringWithTab  thrpt    3  30601083.552 ±  53318975.049  ops/s
+```
+
+

benchmarks/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.10.6

benchmarks/src/main/scala/org/broadinstitute/dsde/firecloud/utils/TsvFormatterBenchmark.scala

@@ -0,0 +1,22 @@
+package org.broadinstitute.dsde.firecloud.utils
+
+import org.openjdk.jmh.annotations.Benchmark
+import org.openjdk.jmh.infra.Blackhole
+
+class TsvFormatterBenchmark {
+
+  @Benchmark
+  def tsvSafeStringNoTab(blackHole: Blackhole): String = {
+    val result = TSVFormatter.tsvSafeString("foo")
+    blackHole.consume(result)
+    result
+  }
+
+  @Benchmark
+  def tsvSafeStringWithTab(blackHole: Blackhole): String = {
+    val result = TSVFormatter.tsvSafeString("foo\tbar")
+    blackHole.consume(result)
+    result
+  }
+
+}

build.sbt

@@ -1,13 +1,28 @@
-import Settings._
-import Testing._
+import Settings.*
+import Testing.*
+import pl.project13.scala.sbt.JmhPlugin
 import spray.revolver.RevolverPlugin
 
 lazy val root = project.in(file("."))
-  .settings(rootSettings:_*)
+  .settings(rootSettings *)
   .withTestSettings
 
 enablePlugins(RevolverPlugin)
 
+// JMH subproject for benchmarking
+lazy val bench = project.in(file("benchmarks"))
+  .dependsOn(root % "compile->compile")
+  .disablePlugins(RevolverPlugin)
+  .enablePlugins(JmhPlugin)
+  .settings(
+    scalaVersion  := (root / scalaVersion).value,
+    // rewire tasks, so that 'bench/Jmh/run' automatically invokes 'bench/Jmh/compile'
+    // and 'bench/Jmh/compile' invokes root's 'compile'
+    Jmh / compile := (Jmh / compile).dependsOn(Compile / compile).value,
+    Jmh / run := (Jmh / run).dependsOn(Jmh / compile).evaluated
+  )
+
+
 Revolver.enableDebugging(port = 5051, suspend = false)
 
 // When JAVA_OPTS are specified in the environment, they are usually meant for the application

project/plugins.sbt

@@ -6,4 +6,6 @@ addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.2.2")
 
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
 
+addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7")
+
 addDependencyTreePlugin