From 92e4dad51acd0af3ad6bf656e43504716b925eaf Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 28 Feb 2019 23:05:30 +0000 Subject: [PATCH] twitter-server: Make /histograms{,.json} endpoints consistent Problem /histograms?h=$HISTO&fmt=raw returns json as a raw array, but /histograms.json returns a big json object where the keys are the names of the histograms, and the values are the raw array. Having two different formats makes it harder to build tools that can consume either of them. Solution Change the format to match the /histograms.json style. Result /admin/histograms can now output data that looks like: { "jvm/gc/eden/pause_msec" : [ { "lowerLimit" : 8, "upperLimit" : 9, "count" : 1 }, { "lowerLimit" : 9, "upperLimit" : 10, "count" : 1 }, { "lowerLimit" : 53, "upperLimit" : 54, "count" : 1 } ] } JIRA Issues: CSL-7717 Differential Revision: https://phabricator.twitter.biz/D279779 --- CHANGELOG.rst | 4 + doc/src/sphinx/Features.rst | 126 +++++++++++++----- .../twitter-server/js/histogram-main.js | 3 +- .../handler/HistogramQueryHandler.scala | 10 +- .../twitter/server/util/JsonConverter.scala | 4 +- .../server/HistogramQueryHandlerTest.scala | 37 +++-- 6 files changed, 133 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 537e8fde..6841423e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* Change the /admin/histograms?h=...-style endpoints to return data in the same style as + /admin/histograms.json. This should make it easier to use tools to parse data from either + endpoint. ``PHAB_ID=D279779`` + 19.2.0 ------- diff --git a/doc/src/sphinx/Features.rst b/doc/src/sphinx/Features.rst index 9b774566..aded3e70 100644 --- a/doc/src/sphinx/Features.rst +++ b/doc/src/sphinx/Features.rst @@ -296,6 +296,52 @@ finagle-stats jar on your classpath. API Parameters ************** +There are two endpoints for histograms, `/admin/histograms.json` and `/admin/histograms`. + +`/admin/histograms.json` will always return the full list of histograms. + +:: + + $ curl http://a.twitter.server/admin/histograms.json + + { + "clnt/p2cslowservertest-server/request_latency_ms" : [ + { + "lowerLimit" : 5, + "upperLimit" : 6, + "count" : 28 + }, + { + "lowerLimit" : 6, + "upperLimit" : 7, + "count" : 9188 + }, + { + "lowerLimit" : 7, + "upperLimit" : 8, + "count" : 25164 + } + ], + "jvm/gc/eden/pause_msec" : [ + { + "lowerLimit" : 8, + "upperLimit" : 9, + "count" : 1 + }, + { + "lowerLimit" : 9, + "upperLimit" : 10, + "count" : 1 + }, + { + "lowerLimit" : 53, + "upperLimit" : 54, + "count" : 1 + } + ] + } + +`/admin/histograms` is more flexible, and can display a single histogram in many ways. The following parameters control how a histogram is displayed. 1) "h": the name of the histogram you want to see. @@ -325,19 +371,23 @@ equal to the bucket's upper limit. $ curl http://a.twitter.server/admin/histograms?h=clnt/p2cslowservertest-server/request_latency_ms&fmt=raw { - "lowerLimit" : 5, - "upperLimit" : 6, - "count" : 28 - }, - { - "lowerLimit" : 6, - "upperLimit" : 7, - "count" : 9188 - }, - { - "lowerLimit" : 7, - "upperLimit" : 8, - "count" : 25164 + "clnt/p2cslowservertest-server/request_latency_ms" : [ + { + "lowerLimit" : 5, + "upperLimit" : 6, + "count" : 28 + }, + { + "lowerLimit" : 6, + "upperLimit" : 7, + "count" : 9188 + }, + { + "lowerLimit" : 7, + "upperLimit" : 8, + "count" : 25164 + } + ] } Here's an example where we query a cumulative distribution function (CDF). Each @@ -349,29 +399,33 @@ equal to the bucket's upper limit. $ curl http://a.twitter.server/admin/histograms?h=clnt/p2cslowservertest-server/request_latency_ms&fmt=cdf { - "lowerLimit" : 5, - "upperLimit" : 6, - "percentage" : 6.444885E-5 - }, - { - "lowerLimit" : 6, - "upperLimit" : 7, - "percentage" : 0.03286891 - }, - { - "lowerLimit" : 7, - "upperLimit" : 8, - "percentage" : 0.15292539 - }, - { - "lowerLimit" : 8, - "upperLimit" : 9, - "percentage" : 0.26998794 - }, - { - "lowerLimit" : 9, - "upperLimit" : 10, - "percentage" : 0.41753477 + "clnt/p2cslowservertest-server/request_latency_ms" : [ + { + "lowerLimit" : 5, + "upperLimit" : 6, + "percentage" : 6.444885E-5 + }, + { + "lowerLimit" : 6, + "upperLimit" : 7, + "percentage" : 0.03286891 + }, + { + "lowerLimit" : 7, + "upperLimit" : 8, + "percentage" : 0.15292539 + }, + { + "lowerLimit" : 8, + "upperLimit" : 9, + "percentage" : 0.26998794 + }, + { + "lowerLimit" : 9, + "upperLimit" : 10, + "percentage" : 0.41753477 + } + ] } Plotting the "finagle/timer/deviation_ms" cumulative distribution function: diff --git a/server/src/main/resources/twitter-server/js/histogram-main.js b/server/src/main/resources/twitter-server/js/histogram-main.js index 390fcd31..2e4a2901 100644 --- a/server/src/main/resources/twitter-server/js/histogram-main.js +++ b/server/src/main/resources/twitter-server/js/histogram-main.js @@ -18,7 +18,8 @@ function histogramRefreshRequest(callback) { xhttp.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); xhttp.onreadystatechange = function() { if (xhttp.readyState === 4 && xhttp.status == 200) { - refreshHistogram(JSON.parse(xhttp.responseText)); + var histoMap = JSON.parse(xhttp.responseText); + refreshHistogram(histoMap[params.h]); callback(); } else { console.log("Histogram refresh request failed with status code: " + xhttp.status + diff --git a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala index 67620d95..7e87c553 100644 --- a/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala +++ b/server/src/main/scala/com/twitter/server/handler/HistogramQueryHandler.scala @@ -62,10 +62,10 @@ object HistogramQueryHandler { } private[HistogramQueryHandler] def deliverData( - counts: Seq[BucketAndCount], + counts: Map[String, Seq[BucketAndCount]], transform: Seq[BucketAndCount] => Any ): String = - JsonConverter.writeToString(transform(counts)) + JsonConverter.writeToString(counts.mapValues(transform)) // Generates html for visualizing histograms private[HistogramQueryHandler] val render: String = { @@ -357,17 +357,17 @@ private[server] class HistogramQueryHandler(details: WithHistogramDetails) case Some(Seq("raw")) => jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(counts, x => x) + deliverData(Map(query -> counts), identity) }) case Some(Seq("pdf")) => jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(counts, x => pdf(x)) + deliverData(Map(query -> counts), x => pdf(x)) }) case Some(Seq("cdf")) => jsonResponse(query, { counts: Seq[BucketAndCount] => - deliverData(counts, x => cdf(x)) + deliverData(Map(query -> counts), x => cdf(x)) }) case _ => diff --git a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala index 15149a9e..71dfca9b 100644 --- a/server/src/main/scala/com/twitter/server/util/JsonConverter.scala +++ b/server/src/main/scala/com/twitter/server/util/JsonConverter.scala @@ -11,9 +11,11 @@ object JsonConverter { private[this] val writer = { val factory = new MappingJsonFactory() factory.disable(JsonFactory.Feature.USE_THREAD_LOCAL_FOR_BUFFER_RECYCLING) - val mapper = new ObjectMapper(factory).registerModule(DefaultScalaModule) + val mapper = new ObjectMapper(factory) + .registerModule(DefaultScalaModule) val printer = new DefaultPrettyPrinter printer.indentArraysWith(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE) + mapper.writer(printer) } diff --git a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala index 136c6cb8..33278d11 100644 --- a/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala +++ b/server/src/test/scala/com/twitter/server/HistogramQueryHandlerTest.scala @@ -1,22 +1,28 @@ package com.twitter.server -import com.twitter.finagle.stats.InMemoryStatsReceiver +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.http.{Request, Response} +import com.twitter.finagle.stats.{BucketAndCount, InMemoryStatsReceiver} import com.twitter.server.handler.HistogramQueryHandler import com.twitter.util.{Await, Future} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class HistogramQueryHandlerTest extends FunSuite { + def await[A](f: Future[A]): A = Await.result(f, 5.seconds) + private[this] val mapper = new ObjectMapper with ScalaObjectMapper { + registerModule(DefaultScalaModule) + } + test("histograms.json works with no stats") { val sr = new InMemoryStatsReceiver val handler = new HistogramQueryHandler(sr) val req = Request("/admin/histograms.json") val resp: Future[Response] = handler(req) - assert(Await.result(resp).contentString.contains("{ }")) + assert(await(resp).contentString.contains("{ }")) } test("histograms.json works with stats") { @@ -27,7 +33,15 @@ class HistogramQueryHandlerTest extends FunSuite { val req = Request("/admin/histograms.json") val resp: Future[Response] = handler(req) - val result = Await.result(resp).contentString + val result = await(resp).contentString + + val map = mapper.readValue[Map[String, Seq[BucketAndCount]]](result) + map("my/cool/stat") match { + case Seq(BucketAndCount(lowerLimit, upperLimit, count)) => + assert(lowerLimit == 5) + assert(upperLimit == 6) + assert(count == 1) + } assert(result.contains("my/cool/stat")) assert(result.contains(raw""""lowerLimit" : 5,""")) @@ -41,7 +55,7 @@ class HistogramQueryHandlerTest extends FunSuite { val req = Request("/admin/histograms?h=my/cool/stat&fmt=pdf") val resp: Future[Response] = handler(req) - assert(Await.result(resp).contentString.contains("not a valid histogram.")) + assert(await(resp).contentString.contains("not a valid histogram.")) } test("histograms work with a stat") { @@ -52,7 +66,14 @@ class HistogramQueryHandlerTest extends FunSuite { val req = Request("/admin/histograms?h=my/cool/stat&fmt=pdf") val resp: Future[Response] = handler(req) - val result = Await.result(resp).contentString + val result = await(resp).contentString + val map = mapper.readValue[Map[String, Seq[HistogramQueryHandler.BucketAndPercentage]]](result) + map("my/cool/stat") match { + case Seq(HistogramQueryHandler.BucketAndPercentage(lowerLimit, upperLimit, percentage)) => + assert(lowerLimit == 0) + assert(upperLimit == 1) + assert(percentage == 1.0) + } assert(result.contains(raw""""lowerLimit" : 0,""")) assert(result.contains(raw""""upperLimit" : 1,""")) assert(result.contains(raw""""percentage" : 1.0"""))