Skip to content

Commit

Permalink
Add histogram to metrics system (#7944)
Browse files Browse the repository at this point in the history
Signed-off-by: Fabio Di Fabio <[email protected]>
  • Loading branch information
fab-10 authored Nov 27, 2024
1 parent 5b2da5a commit 14dec7b
Show file tree
Hide file tree
Showing 12 changed files with 304 additions and 60 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
- Add a new metric collector for counters which get their value from suppliers [#7894](https://github.com/hyperledger/besu/pull/7894)
- Add account and state overrides to `eth_call` [#7801](https://github.com/hyperledger/besu/pull/7801) and `eth_estimateGas` [#7890](https://github.com/hyperledger/besu/pull/7890)
- Prometheus Java Metrics library upgraded to version 1.3.3 [#7880](https://github.com/hyperledger/besu/pull/7880)
- Add histogram to Prometheus metrics system [#7944](https://github.com/hyperledger/besu/pull/7944)

### Bug fixes
- Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.hyperledger.besu.metrics.Observation;
import org.hyperledger.besu.plugin.services.metrics.Counter;
import org.hyperledger.besu.plugin.services.metrics.ExternalSummary;
import org.hyperledger.besu.plugin.services.metrics.Histogram;
import org.hyperledger.besu.plugin.services.metrics.LabelledGauge;
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric;
Expand Down Expand Up @@ -63,6 +64,9 @@ public class NoOpMetricsSystem implements ObservableMetricsSystem {
public static final LabelledMetric<OperationTimer> NO_OP_LABELLED_1_OPERATION_TIMER =
new LabelCountingNoOpMetric<>(1, NO_OP_OPERATION_TIMER);

/** The constant NO_OP_HISTOGRAM. */
public static final Histogram NO_OP_HISTOGRAM = d -> {};

/** Default constructor */
public NoOpMetricsSystem() {}

Expand Down Expand Up @@ -130,6 +134,26 @@ public void createGauge(
final String help,
final DoubleSupplier valueSupplier) {}

@Override
public LabelledMetric<Histogram> createLabelledHistogram(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
return getHistogramLabelledMetric(labelNames.length);
}

/**
* Gets histogram labelled metric.
*
* @param labelCount the label count
* @return the histogram labelled metric
*/
public static LabelledMetric<Histogram> getHistogramLabelledMetric(final int labelCount) {
return new LabelCountingNoOpMetric<>(labelCount, NO_OP_HISTOGRAM);
}

@Override
public void createGuavaCacheCollector(
final MetricCategory category, final String name, final Cache<?, ?> cache) {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.hyperledger.besu.metrics.StandardMetricCategory;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
import org.hyperledger.besu.plugin.services.metrics.Histogram;
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedSummary;
Expand Down Expand Up @@ -289,6 +290,17 @@ public void createGauge(
}
}

@Override
public LabelledMetric<Histogram> createLabelledHistogram(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
// not yet supported
return NoOpMetricsSystem.getHistogramLabelledMetric(labelNames.length);
}

@Override
public void createGuavaCacheCollector(
final MetricCategory category, final String name, final Cache<?, ?> cache) {}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
* Copyright contributors to Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.metrics.prometheus;

import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.addLabelValues;
import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.getLabelValues;

import org.hyperledger.besu.metrics.Observation;
import org.hyperledger.besu.plugin.services.metrics.MetricCategory;

import java.util.ArrayList;
import java.util.stream.Stream;

import io.prometheus.metrics.core.metrics.Histogram;
import io.prometheus.metrics.model.registry.PrometheusRegistry;

/**
* Abstract base class for Prometheus histogram collectors. A histogram samples durations and counts
* them in configurable buckets. * It also provides a sum of all observed values.
*/
abstract class AbstractPrometheusHistogram extends CategorizedPrometheusCollector {
protected Histogram histogram;

protected AbstractPrometheusHistogram(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
super(category, name);
this.histogram =
Histogram.builder()
.name(this.prefixedName)
.help(help)
.labelNames(labelNames)
.classicOnly()
.classicUpperBounds(buckets)
.build();
}

@Override
public String getIdentifier() {
return histogram.getPrometheusName();
}

@Override
public void register(final PrometheusRegistry registry) {
registry.register(histogram);
}

@Override
public void unregister(final PrometheusRegistry registry) {
registry.unregister(histogram);
}

@Override
public Stream<Observation> streamObservations() {
final var snapshot = histogram.collect();
return snapshot.getDataPoints().stream()
.flatMap(
dataPoint -> {
if (!dataPoint.hasClassicHistogramData()) {
throw new IllegalStateException("Only classic histogram are supported");
}

final var labelValues = getLabelValues(dataPoint.getLabels());
final var classicBuckets = dataPoint.getClassicBuckets();
final var observations = new ArrayList<Observation>(classicBuckets.size() + 2);

if (dataPoint.hasSum()) {
observations.add(
new Observation(
category, name, dataPoint.getSum(), addLabelValues(labelValues, "sum")));
}

if (dataPoint.hasCount()) {
observations.add(
new Observation(
category,
name,
dataPoint.getCount(),
addLabelValues(labelValues, "count")));
}

classicBuckets.stream()
.forEach(
bucket ->
observations.add(
new Observation(
category,
name,
bucket.getCount(),
addLabelValues(
labelValues,
"bucket",
Double.toString(bucket.getUpperBound())))));

return observations.stream();
});
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright contributors to Besu.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*/
package org.hyperledger.besu.metrics.prometheus;

import org.hyperledger.besu.plugin.services.metrics.Histogram;
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
import org.hyperledger.besu.plugin.services.metrics.MetricCategory;

/**
* A Prometheus histogram. A histogram samples durations and counts them in configurable buckets. It
* also provides a sum of all observed values.
*/
class PrometheusHistogram extends AbstractPrometheusHistogram implements LabelledMetric<Histogram> {

public PrometheusHistogram(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
super(category, name, help, buckets, labelNames);
}

@Override
public Histogram labels(final String... labels) {
final var ddp = histogram.labelValues(labels);
return ddp::observe;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.hyperledger.besu.metrics.StandardMetricCategory;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
import org.hyperledger.besu.plugin.services.metrics.Histogram;
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedSummary;
Expand Down Expand Up @@ -67,6 +68,9 @@ public class PrometheusMetricsSystem implements ObservableMetricsSystem {
new ConcurrentHashMap<>();
private final Map<CachedMetricKey, LabelledMetric<OperationTimer>> cachedTimers =
new ConcurrentHashMap<>();
private final Map<CachedMetricKey, LabelledMetric<Histogram>> cachedHistograms =
new ConcurrentHashMap<>();

private final PrometheusGuavaCache.Context guavaCacheCollectorContext =
new PrometheusGuavaCache.Context();

Expand Down Expand Up @@ -164,6 +168,26 @@ public LabelledMetric<OperationTimer> createSimpleLabelledTimer(
});
}

@Override
public LabelledMetric<Histogram> createLabelledHistogram(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
return cachedHistograms.computeIfAbsent(
CachedMetricKey.of(category, name),
k -> {
if (isCategoryEnabled(category)) {
final var histogram =
new PrometheusHistogram(category, name, help, buckets, labelNames);
registerCollector(category, histogram);
return histogram;
}
return NoOpMetricsSystem.getHistogramLabelledMetric(labelNames.length);
});
}

@Override
public LabelledSuppliedSummary createLabelledSuppliedSummary(
final MetricCategory category,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,86 +14,29 @@
*/
package org.hyperledger.besu.metrics.prometheus;

import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.addLabelValues;
import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.getLabelValues;

import org.hyperledger.besu.metrics.Observation;
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
import org.hyperledger.besu.plugin.services.metrics.OperationTimer;

import java.util.stream.Stream;

import io.prometheus.metrics.core.metrics.Histogram;
import io.prometheus.metrics.model.registry.PrometheusRegistry;

/**
* An implementation of Besu simple timer backed by a Prometheus histogram. The histogram samples
* durations and counts them in configurable buckets. It also provides a sum of all observed values.
*/
class PrometheusSimpleTimer extends CategorizedPrometheusCollector
class PrometheusSimpleTimer extends AbstractPrometheusHistogram
implements LabelledMetric<OperationTimer> {

private final Histogram histogram;

public PrometheusSimpleTimer(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
super(category, name);
this.histogram =
Histogram.builder()
.name(this.prefixedName)
.help(help)
.labelNames(labelNames)
.classicOnly()
.classicUpperBounds(buckets)
.build();
super(category, name, help, buckets, labelNames);
}

@Override
public OperationTimer labels(final String... labels) {
final var ddp = histogram.labelValues(labels);
return () -> ddp.startTimer()::observeDuration;
}

@Override
public String getIdentifier() {
return histogram.getPrometheusName();
}

@Override
public void register(final PrometheusRegistry registry) {
registry.register(histogram);
}

@Override
public void unregister(final PrometheusRegistry registry) {
registry.unregister(histogram);
}

@Override
public Stream<Observation> streamObservations() {
final var snapshot = histogram.collect();
return snapshot.getDataPoints().stream()
.flatMap(
dataPoint -> {
final var labelValues = getLabelValues(dataPoint.getLabels());
if (!dataPoint.hasClassicHistogramData()) {
throw new IllegalStateException("Only classic histogram are supported");
}

return dataPoint.getClassicBuckets().stream()
.map(
bucket ->
new Observation(
category,
name,
bucket.getCount(),
addLabelValues(
labelValues, Double.toString(bucket.getUpperBound()))));
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.Counter;
import org.hyperledger.besu.plugin.services.metrics.Histogram;
import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedMetric;
import org.hyperledger.besu.plugin.services.metrics.LabelledSuppliedSummary;
Expand Down Expand Up @@ -114,6 +115,16 @@ public void createGauge(
gauges.put(name, valueSupplier);
}

@Override
public LabelledMetric<Histogram> createLabelledHistogram(
final MetricCategory category,
final String name,
final String help,
final double[] buckets,
final String... labelNames) {
return NoOpMetricsSystem.getHistogramLabelledMetric(labelNames.length);
}

@Override
public void createGuavaCacheCollector(
final MetricCategory category, final String name, final Cache<?, ?> cache) {}
Expand Down
Loading

0 comments on commit 14dec7b

Please sign in to comment.