From 4941362121f250b580d58ffad50ff08f8d1f0bf4 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Sun, 25 Aug 2024 08:26:42 -0400 Subject: [PATCH] #122 Removing metrics service. --- phileas-core/pom.xml | 5 - .../services/PhileasFilterService.java | 9 +- .../metrics/LoggingMetricsService.java | 54 ++++ .../configuration/PhileasConfiguration.java | 53 ---- .../phileas-services-metrics/pom.xml | 49 --- .../metrics/PhileasMetricsService.java | 279 ------------------ phileas-services/pom.xml | 1 - pom.xml | 5 - 8 files changed, 59 insertions(+), 396 deletions(-) create mode 100644 phileas-core/src/main/java/ai/philterd/phileas/services/metrics/LoggingMetricsService.java delete mode 100644 phileas-services/phileas-services-metrics/pom.xml delete mode 100644 phileas-services/phileas-services-metrics/src/main/java/ai/philterd/phileas/metrics/PhileasMetricsService.java diff --git a/phileas-core/pom.xml b/phileas-core/pom.xml index 1e832b1a..d8aa36b8 100644 --- a/phileas-core/pom.xml +++ b/phileas-core/pom.xml @@ -42,11 +42,6 @@ phileas-processors-images ${project.version} --> - - ai.philterd - phileas-services-metrics - ${project.version} - ai.philterd phileas-services-disambiguation diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java b/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java index 6667bfd8..bc04b548 100644 --- a/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/PhileasFilterService.java @@ -15,7 +15,6 @@ */ package ai.philterd.phileas.services; -import ai.philterd.phileas.metrics.PhileasMetricsService; import ai.philterd.phileas.model.configuration.PhileasConfiguration; import ai.philterd.phileas.model.domain.Domain; import ai.philterd.phileas.model.domain.HealthDomain; @@ -38,6 +37,7 @@ import ai.philterd.phileas.services.alerts.AlertServiceFactory; import ai.philterd.phileas.services.anonymization.cache.AnonymizationCacheServiceFactory; import ai.philterd.phileas.services.disambiguation.VectorBasedSpanDisambiguationService; +import ai.philterd.phileas.services.metrics.LoggingMetricsService; import ai.philterd.phileas.services.policies.LocalPolicyService; import ai.philterd.phileas.services.policies.S3PolicyService; import ai.philterd.phileas.services.policies.utils.PolicyUtils; @@ -76,6 +76,10 @@ public class PhileasFilterService implements FilterService { //private final ImageProcessor imageProcessor; public PhileasFilterService(final PhileasConfiguration phileasConfiguration) throws IOException { + this(phileasConfiguration, new LoggingMetricsService()); + } + + public PhileasFilterService(final PhileasConfiguration phileasConfiguration, final MetricsService metricsService) throws IOException { LOGGER.info("Initializing Phileas engine."); @@ -86,9 +90,6 @@ public PhileasFilterService(final PhileasConfiguration phileasConfiguration) thr gsonBuilder.registerTypeAdapter(String.class, new PlaceholderDeserializer()); final Gson gson = gsonBuilder.create(); - // Configure metrics. - MetricsService metricsService = new PhileasMetricsService(phileasConfiguration); - // Set the policy services. this.policyService = buildPolicyService(phileasConfiguration); this.policyUtils = new PolicyUtils(policyService, gson); diff --git a/phileas-core/src/main/java/ai/philterd/phileas/services/metrics/LoggingMetricsService.java b/phileas-core/src/main/java/ai/philterd/phileas/services/metrics/LoggingMetricsService.java new file mode 100644 index 00000000..cd35940c --- /dev/null +++ b/phileas-core/src/main/java/ai/philterd/phileas/services/metrics/LoggingMetricsService.java @@ -0,0 +1,54 @@ +package ai.philterd.phileas.services.metrics; + +import ai.philterd.phileas.model.enums.FilterType; +import ai.philterd.phileas.model.filter.Filter; +import ai.philterd.phileas.model.services.MetricsService; +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; + +public class LoggingMetricsService implements MetricsService { + + private static final Logger LOGGER = LogManager.getLogger(LoggingMetricsService.class); + + long processed = 0; + private final Map filterTypes = new HashMap<>(); + private final Map filterTimes = new HashMap<>(); + + @Override + public void incrementProcessed() { + incrementProcessed(1); + } + + @Override + public void incrementProcessed(long count) { + processed += count; + LOGGER.debug("Documents processed: {}", processed); + } + + @Override + public void incrementFilterType(FilterType filterType) { + if(filterTypes.containsKey(filterType)) { + filterTypes.put(filterType, filterTypes.get(filterType) + 1); + } else { + filterTypes.put(filterType, 1L); + } + LOGGER.debug("Filter type: {}, Count: {}", filterType, filterTypes.get(filterType)); + } + + @Override + public void logFilterTime(FilterType filterType, long timeMs) { + if(filterTimes.containsKey(filterType)) { + filterTimes.get(filterType).addValue(timeMs); + } else { + final DescriptiveStatistics stats = new DescriptiveStatistics(); + stats.addValue(timeMs); + filterTimes.put(filterType, stats); + } + LOGGER.debug("Filter type: {}, Average Time: {} ms", filterType, filterTimes.get(filterType).getMean()); + } + +} diff --git a/phileas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java b/phileas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java index 47d1f909..9ec5fa42 100644 --- a/phileas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java +++ b/phileas-model/src/main/java/ai/philterd/phileas/model/configuration/PhileasConfiguration.java @@ -145,59 +145,6 @@ public String policiesS3Region() { return getProperty("filter.policies.s3.region", "us-east-1"); } - // Metrics - - public String metricsPrefix() { - return getProperty("metrics.prefix", applicationName); - } - - // See: https://github.com/micrometer-metrics/micrometer/blob/master/implementations/micrometer-registry-prometheus/src/main/java/io/micrometer/prometheus/PrometheusConfig.java - // The step size to use in computing windowed statistics like max. The default is 1 minute. - // To get the most out of these statistics, align the step interval to be close to your scrape interval. - public int metricsStep() { - return Integer.parseInt(getProperty("metrics.step", "60")); - } - - public boolean metricsJmxEnabled() { - return Boolean.parseBoolean(getProperty("metrics.jmx.enabled", "false")); - } - - public boolean metricsPrometheusEnabled() { - return Boolean.parseBoolean(getProperty("metrics.prometheus.enabled", "false")); - } - - public int metricsPrometheusPort() { - return Integer.parseInt(getProperty("metrics.prometheus.port", "9100")); - } - - public String metricsPrometheusContext() { - return getProperty("metrics.prometheus.context", "metrics"); - } - - public boolean metricsDataDogEnabled() { - return Boolean.parseBoolean(getProperty("metrics.datadog.enabled", "false")); - } - - public String metricsDataDogApiKey() { - return getProperty("metrics.datadog.apikey", "metrics"); - } - - public boolean metricsCloudWatchEnabled() { - return Boolean.parseBoolean(getProperty("metrics.cloudwatch.enabled", "false")); - } - - public String metricsCloudWatchRegion() { - return String.valueOf(getProperty("metrics.cloudwatch.region", "us-east-1")); - } - - public String metricsCloudWatchNamespace() { - return String.valueOf(getProperty("metrics.cloudwatch.namespace", applicationName)); - } - - public String metricsHostname() { - return String.valueOf(getProperty("metrics.hostname", "")); - } - private String getProperty(final String property, final String defaultValue) { final String environmentVariableValue = getEnvironmentVariable(property); diff --git a/phileas-services/phileas-services-metrics/pom.xml b/phileas-services/phileas-services-metrics/pom.xml deleted file mode 100644 index d0242bf5..00000000 --- a/phileas-services/phileas-services-metrics/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - 4.0.0 - - phileas-services - ai.philterd - 2.7.0-SNAPSHOT - - phileas-services-metrics - phileas-services-metrics - - - ai.philterd - phileas-model - ${project.version} - - - io.micrometer - micrometer-registry-jmx - ${micrometer.version} - - - io.micrometer - micrometer-registry-prometheus - ${dropwizard-metrics-prometheus.version} - - - io.micrometer - micrometer-registry-datadog - ${micrometer.version} - - - io.micrometer - micrometer-registry-cloudwatch - ${dropwizard-metrics-cloudwatch.version} - - - io.micrometer - micrometer-core - ${micrometer.version} - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - diff --git a/phileas-services/phileas-services-metrics/src/main/java/ai/philterd/phileas/metrics/PhileasMetricsService.java b/phileas-services/phileas-services-metrics/src/main/java/ai/philterd/phileas/metrics/PhileasMetricsService.java deleted file mode 100644 index d5db9cc5..00000000 --- a/phileas-services/phileas-services-metrics/src/main/java/ai/philterd/phileas/metrics/PhileasMetricsService.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright 2024 Philterd, LLC @ https://www.philterd.ai - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package ai.philterd.phileas.metrics; - -import ai.philterd.phileas.model.configuration.PhileasConfiguration; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.cloudwatch.AmazonCloudWatchAsync; -import com.amazonaws.services.cloudwatch.AmazonCloudWatchAsyncClientBuilder; -import ai.philterd.phileas.model.enums.FilterType; -import ai.philterd.phileas.model.services.MetricsService; -import com.sun.net.httpserver.HttpServer; -import io.micrometer.cloudwatch.CloudWatchConfig; -import io.micrometer.cloudwatch.CloudWatchMeterRegistry; -import io.micrometer.core.instrument.Clock; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Timer; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; -import io.micrometer.datadog.DatadogConfig; -import io.micrometer.datadog.DatadogMeterRegistry; -import io.micrometer.jmx.JmxConfig; -import io.micrometer.jmx.JmxMeterRegistry; -import io.micrometer.prometheusmetrics.PrometheusConfig; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -public class PhileasMetricsService implements MetricsService { - - private static final Logger LOGGER = LogManager.getLogger(PhileasMetricsService.class); - - private static final String TOTAL_DOCUMENTS_PROCESSED = "total.documents.processed"; - private static final String DOCUMENTS_PROCESSED = "documents.processed"; - - private final transient Counter processed; - private final transient Counter documents; - private final transient Map filterTypes; - private final transient Map filterTimers; - - /** - * Creates a new metrics service. - * @param phileasConfiguration The {@link PhileasConfiguration phileasConfiguration} used to initialize the application. - */ - public PhileasMetricsService(PhileasConfiguration phileasConfiguration) { - - final CompositeMeterRegistry compositeMeterRegistry = new CompositeMeterRegistry(); - - compositeMeterRegistry.config().commonTags("application", "philter"); - - if(StringUtils.isNotEmpty(phileasConfiguration.metricsHostname())) { - compositeMeterRegistry.config().commonTags("hostname", phileasConfiguration.metricsHostname()); - } - - final int step = phileasConfiguration.metricsStep(); - - if(phileasConfiguration.metricsJmxEnabled()) { - - LOGGER.info("Initializing JMX metric reporting."); - - final JmxConfig jmxConfig = new JmxConfig() { - @Override - public String get(String s) { - return null; - } - - @Override - public Duration step() { - return Duration.ofSeconds(step); - } - - @Override - public String prefix() { - return phileasConfiguration.metricsPrefix(); - } - }; - - compositeMeterRegistry.add(new JmxMeterRegistry(jmxConfig, Clock.SYSTEM)); - - } - - if(phileasConfiguration.metricsPrometheusEnabled()) { - - LOGGER.info("Initializing Prometheus metric reporting."); - - final PrometheusConfig prometheusConfig = new PrometheusConfig() { - - @Override - public Duration step() { - return Duration.ofSeconds(step); - } - - @Override - public String get(String k) { - return null; - } - - @Override - public String prefix() { - return phileasConfiguration.metricsPrefix(); - } - - }; - - final PrometheusMeterRegistry prometheusRegistry = new PrometheusMeterRegistry(prometheusConfig); - final int port = phileasConfiguration.metricsPrometheusPort(); - final String context = phileasConfiguration.metricsPrometheusContext(); - - try { - HttpServer server = HttpServer.create(new InetSocketAddress(port), 0); - server.createContext("/" + context, httpExchange -> { - final String response = prometheusRegistry.scrape(); - httpExchange.sendResponseHeaders(200, response.getBytes().length); - try (OutputStream os = httpExchange.getResponseBody()) { - os.write(response.getBytes()); - } - }); - - new Thread(server::start).start(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - compositeMeterRegistry.add(prometheusRegistry); - - } - - if(phileasConfiguration.metricsDataDogEnabled()) { - - LOGGER.info("Initializing Datadog metric reporting."); - - final String datadogApiKey = phileasConfiguration.metricsDataDogApiKey(); - - if (StringUtils.isEmpty(datadogApiKey)) { - - LOGGER.warn("Datadog metric reporting enabled but no Datadog API key provided. Reporting will not be enabled."); - - } else { - - final DatadogConfig datadogConfig = new DatadogConfig() { - @Override - public String apiKey() { - return phileasConfiguration.metricsDataDogApiKey(); - } - - @Override - public Duration step() { - return Duration.ofSeconds(step); - } - - @Override - public String get(String k) { - return null; - } - - @Override - public String prefix() { - return phileasConfiguration.metricsPrefix(); - } - - }; - - compositeMeterRegistry.add(new DatadogMeterRegistry(datadogConfig, Clock.SYSTEM)); - - } - - } - - if(phileasConfiguration.metricsCloudWatchEnabled()) { - - LOGGER.info("Initializing AWS CloudWatch metric reporting."); - - final CloudWatchConfig cloudWatchConfig = new CloudWatchConfig() { - - @Override - public String get(String s) { - return null; - } - - @Override - public Duration step() { - return Duration.ofSeconds(step); - } - - @Override - public String namespace() { - return phileasConfiguration.metricsCloudWatchNamespace(); - } - - @Override - public String prefix() { - return phileasConfiguration.metricsPrefix(); - } - - @Override - public int batchSize() { - // 20 is the maximum batch size. - return CloudWatchConfig.MAX_BATCH_SIZE; - } - - }; - - final AmazonCloudWatchAsync amazonCloudWatchAsync = AmazonCloudWatchAsyncClientBuilder - .standard() - .withRegion(Regions.fromName(phileasConfiguration.metricsCloudWatchRegion())) - .build(); - - compositeMeterRegistry.add(new CloudWatchMeterRegistry(cloudWatchConfig, Clock.SYSTEM, amazonCloudWatchAsync)); - - } - - this.processed = compositeMeterRegistry.counter(phileasConfiguration.metricsPrefix() + "." + TOTAL_DOCUMENTS_PROCESSED); - this.documents = compositeMeterRegistry.counter(phileasConfiguration.metricsPrefix() + "." + DOCUMENTS_PROCESSED); - - // Add a counter for each filter type. - this.filterTypes = new HashMap<>(); - for(final FilterType filterType : FilterType.values()) { - filterTypes.put(filterType, compositeMeterRegistry.counter(phileasConfiguration.metricsPrefix() + "." + filterType.name().toLowerCase().replace("-", "."))); - } - - // Add a timer for each filter type. - this.filterTimers = new HashMap<>(); - for(final FilterType filterType : FilterType.values()) { - final String name = phileasConfiguration.metricsPrefix() + "." + filterType.name().toLowerCase().replace("-", ".") + ".time.ms"; - filterTimers.put(filterType, compositeMeterRegistry.timer(name)); - } - - } - - @Override - public void incrementFilterType(FilterType filterType) { - - filterTypes.get(filterType).increment(); - - } - - @Override - public void incrementProcessed() { - - incrementProcessed(1); - - } - - @Override - public void incrementProcessed(long count) { - - processed.increment(count); - documents.increment(count); - - } - - @Override - public void logFilterTime(FilterType filterType, long timeMs) { - - filterTimers.get(filterType).record(timeMs, TimeUnit.MILLISECONDS); - - } - -} diff --git a/phileas-services/pom.xml b/phileas-services/pom.xml index 7257a511..58398e85 100644 --- a/phileas-services/pom.xml +++ b/phileas-services/pom.xml @@ -10,7 +10,6 @@ phileas-services pom - phileas-services-metrics phileas-services-anonymization phileas-services-disambiguation phileas-services-ai diff --git a/pom.xml b/pom.xml index fe760214..e870349d 100644 --- a/pom.xml +++ b/pom.xml @@ -87,10 +87,6 @@ 3.6.1 1.10.0 1.9.0 - 4.1.7 - 1.12.7 - 1.1.14 - 1.13.1 3.16.1 1.0 33.2.1-jre @@ -106,7 +102,6 @@ 3.8.1 3.2.5 3.2.5 - 1.13.1 5.12.0 1.27 2.1.0