From f94574ea4182f1cfa83543149c47bf2b5861583d Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Tue, 16 Jul 2024 18:12:04 -0600 Subject: [PATCH 1/4] unrecognized paths OOM fix --- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 6 +- .../monitoring/StatsCollectorVerticle.java | 14 ++++- .../com/uid2/operator/vertx/Endpoints.java | 59 +++++++++++++++++ .../operator/vertx/UIDOperatorVerticle.java | 55 ++++++++-------- .../operator/StatsCollectorVerticleTest.java | 63 ++++++++++++++++++- 6 files changed, 165 insertions(+), 33 deletions(-) create mode 100644 src/main/java/com/uid2/operator/vertx/Endpoints.java diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 48dd16648..09d183ddd 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -25,5 +25,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String GcpSecretVersionNameProp = "gcp_secret_version_name"; public static final String OptOutStatusApiEnabled = "optout_status_api_enabled"; public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size"; + public static String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 69eb83c19..37e06fbc0 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -9,6 +9,7 @@ import com.uid2.operator.monitoring.StatsCollectorVerticle; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; +import com.uid2.operator.vertx.Endpoints; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.store.CloudSyncOptOutStore; import com.uid2.operator.store.OptOutCloudStorage; @@ -363,7 +364,7 @@ private Future createAndDeployCloudSyncStoreVerticle(String name, ICloud private Future createAndDeployStatsCollector() { Promise promise = Promise.promise(); - StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000); + StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50)); vertx.deployVerticle(statsCollectorVerticle, promise); _statsCollectorQueue = statsCollectorVerticle; return promise.future(); @@ -425,7 +426,8 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) { .meterFilter(new PrometheusRenameFilter()) .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), actualPath -> { try { - return HttpUtils.normalizePath(actualPath).split("\\?")[0]; + String normalized = HttpUtils.normalizePath(actualPath).split("\\?")[0]; + return Endpoints.pathSet().contains(normalized) ? normalized : "/unknown"; } catch (IllegalArgumentException e) { return actualPath; } diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java index 26c0653e0..faebfae5f 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.uid2.operator.Const; import com.uid2.operator.model.StatsCollectorMessageItem; +import com.uid2.operator.vertx.Endpoints; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.vertx.core.AbstractVerticle; @@ -24,6 +25,7 @@ public class StatsCollectorVerticle extends AbstractVerticle implements IStatsCo private HashMap pathMap; private static final int MAX_AVAILABLE = 1000; + private final int maxInvalidPaths; private final Duration jsonProcessingInterval; private Instant lastJsonProcessTime; @@ -39,13 +41,14 @@ public class StatsCollectorVerticle extends AbstractVerticle implements IStatsCo private final ObjectMapper mapper; private final Counter queueFullCounter; - public StatsCollectorVerticle(long jsonIntervalMS) { + public StatsCollectorVerticle(long jsonIntervalMS, int maxInvalidPaths) { pathMap = new HashMap<>(); _statsCollectorCount = new AtomicInteger(); _runningSerializer = false; jsonProcessingInterval = Duration.ofMillis(jsonIntervalMS); + this.maxInvalidPaths = maxInvalidPaths; logCycleSkipperCounter = Counter .builder("uid2.api_usage_log_cycle_skipped") @@ -113,7 +116,11 @@ public void handleMessage(Message message) { EndpointStat endpointStat = new EndpointStat(endpoint, siteId, apiVersion, domain); - pathMap.merge(path, endpointStat, this::mergeEndpoint); + Set endpoints = Endpoints.pathSet(); + if(endpoints.contains(path) || pathMap.containsKey(path) || (pathMap.size() < this.maxInvalidPaths + endpoints.size() && messageItem.getApiContact() != null)) { + pathMap.merge(path, endpointStat, this::mergeEndpoint); + } + _statsCollectorCount.decrementAndGet(); @@ -123,6 +130,9 @@ public void handleMessage(Message message) { logCycleSkipperCounter.increment(); } else { _runningSerializer = true; + if(pathMap.size() == this.maxInvalidPaths + endpoints.size()) { + LOGGER.error("max invalid paths reached; a large number of invalid paths have been requested from authenticated participants"); + } Object[] stats = pathMap.values().toArray(); this.jsonSerializerExecutor.executeBlocking( promise -> promise.complete(this.serializeToLogs(stats)), diff --git a/src/main/java/com/uid2/operator/vertx/Endpoints.java b/src/main/java/com/uid2/operator/vertx/Endpoints.java new file mode 100644 index 000000000..2643f943b --- /dev/null +++ b/src/main/java/com/uid2/operator/vertx/Endpoints.java @@ -0,0 +1,59 @@ +package com.uid2.operator.vertx; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public enum Endpoints { + OPS_HEALTHCHECK("/ops/healthcheck"), + + V0_KEY_LATEST("/key/latest"), + V0_TOKEN_GENERATE("/token/generate"), + V0_TOKEN_REFRESH("/token/refresh"), + V0_TOKEN_VALIDATE("/token/validate"), + V0_IDENTITY_MAP("/identity/map"), + V0_TOKEN_LOGOUT("/token/logout"), + + V1_TOKEN_GENERATE("/v1/token/generate"), + V1_TOKEN_VALIDATE("/v1/token/validate"), + V1_TOKEN_REFRESH("/v1/token/refresh"), + V1_IDENTITY_BUCKETS("/v1/identity/buckets"), + V1_IDENTITY_MAP("/v1/identity/map"), + V1_KEY_LATEST("/v1/key/latest"), + + V2_TOKEN_GENERATE("/v2/token/generate"), + V2_TOKEN_REFRESH("/v2/token/refresh"), + V2_TOKEN_VALIDATE("/v2/token/validate"), + V2_IDENTITY_BUCKETS("/v2/identity/buckets"), + V2_IDENTITY_MAP("/v2/identity/map"), + V2_KEY_LATEST("/v2/key/latest"), + V2_KEY_SHARING("/v2/key/sharing"), + V2_KEY_BIDSTREAM("/v2/key/bidstream"), + V2_TOKEN_LOGOUT("/v2/token/logout"), + V2_OPTOUT_STATUS("/v2/optout/status"), + V2_TOKEN_CLIENTGENERATE("/v2/token/client-generate"), + + EUID_SDK_1_0_0("/static/js/euid-sdk-1.0.0.js"), + OPENID_SDK_1_0("/static/js/openid-sdk-1.0.js"), + UID2_ESP_0_0_1A("/static/js/uid2-esp-0.0.1a.js"), + UID2_SDK_0_0_1A("/static/js/uid2-sdk-0.0.1a.js"), + UID2_SDK_0_0_1A_SOURCE("/static/js/uid2-sdk-0.0.1a-source.ts"), + UID2_SDK_0_0_1B("/static/js/uid2-sdk-0.0.1b.js"), + UID2_SDK_1_0_0("/static/js/uid2-sdk-1.0.0.js"), + UID2_SDK_2_0_0("/static/js/uid2-sdk-2.0.0.js") + ; + private final String path; + + Endpoints(final String path) { + this.path = path; + } + + public static Set pathSet() { + return Stream.of(Endpoints.values()).map(Endpoints::toString).collect(Collectors.toSet()); + } + + @Override + public String toString() { + return path; + } +} diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 372935da7..b3612f187 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -69,6 +69,7 @@ import static com.uid2.operator.IdentityConst.*; import static com.uid2.operator.service.ResponseUtil.*; +import static com.uid2.operator.vertx.Endpoints.*; public class UIDOperatorVerticle extends AbstractVerticle { private static final Logger LOGGER = LoggerFactory.getLogger(UIDOperatorVerticle.class); @@ -236,28 +237,28 @@ private Router createRoutesSetup() throws IOException { setupV2Routes(router, bodyHandler); // Static and health check - router.get("/ops/healthcheck").handler(this::handleHealthCheck); + router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); if (this.config.getBoolean(Const.Config.AllowLegacyAPIProp, true)) { // V1 APIs - router.get("/v1/token/generate").handler(auth.handleV1(this::handleTokenGenerateV1, Role.GENERATOR)); - router.get("/v1/token/validate").handler(this::handleTokenValidateV1); - router.get("/v1/token/refresh").handler(auth.handleWithOptionalAuth(this::handleTokenRefreshV1)); - router.get("/v1/identity/buckets").handler(auth.handle(this::handleBucketsV1, Role.MAPPER)); - router.get("/v1/identity/map").handler(auth.handle(this::handleIdentityMapV1, Role.MAPPER)); - router.post("/v1/identity/map").handler(bodyHandler).handler(auth.handle(this::handleIdentityMapBatchV1, Role.MAPPER)); - router.get("/v1/key/latest").handler(auth.handle(this::handleKeysRequestV1, Role.ID_READER)); + router.get(V1_TOKEN_GENERATE.toString()).handler(auth.handleV1(this::handleTokenGenerateV1, Role.GENERATOR)); + router.get(V1_TOKEN_VALIDATE.toString()).handler(this::handleTokenValidateV1); + router.get(V1_TOKEN_REFRESH.toString()).handler(auth.handleWithOptionalAuth(this::handleTokenRefreshV1)); + router.get(V1_IDENTITY_BUCKETS.toString()).handler(auth.handle(this::handleBucketsV1, Role.MAPPER)); + router.get(V1_IDENTITY_MAP.toString()).handler(auth.handle(this::handleIdentityMapV1, Role.MAPPER)); + router.post(V1_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handle(this::handleIdentityMapBatchV1, Role.MAPPER)); + router.get(V1_KEY_LATEST.toString()).handler(auth.handle(this::handleKeysRequestV1, Role.ID_READER)); // Deprecated APIs - router.get("/key/latest").handler(auth.handle(this::handleKeysRequest, Role.ID_READER)); - router.get("/token/generate").handler(auth.handle(this::handleTokenGenerate, Role.GENERATOR)); - router.get("/token/refresh").handler(this::handleTokenRefresh); - router.get("/token/validate").handler(this::handleValidate); - router.get("/identity/map").handler(auth.handle(this::handleIdentityMap, Role.MAPPER)); - router.post("/identity/map").handler(bodyHandler).handler(auth.handle(this::handleIdentityMapBatch, Role.MAPPER)); + router.get(V0_KEY_LATEST.toString()).handler(auth.handle(this::handleKeysRequest, Role.ID_READER)); + router.get(V0_TOKEN_GENERATE.toString()).handler(auth.handle(this::handleTokenGenerate, Role.GENERATOR)); + router.get(V0_TOKEN_REFRESH.toString()).handler(this::handleTokenRefresh); + router.get(V0_TOKEN_VALIDATE.toString()).handler(this::handleValidate); + router.get(V0_IDENTITY_MAP.toString()).handler(auth.handle(this::handleIdentityMap, Role.MAPPER)); + router.post(V0_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handle(this::handleIdentityMapBatch, Role.MAPPER)); // Internal service APIs - router.get("/token/logout").handler(auth.handle(this::handleLogoutAsync, Role.OPTOUT)); + router.get(V0_TOKEN_LOGOUT.toString()).handler(auth.handle(this::handleLogoutAsync, Role.OPTOUT)); // only uncomment to do local testing //router.get("/internal/optout/get").handler(auth.loopbackOnly(this::handleOptOutGet)); @@ -268,36 +269,34 @@ private Router createRoutesSetup() throws IOException { } private void setupV2Routes(Router mainRouter, BodyHandler bodyHandler) { - final Router v2Router = Router.router(vertx); - v2Router.post("/token/generate").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_TOKEN_GENERATE.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handleTokenGenerate(rc, this::handleTokenGenerateV2), Role.GENERATOR)); - v2Router.post("/token/refresh").handler(bodyHandler).handler(auth.handleWithOptionalAuth( + mainRouter.post(V2_TOKEN_REFRESH.toString()).handler(bodyHandler).handler(auth.handleWithOptionalAuth( rc -> v2PayloadHandler.handleTokenRefresh(rc, this::handleTokenRefreshV2))); - v2Router.post("/token/validate").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_TOKEN_VALIDATE.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleTokenValidateV2), Role.GENERATOR)); - v2Router.post("/identity/buckets").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_IDENTITY_BUCKETS.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleBucketsV2), Role.MAPPER)); - v2Router.post("/identity/map").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_IDENTITY_MAP.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleIdentityMapV2), Role.MAPPER)); - v2Router.post("/key/latest").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_KEY_LATEST.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleKeysRequestV2), Role.ID_READER)); - v2Router.post("/key/sharing").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_KEY_SHARING.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleKeysSharing), Role.SHARER, Role.ID_READER)); - v2Router.post("/key/bidstream").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_KEY_BIDSTREAM.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleKeysBidstream), Role.ID_READER)); - v2Router.post("/token/logout").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_TOKEN_LOGOUT.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handleAsync(rc, this::handleLogoutAsyncV2), Role.OPTOUT)); if (this.optOutStatusApiEnabled) { - v2Router.post("/optout/status").handler(bodyHandler).handler(auth.handleV1( + mainRouter.post(V2_OPTOUT_STATUS.toString()).handler(bodyHandler).handler(auth.handleV1( rc -> v2PayloadHandler.handle(rc, this::handleOptoutStatus), Role.MAPPER, Role.SHARER, Role.ID_READER)); } if (this.clientSideTokenGenerate) - v2Router.post("/token/client-generate").handler(bodyHandler).handler(this::handleClientSideTokenGenerate); + mainRouter.post(V2_TOKEN_CLIENTGENERATE.toString()).handler(bodyHandler).handler(this::handleClientSideTokenGenerate); - mainRouter.route("/v2/*").subRouter(v2Router); } diff --git a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java index 81162e69f..a633d6cd7 100644 --- a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java +++ b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java @@ -1,9 +1,13 @@ package com.uid2.operator; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.read.ListAppender; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.uid2.operator.model.StatsCollectorMessageItem; import com.uid2.operator.monitoring.StatsCollectorVerticle; +import com.uid2.operator.vertx.Endpoints; import io.vertx.core.Vertx; import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; @@ -11,16 +15,19 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.LoggerFactory; +import java.util.Set; import java.util.concurrent.TimeUnit; @ExtendWith(VertxExtension.class) public class StatsCollectorVerticleTest { + private static final int MAX_INVALID_PATHS = 5; private StatsCollectorVerticle verticle; @BeforeEach void deployVerticle(Vertx vertx, VertxTestContext testContext) throws Throwable { - verticle = new StatsCollectorVerticle(1000); + verticle = new StatsCollectorVerticle(1000, MAX_INVALID_PATHS); vertx.deployVerticle(verticle, testContext.succeeding(id -> testContext.completeNow())); } @@ -83,4 +90,58 @@ void testJSONSerializeWithV2AndUnknownPaths(Vertx vertx, VertxTestContext testCo testContext.completeNow(); } + + @Test + void invalidPathsFiltering(Vertx vertx, VertxTestContext testContext) throws InterruptedException, JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + Set validEndpoints = Endpoints.pathSet(); + + for(String endpoint : validEndpoints) { + StatsCollectorMessageItem messageItem = new StatsCollectorMessageItem(endpoint, "https://test.com", "test", 1); + vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); + } + + for(int i = 0; i < MAX_INVALID_PATHS + 5; i++) { + StatsCollectorMessageItem messageItem = new StatsCollectorMessageItem("/bad" + i, "https://test.com", "test", 1); + vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); + } + + testContext.awaitCompletion(2000, TimeUnit.MILLISECONDS); + + String results = verticle.getEndpointStats(); + + for(String endpoint: validEndpoints) { + String withoutVersion = endpoint; + if (endpoint.startsWith("/v1/") || endpoint.startsWith("/v2/")) { + withoutVersion = endpoint.substring(4); + } else if (endpoint.startsWith("/")) { + withoutVersion = endpoint.substring(1); + } + + String expected = "{\"endpoint\":\"" + withoutVersion + "\",\"siteId\":1,"; + Assertions.assertTrue(results.contains(expected)); + } + + for(int i = 0; i < MAX_INVALID_PATHS; i++) { + String expected = "{\"endpoint\":\"bad" + i + "\",\"siteId\":1,\"apiVersion\":\"v0\",\"domainList\":[{\"domain\":\"test.com\",\"count\":1,\"apiContact\":\"test\"}]}"; + Assertions.assertTrue(results.contains(expected)); + } + for(int i = MAX_INVALID_PATHS; i < MAX_INVALID_PATHS + 5; i++) { + String expected = "{\"endpoint\":\"bad" + i + "\",\"siteId\":1,\"apiVersion\":\"v0\",\"domainList\":[{\"domain\":\"test.com\",\"count\":1,\"apiContact\":\"test\"}]}"; + Assertions.assertFalse(results.contains(expected)); + } + + ListAppender logWatcher = new ListAppender<>(); + logWatcher.start(); + ((Logger) LoggerFactory.getLogger(StatsCollectorVerticle.class)).addAppender(logWatcher); + + StatsCollectorMessageItem messageItem = new StatsCollectorMessageItem("/triggerSerialize", "https://test.com", "test", 1); + vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); + + testContext.awaitCompletion(1000, TimeUnit.MILLISECONDS); + + Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("max invalid paths reached; a large number of invalid paths have been requested from authenticated participants")); + + testContext.completeNow(); + } } From 988ea51ed3ea9c1c5e85c89878283f77b4ae3960 Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Thu, 18 Jul 2024 15:14:00 -0600 Subject: [PATCH 2/4] get valid paths from route definitions --- src/main/java/com/uid2/operator/Main.java | 16 +++++---- .../monitoring/StatsCollectorVerticle.java | 9 ++--- .../operator/vertx/UIDOperatorVerticle.java | 34 ++++++++++++++++++- .../operator/StatsCollectorVerticleTest.java | 4 +-- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 37e06fbc0..a7c226941 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -63,6 +63,7 @@ public class Main { private final JsonObject config; private final Vertx vertx; + private final Set validPaths; private final ApplicationVersion appVersion; private final ICloudStorage fsLocal; private final ICloudStorage fsOptOut; @@ -180,6 +181,9 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } } metrics = new OperatorMetrics(getKeyManager(), saltProvider); + + UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, true); + this.validPaths = verticle.getVerticleRoutes(); } private KeyManager getKeyManager() { @@ -264,6 +268,8 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { + setupMetrics(this.validPaths); + Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; @@ -364,7 +370,7 @@ private Future createAndDeployCloudSyncStoreVerticle(String name, ICloud private Future createAndDeployStatsCollector() { Promise promise = Promise.promise(); - StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50)); + StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50), this.validPaths); vertx.deployVerticle(statsCollectorVerticle, promise); _statsCollectorQueue = statsCollectorVerticle; return promise.future(); @@ -399,7 +405,7 @@ private static Vertx createVertx() { .setLabels(EnumSet.of(Label.HTTP_METHOD, Label.HTTP_CODE, Label.HTTP_PATH)) .setJvmMetricsEnabled(true) .setEnabled(true); - setupMetrics(metricOptions); + BackendRegistries.setupBackend(metricOptions); final int threadBlockedCheckInterval = Utils.isProductionEnvironment() ? 60 * 1000 @@ -412,9 +418,7 @@ private static Vertx createVertx() { return Vertx.vertx(vertxOptions); } - private static void setupMetrics(MicrometerMetricsOptions metricOptions) { - BackendRegistries.setupBackend(metricOptions); - + private static void setupMetrics(Set validPaths) { MeterRegistry backendRegistry = BackendRegistries.getDefaultNow(); if (backendRegistry instanceof PrometheusMeterRegistry) { // prometheus specific configuration @@ -427,7 +431,7 @@ private static void setupMetrics(MicrometerMetricsOptions metricOptions) { .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), actualPath -> { try { String normalized = HttpUtils.normalizePath(actualPath).split("\\?")[0]; - return Endpoints.pathSet().contains(normalized) ? normalized : "/unknown"; + return validPaths.contains(normalized) ? normalized : "/unknown"; } catch (IllegalArgumentException e) { return actualPath; } diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java index faebfae5f..65d9a0886 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java @@ -22,6 +22,7 @@ public class StatsCollectorVerticle extends AbstractVerticle implements IStatsCollectorQueue { private static final Logger LOGGER = LoggerFactory.getLogger(StatsCollectorVerticle.class); + private final Set validPaths; private HashMap pathMap; private static final int MAX_AVAILABLE = 1000; @@ -41,7 +42,8 @@ public class StatsCollectorVerticle extends AbstractVerticle implements IStatsCo private final ObjectMapper mapper; private final Counter queueFullCounter; - public StatsCollectorVerticle(long jsonIntervalMS, int maxInvalidPaths) { + public StatsCollectorVerticle(long jsonIntervalMS, int maxInvalidPaths, Set validPaths) { + this.validPaths = validPaths; pathMap = new HashMap<>(); _statsCollectorCount = new AtomicInteger(); @@ -116,8 +118,7 @@ public void handleMessage(Message message) { EndpointStat endpointStat = new EndpointStat(endpoint, siteId, apiVersion, domain); - Set endpoints = Endpoints.pathSet(); - if(endpoints.contains(path) || pathMap.containsKey(path) || (pathMap.size() < this.maxInvalidPaths + endpoints.size() && messageItem.getApiContact() != null)) { + if(validPaths.contains(path) || pathMap.containsKey(path) || (pathMap.size() < this.maxInvalidPaths + validPaths.size() && messageItem.getApiContact() != null)) { pathMap.merge(path, endpointStat, this::mergeEndpoint); } @@ -130,7 +131,7 @@ public void handleMessage(Message message) { logCycleSkipperCounter.increment(); } else { _runningSerializer = true; - if(pathMap.size() == this.maxInvalidPaths + endpoints.size()) { + if(pathMap.size() == this.maxInvalidPaths + validPaths.size()) { LOGGER.error("max invalid paths reached; a large number of invalid paths have been requested from authenticated participants"); } Object[] stats = pathMap.values().toArray(); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index b3612f187..e7e82e8c5 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -45,6 +45,7 @@ import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.ext.web.AllowForwardHeaders; +import io.vertx.ext.web.Route; import io.vertx.ext.web.Router; import io.vertx.ext.web.RoutingContext; import io.vertx.ext.web.handler.BodyHandler; @@ -130,6 +131,8 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); public final static String ORIGIN_HEADER = "Origin"; + private final Set validPaths; + public UIDOperatorVerticle(JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, @@ -142,6 +145,22 @@ public UIDOperatorVerticle(JsonObject config, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidatorService, Handler saltRetrievalResponseHandler) { + this(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, saltRetrievalResponseHandler, false); + } + + public UIDOperatorVerticle(JsonObject config, + boolean clientSideTokenGenerate, + ISiteStore siteProvider, + IClientKeyProvider clientKeyProvider, + IClientSideKeypairStore clientSideKeypairProvider, + KeyManager keyManager, + ISaltProvider saltProvider, + IOptOutStore optOutStore, + Clock clock, + IStatsCollectorQueue statsCollectorQueue, + SecureLinkValidatorService secureLinkValidatorService, + Handler saltRetrievalResponseHandler, + boolean setHealthy) { this.keyManager = keyManager; this.secureLinkValidatorService = secureLinkValidatorService; try { @@ -151,7 +170,11 @@ public UIDOperatorVerticle(JsonObject config, } this.config = config; this.clientSideTokenGenerate = clientSideTokenGenerate; - this.healthComponent.setHealthStatus(false, "not started"); + if(setHealthy) { + this.healthComponent.setHealthStatus(true); + } else { + this.healthComponent.setHealthStatus(false, "not started"); + } this.auth = new AuthMiddleware(clientKeyProvider); this.encoder = new EncryptedTokenEncoder(keyManager); this.siteProvider = siteProvider; @@ -179,6 +202,11 @@ public UIDOperatorVerticle(JsonObject config, this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, false); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); + try { + this.validPaths = createRoutesSetup().getRoutes().stream().map(Route::getPath).collect(Collectors.toSet()); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override @@ -2104,4 +2132,8 @@ public enum UserConsentStatus { INSUFFICIENT, INVALID, } + + public Set getVerticleRoutes() { + return this.validPaths; + } } diff --git a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java index a633d6cd7..cf97b42c2 100644 --- a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java +++ b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java @@ -26,8 +26,8 @@ public class StatsCollectorVerticleTest { private StatsCollectorVerticle verticle; @BeforeEach - void deployVerticle(Vertx vertx, VertxTestContext testContext) throws Throwable { - verticle = new StatsCollectorVerticle(1000, MAX_INVALID_PATHS); + void deployVerticle(Vertx vertx, VertxTestContext testContext) { + verticle = new StatsCollectorVerticle(1000, MAX_INVALID_PATHS, Endpoints.pathSet()); vertx.deployVerticle(verticle, testContext.succeeding(id -> testContext.completeNow())); } From e707deba255b529df1b8c8e4670126839e20c811 Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 19 Jul 2024 11:47:33 -0600 Subject: [PATCH 3/4] use ENUM --- src/main/java/com/uid2/operator/Const.java | 2 +- src/main/java/com/uid2/operator/Main.java | 15 ++++----- .../monitoring/StatsCollectorVerticle.java | 5 ++- .../operator/vertx/UIDOperatorVerticle.java | 33 +------------------ .../operator/StatsCollectorVerticleTest.java | 31 +++++++++++------ 5 files changed, 31 insertions(+), 55 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 09d183ddd..9b9e23a24 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -25,6 +25,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String GcpSecretVersionNameProp = "gcp_secret_version_name"; public static final String OptOutStatusApiEnabled = "optout_status_api_enabled"; public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size"; - public static String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; + public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index a7c226941..7ac3d083a 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -63,7 +63,6 @@ public class Main { private final JsonObject config; private final Vertx vertx; - private final Set validPaths; private final ApplicationVersion appVersion; private final ICloudStorage fsLocal; private final ICloudStorage fsOptOut; @@ -181,9 +180,6 @@ public Main(Vertx vertx, JsonObject config) throws Exception { } } metrics = new OperatorMetrics(getKeyManager(), saltProvider); - - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse, true); - this.validPaths = verticle.getVerticleRoutes(); } private KeyManager getKeyManager() { @@ -268,7 +264,6 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { - setupMetrics(this.validPaths); Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); @@ -370,7 +365,7 @@ private Future createAndDeployCloudSyncStoreVerticle(String name, ICloud private Future createAndDeployStatsCollector() { Promise promise = Promise.promise(); - StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50), this.validPaths); + StatsCollectorVerticle statsCollectorVerticle = new StatsCollectorVerticle(60000, config.getInteger(Const.Config.MaxInvalidPaths, 50)); vertx.deployVerticle(statsCollectorVerticle, promise); _statsCollectorQueue = statsCollectorVerticle; return promise.future(); @@ -405,7 +400,7 @@ private static Vertx createVertx() { .setLabels(EnumSet.of(Label.HTTP_METHOD, Label.HTTP_CODE, Label.HTTP_PATH)) .setJvmMetricsEnabled(true) .setEnabled(true); - BackendRegistries.setupBackend(metricOptions); + setupMetrics(metricOptions); final int threadBlockedCheckInterval = Utils.isProductionEnvironment() ? 60 * 1000 @@ -418,7 +413,9 @@ private static Vertx createVertx() { return Vertx.vertx(vertxOptions); } - private static void setupMetrics(Set validPaths) { + private static void setupMetrics(MicrometerMetricsOptions metricOptions) { + BackendRegistries.setupBackend(metricOptions); + MeterRegistry backendRegistry = BackendRegistries.getDefaultNow(); if (backendRegistry instanceof PrometheusMeterRegistry) { // prometheus specific configuration @@ -431,7 +428,7 @@ private static void setupMetrics(Set validPaths) { .meterFilter(MeterFilter.replaceTagValues(Label.HTTP_PATH.toString(), actualPath -> { try { String normalized = HttpUtils.normalizePath(actualPath).split("\\?")[0]; - return validPaths.contains(normalized) ? normalized : "/unknown"; + return Endpoints.pathSet().contains(normalized) ? normalized : "/unknown"; } catch (IllegalArgumentException e) { return actualPath; } diff --git a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java index 65d9a0886..c764010b0 100644 --- a/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java +++ b/src/main/java/com/uid2/operator/monitoring/StatsCollectorVerticle.java @@ -22,7 +22,6 @@ public class StatsCollectorVerticle extends AbstractVerticle implements IStatsCollectorQueue { private static final Logger LOGGER = LoggerFactory.getLogger(StatsCollectorVerticle.class); - private final Set validPaths; private HashMap pathMap; private static final int MAX_AVAILABLE = 1000; @@ -42,8 +41,7 @@ public class StatsCollectorVerticle extends AbstractVerticle implements IStatsCo private final ObjectMapper mapper; private final Counter queueFullCounter; - public StatsCollectorVerticle(long jsonIntervalMS, int maxInvalidPaths, Set validPaths) { - this.validPaths = validPaths; + public StatsCollectorVerticle(long jsonIntervalMS, int maxInvalidPaths) { pathMap = new HashMap<>(); _statsCollectorCount = new AtomicInteger(); @@ -118,6 +116,7 @@ public void handleMessage(Message message) { EndpointStat endpointStat = new EndpointStat(endpoint, siteId, apiVersion, domain); + Set validPaths = Endpoints.pathSet(); if(validPaths.contains(path) || pathMap.containsKey(path) || (pathMap.size() < this.maxInvalidPaths + validPaths.size() && messageItem.getApiContact() != null)) { pathMap.merge(path, endpointStat, this::mergeEndpoint); } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index e7e82e8c5..fb2625d21 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -131,8 +131,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final static List SUPPORTED_IN_APP = Arrays.asList("Android", "ios", "tvos"); public final static String ORIGIN_HEADER = "Origin"; - private final Set validPaths; - public UIDOperatorVerticle(JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, @@ -145,22 +143,6 @@ public UIDOperatorVerticle(JsonObject config, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidatorService, Handler saltRetrievalResponseHandler) { - this(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, saltRetrievalResponseHandler, false); - } - - public UIDOperatorVerticle(JsonObject config, - boolean clientSideTokenGenerate, - ISiteStore siteProvider, - IClientKeyProvider clientKeyProvider, - IClientSideKeypairStore clientSideKeypairProvider, - KeyManager keyManager, - ISaltProvider saltProvider, - IOptOutStore optOutStore, - Clock clock, - IStatsCollectorQueue statsCollectorQueue, - SecureLinkValidatorService secureLinkValidatorService, - Handler saltRetrievalResponseHandler, - boolean setHealthy) { this.keyManager = keyManager; this.secureLinkValidatorService = secureLinkValidatorService; try { @@ -170,11 +152,7 @@ public UIDOperatorVerticle(JsonObject config, } this.config = config; this.clientSideTokenGenerate = clientSideTokenGenerate; - if(setHealthy) { - this.healthComponent.setHealthStatus(true); - } else { - this.healthComponent.setHealthStatus(false, "not started"); - } + this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); this.encoder = new EncryptedTokenEncoder(keyManager); this.siteProvider = siteProvider; @@ -202,11 +180,6 @@ public UIDOperatorVerticle(JsonObject config, this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, false); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); - try { - this.validPaths = createRoutesSetup().getRoutes().stream().map(Route::getPath).collect(Collectors.toSet()); - } catch (IOException e) { - throw new RuntimeException(e); - } } @Override @@ -2132,8 +2105,4 @@ public enum UserConsentStatus { INSUFFICIENT, INVALID, } - - public Set getVerticleRoutes() { - return this.validPaths; - } } diff --git a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java index cf97b42c2..ac594c44c 100644 --- a/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java +++ b/src/test/java/com/uid2/operator/StatsCollectorVerticleTest.java @@ -27,7 +27,7 @@ public class StatsCollectorVerticleTest { @BeforeEach void deployVerticle(Vertx vertx, VertxTestContext testContext) { - verticle = new StatsCollectorVerticle(1000, MAX_INVALID_PATHS, Endpoints.pathSet()); + verticle = new StatsCollectorVerticle(1000, MAX_INVALID_PATHS); vertx.deployVerticle(verticle, testContext.succeeding(id -> testContext.completeNow())); } @@ -92,7 +92,7 @@ void testJSONSerializeWithV2AndUnknownPaths(Vertx vertx, VertxTestContext testCo } @Test - void invalidPathsFiltering(Vertx vertx, VertxTestContext testContext) throws InterruptedException, JsonProcessingException { + void allValidPathsAllowed(Vertx vertx, VertxTestContext testContext) throws InterruptedException, JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); Set validEndpoints = Endpoints.pathSet(); @@ -101,11 +101,6 @@ void invalidPathsFiltering(Vertx vertx, VertxTestContext testContext) throws Int vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); } - for(int i = 0; i < MAX_INVALID_PATHS + 5; i++) { - StatsCollectorMessageItem messageItem = new StatsCollectorMessageItem("/bad" + i, "https://test.com", "test", 1); - vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); - } - testContext.awaitCompletion(2000, TimeUnit.MILLISECONDS); String results = verticle.getEndpointStats(); @@ -122,11 +117,27 @@ void invalidPathsFiltering(Vertx vertx, VertxTestContext testContext) throws Int Assertions.assertTrue(results.contains(expected)); } - for(int i = 0; i < MAX_INVALID_PATHS; i++) { + testContext.completeNow(); + } + + @Test + void invalidPathsLimit(Vertx vertx, VertxTestContext testContext) throws InterruptedException, JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + + for(int i = 0; i < MAX_INVALID_PATHS + Endpoints.pathSet().size() + 5; i++) { + StatsCollectorMessageItem messageItem = new StatsCollectorMessageItem("/bad" + i, "https://test.com", "test", 1); + vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); + } + + testContext.awaitCompletion(2000, TimeUnit.MILLISECONDS); + String results = verticle.getEndpointStats(); + + // MAX_INVALID_PATHS is not the hard limit. The maximum paths that can be recorded, including valid ones, is MAX_INVALID_PATHS + validPaths.size * 2 + for(int i = 0; i < MAX_INVALID_PATHS + Endpoints.pathSet().size(); i++) { String expected = "{\"endpoint\":\"bad" + i + "\",\"siteId\":1,\"apiVersion\":\"v0\",\"domainList\":[{\"domain\":\"test.com\",\"count\":1,\"apiContact\":\"test\"}]}"; Assertions.assertTrue(results.contains(expected)); } - for(int i = MAX_INVALID_PATHS; i < MAX_INVALID_PATHS + 5; i++) { + for(int i = MAX_INVALID_PATHS + Endpoints.pathSet().size(); i < MAX_INVALID_PATHS + 5; i++) { String expected = "{\"endpoint\":\"bad" + i + "\",\"siteId\":1,\"apiVersion\":\"v0\",\"domainList\":[{\"domain\":\"test.com\",\"count\":1,\"apiContact\":\"test\"}]}"; Assertions.assertFalse(results.contains(expected)); } @@ -139,9 +150,9 @@ void invalidPathsFiltering(Vertx vertx, VertxTestContext testContext) throws Int vertx.eventBus().send(Const.Config.StatsCollectorEventBus, mapper.writeValueAsString(messageItem)); testContext.awaitCompletion(1000, TimeUnit.MILLISECONDS); - Assertions.assertTrue(logWatcher.list.get(0).getFormattedMessage().contains("max invalid paths reached; a large number of invalid paths have been requested from authenticated participants")); testContext.completeNow(); } + } From 48be36954c0682a2e4c1c489be9a9448124aedb1 Mon Sep 17 00:00:00 2001 From: "ian.nara" Date: Fri, 19 Jul 2024 11:48:54 -0600 Subject: [PATCH 4/4] remove unnecessary newline --- src/main/java/com/uid2/operator/Main.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 7ac3d083a..37e06fbc0 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -264,7 +264,6 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { - Supplier operatorVerticleSupplier = () -> { UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle;