From eb9577c58c97d3725e75c0784cc9f8602fda93c7 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 16 Dec 2024 14:35:54 +1100 Subject: [PATCH 01/29] Added ConfigService interface and concrete class. Added required constants and config. --- conf/default-config.json | 5 +- src/main/java/com/uid2/operator/Const.java | 3 + .../uid2/operator/service/ConfigService.java | 73 +++++++++++++++++++ .../uid2/operator/service/IConfigService.java | 7 ++ 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/uid2/operator/service/ConfigService.java create mode 100644 src/main/java/com/uid2/operator/service/IConfigService.java diff --git a/conf/default-config.json b/conf/default-config.json index 224df8906..849501309 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -35,6 +35,7 @@ "enclave_platform": null, "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, - "operator_type": "public" - + "operator_type": "public", + "core_config_url": "http://localhost:8088/config", + "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 4d32b9034..45ecd4c92 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -29,5 +29,8 @@ public class Config extends com.uid2.shared.Const.Config { public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size"; public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; + + public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised + public static final String ConfigScanPeriodMs = "config_scan_period_ms"; } } diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java new file mode 100644 index 000000000..47ccbf1d7 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -0,0 +1,73 @@ +package com.uid2.operator.service; + +import io.vertx.config.ConfigRetriever; +import io.vertx.config.ConfigRetrieverOptions; +import io.vertx.config.ConfigStoreOptions; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; + +import static com.uid2.operator.Const.Config.*; + +public class ConfigService implements IConfigService { + + private static volatile ConfigService instance; + private JsonObject config; + private ConfigRetriever configRetriever; + + + private ConfigService(Vertx vertx, JsonObject bootstrapConfig) { + this.initialiseConfigRetriever(vertx, bootstrapConfig); + } + + public static ConfigService getInstance(Vertx vertx, JsonObject bootstrapConfig) { + ConfigService configService = instance; + + if (configService == null) { + synchronized (ConfigService.class) { + configService = instance; + if (configService == null) { + instance = configService = new ConfigService(vertx, bootstrapConfig); + } + } + } + + return configService; + } + + @Override + public JsonObject getConfig() { + return config; + } + + private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { + String configUrl = bootstrapConfig.getString(CoreConfigUrl); + + ConfigStoreOptions httpStore = new ConfigStoreOptions() + .setType("http") + .setConfig(new JsonObject() + .put("url", configUrl) + .put("method", "GET")); + + ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() + .setType("json") + .setConfig(bootstrapConfig); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) + .addStore(bootstrapStore) + .addStore(httpStore); + + this.configRetriever = ConfigRetriever.create(vertx, retrieverOptions); + + this.configRetriever.getConfig(ar -> { + if (ar.succeeded()) { + this.config = ar.result(); + } else { + System.err.println("Failed to load config: " + ar.cause().getMessage()); + } + }); + + this.configRetriever.listen(change -> this.config = change.getNewConfiguration()); + + } +} diff --git a/src/main/java/com/uid2/operator/service/IConfigService.java b/src/main/java/com/uid2/operator/service/IConfigService.java new file mode 100644 index 000000000..0fb863242 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/IConfigService.java @@ -0,0 +1,7 @@ +package com.uid2.operator.service; + +import io.vertx.core.json.JsonObject; + +public interface IConfigService { + JsonObject getConfig(); +} From c98584191592b5f79b9055070144274e10c322bb Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 16 Dec 2024 17:07:53 +1100 Subject: [PATCH 02/29] Add config to RoutingContext and implement ConfigService injection --- src/main/java/com/uid2/operator/Main.java | 5 ++++- .../com/uid2/operator/vertx/UIDOperatorVerticle.java | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6b88d715b..6e25029bd 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,6 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; +import com.uid2.operator.service.ConfigService; import com.uid2.operator.service.SecureLinkValidatorService; import com.uid2.operator.service.ShutdownService; import com.uid2.operator.vertx.Endpoints; @@ -265,8 +266,10 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { + ConfigService configService = ConfigService.getInstance(vertx, config); + Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index c3784ae38..6ecc96dc6 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -84,6 +84,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String REQUEST = "request"; private final HealthComponent healthComponent = HealthManager.instance.registerComponent("http-server"); private final Cipher aesGcm; + private final IConfigService configService; private final JsonObject config; private final boolean clientSideTokenGenerate; private final AuthMiddleware auth; @@ -135,7 +136,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(JsonObject config, + public UIDOperatorVerticle(IConfigService configService, JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -154,6 +155,7 @@ public UIDOperatorVerticle(JsonObject config, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } + this.configService = configService; this.config = config; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); @@ -236,6 +238,11 @@ private Router createRoutesSetup() throws IOException { .allowedHeader("Content-Type")); router.route().handler(new StatsCollectorHandler(_statsCollectorQueue, vertx)); router.route("/static/*").handler(StaticHandler.create("static")); + router.route().handler(ctx -> { + JsonObject curConfig = configService.getConfig(); + ctx.put("config", curConfig); + ctx.next(); + }); router.route().failureHandler(new GenericFailureHandler()); final BodyHandler bodyHandler = BodyHandler.create().setHandleFileUploads(false).setBodyLimit(MAX_REQUEST_BODY_SIZE); From 83fe8d7ade3118e7c968f6dd673300d599e99efa Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 17 Dec 2024 14:10:20 +1100 Subject: [PATCH 03/29] Updated ConfigService getConfig method to use ConfigRetriever getCachedConfig method rather than storing config --- src/main/java/com/uid2/operator/service/ConfigService.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 47ccbf1d7..fcdfa7598 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -11,7 +11,6 @@ public class ConfigService implements IConfigService { private static volatile ConfigService instance; - private JsonObject config; private ConfigRetriever configRetriever; @@ -36,7 +35,7 @@ public static ConfigService getInstance(Vertx vertx, JsonObject bootstrapConfig) @Override public JsonObject getConfig() { - return config; + return configRetriever.getCachedConfig(); } private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { @@ -61,13 +60,11 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) this.configRetriever.getConfig(ar -> { if (ar.succeeded()) { - this.config = ar.result(); + System.out.println("Successfully loaded config"); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); } }); - this.configRetriever.listen(change -> this.config = change.getNewConfiguration()); - } } From 6c8aefb6a2383d39f300f8aaa4a4b4fd79f7acbc Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 17 Dec 2024 17:05:11 +1100 Subject: [PATCH 04/29] Updated UIDOperatorVerticle to use ConfigService, Mocked ConfigService in UIDOperatorVerticleTest "identity_token_expires_after_seconds", "max_bidstream_lifetime_seconds", "max_sharing_lifetime_seconds" and "sharing_token_expiry_seconds" now are retrieved from RoutingContext in handlers. --- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 2 +- .../operator/vertx/UIDOperatorVerticle.java | 51 +++++++++++-------- .../operator/ExtendedUIDOperatorVerticle.java | 6 +-- .../operator/UIDOperatorVerticleTest.java | 4 +- 5 files changed, 37 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 45ecd4c92..8326db6ce 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -32,5 +32,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised public static final String ConfigScanPeriodMs = "config_scan_period_ms"; + public static final String Config = "config"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6e25029bd..7dbf64153 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -269,7 +269,7 @@ private void run() throws Exception { ConfigService configService = ConfigService.getInstance(vertx, config); Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 6ecc96dc6..cf8fffaa8 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -67,6 +67,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.IdentityConst.*; import static com.uid2.operator.service.ResponseUtil.*; import static com.uid2.operator.vertx.Endpoints.*; @@ -85,7 +86,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final HealthComponent healthComponent = HealthManager.instance.registerComponent("http-server"); private final Cipher aesGcm; private final IConfigService configService; - private final JsonObject config; private final boolean clientSideTokenGenerate; private final AuthMiddleware auth; private final ISiteStore siteProvider; @@ -117,7 +117,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static int MASTER_KEYSET_ID_FOR_SDKS = 9999999; //this is because SDKs have an issue where they assume keyset ids are always positive; that will be fixed. public final static long OPT_OUT_CHECK_CUTOFF_DATE = Instant.parse("2023-09-01T00:00:00.00Z").getEpochSecond(); private final Handler saltRetrievalResponseHandler; - private final int maxBidstreamLifetimeSeconds; private final int allowClockSkewSeconds; protected int maxSharingLifetimeSeconds; protected Map> siteIdToInvalidOriginsAndAppNames = new HashMap<>(); @@ -136,7 +135,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(IConfigService configService, JsonObject config, + public UIDOperatorVerticle(IConfigService configService, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -156,7 +155,6 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, throw new RuntimeException(e); } this.configService = configService; - this.config = config; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); @@ -166,6 +164,7 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, this.saltProvider = saltProvider; this.optOutStore = optOutStore; this.clock = clock; + JsonObject config = configService.getConfig(); this.identityScope = IdentityScope.fromString(config.getString("identity_scope", "uid2")); this.v2PayloadHandler = new V2PayloadHandler(keyManager, config.getBoolean("enable_v2_encryption", true), this.identityScope, siteProvider); this.phoneSupport = config.getBoolean("enable_phone_support", true); @@ -175,14 +174,7 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, this._statsCollectorQueue = statsCollectorQueue; this.clientKeyProvider = clientKeyProvider; this.clientSideTokenGenerateLogInvalidHttpOrigin = config.getBoolean("client_side_token_generate_log_invalid_http_origins", false); - final Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - this.maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); - if (this.maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); - throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); - } this.allowClockSkewSeconds = config.getInteger(Const.Config.AllowClockSkewSecondsProp, 1800); - this.maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); this.saltRetrievalResponseHandler = saltRetrievalResponseHandler; this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, true); this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000); @@ -192,7 +184,7 @@ public UIDOperatorVerticle(IConfigService configService, JsonObject config, public void start(Promise startPromise) throws Exception { this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( - this.config, + this.configService.getConfig(), //TODO this.optOutStore, this.saltProvider, this.encoder, @@ -240,7 +232,7 @@ private Router createRoutesSetup() throws IOException { router.route("/static/*").handler(StaticHandler.create("static")); router.route().handler(ctx -> { JsonObject curConfig = configService.getConfig(); - ctx.put("config", curConfig); + ctx.put(Config, curConfig); ctx.next(); }); router.route().failureHandler(new GenericFailureHandler()); @@ -251,7 +243,7 @@ private Router createRoutesSetup() throws IOException { // Static and health check router.get(OPS_HEALTHCHECK.toString()).handler(this::handleHealthCheck); - if (this.config.getBoolean(Const.Config.AllowLegacyAPIProp, true)) { + if (this.configService.getConfig().getBoolean(Const.Config.AllowLegacyAPIProp, true)) { // V1 APIs router.get(V1_TOKEN_GENERATE.toString()).handler(auth.handleV1(this::handleTokenGenerateV1, Role.GENERATOR)); router.get(V1_TOKEN_VALIDATE.toString()).handler(this::handleTokenValidateV1); @@ -320,6 +312,9 @@ private void handleClientSideTokenGenerate(RoutingContext rc) { } } + private JsonObject getConfigFromRc(RoutingContext rc) { + return rc.get(Config); + } private Set getDomainNameListForClientSideTokenGenerate(ClientSideKeypair keypair) { Site s = siteProvider.getSite(keypair.getSiteId()); @@ -611,11 +606,14 @@ public void handleKeysRequest(RoutingContext rc) { } } - private String getSharingTokenExpirySeconds() { - return config.getString(Const.Config.SharingTokenExpiryProp); - } +// private String getSharingTokenExpirySeconds() { +// return config.getString(Const.Config.SharingTokenExpiryProp); +// } public void handleKeysSharing(RoutingContext rc) { + JsonObject config = this.getConfigFromRc(rc); + Integer maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp)); + String sharingTokenExpirySeconds = config.getString(Const.Config.SharingTokenExpiryProp); try { final ClientKey clientKey = AuthMiddleware.getAuthClient(ClientKey.class, rc); @@ -624,7 +622,7 @@ public void handleKeysSharing(RoutingContext rc) { Map keysetMap = keyManagerSnapshot.getAllKeysets(); final JsonObject resp = new JsonObject(); - addSharingHeaderFields(resp, keyManagerSnapshot, clientKey); + addSharingHeaderFields(resp, keyManagerSnapshot, clientKey, maxSharingLifetimeSeconds, sharingTokenExpirySeconds); final List accessibleKeys = getAccessibleKeys(keysetKeyStore, keyManagerSnapshot, clientKey); @@ -669,14 +667,23 @@ public void handleKeysBidstream(RoutingContext rc) { .collect(Collectors.toList()); final JsonObject resp = new JsonObject(); - addBidstreamHeaderFields(resp); + + JsonObject config = this.getConfigFromRc(rc); + Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); + if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { + LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); + throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); + } + + addBidstreamHeaderFields(resp, maxBidstreamLifetimeSeconds); resp.put("keys", keysJson); addSites(resp, accessibleKeys, keysetMap); ResponseUtil.SuccessV2(rc, resp); } - private void addBidstreamHeaderFields(JsonObject resp) { + private void addBidstreamHeaderFields(JsonObject resp, Integer maxBidstreamLifetimeSeconds) { resp.put("max_bidstream_lifetime_seconds", maxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds()); addIdentityScopeField(resp); addAllowClockSkewSecondsField(resp); @@ -714,7 +721,7 @@ private void addSites(JsonObject resp, List keys, Map saltRetrievalResponseHandler) { - super(config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); + super(configService, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); } public IUIDOperatorService getIdService() { diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index d9a91ae01..8a2888d61 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -117,6 +117,7 @@ public class UIDOperatorVerticleTest { @Mock private Clock clock; @Mock private IStatsCollectorQueue statsCollectorQueue; @Mock private OperatorShutdownHandler shutdownHandler; + @Mock private IConfigService configService; private SimpleMeterRegistry registry; private ExtendedUIDOperatorVerticle uidOperatorVerticle; @@ -134,8 +135,9 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t if(testInfo.getDisplayName().equals("cstgNoPhoneSupport(Vertx, VertxTestContext)")) { config.put("enable_phone_support", false); } + when(configService.getConfig()).thenReturn(config); - this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); + this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); vertx.deployVerticle(uidOperatorVerticle, testContext.succeeding(id -> testContext.completeNow())); From c7a788ffcca4373a0ad4b882a56b3e077e8d4dc3 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 09:51:43 +1100 Subject: [PATCH 05/29] Updated UIDOperatorService to get "identity_v3" config from UIDOperatorVerticle --- .../java/com/uid2/operator/service/UIDOperatorService.java | 4 ++-- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 6d4ff86d0..80a1a5735 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -52,7 +52,7 @@ public class UIDOperatorService implements IUIDOperatorService { private final Handler saltRetrievalResponseHandler; public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, - IdentityScope identityScope, Handler saltRetrievalResponseHandler) { + IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { this.saltProvider = saltProvider; this.encoder = encoder; this.optOutStore = optOutStore; @@ -90,7 +90,7 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv } this.refreshTokenVersion = TokenVersion.V3; - this.identityV3Enabled = config.getBoolean("identity_v3", false); + this.identityV3Enabled = identityV3Enabled; } @Override diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index cf8fffaa8..4cf97d9d5 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -182,6 +182,7 @@ public UIDOperatorVerticle(IConfigService configService, @Override public void start(Promise startPromise) throws Exception { + Boolean identityV3Enabled = this.configService.getConfig().getBoolean("identity_v3", false); this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( this.configService.getConfig(), //TODO @@ -190,7 +191,8 @@ public void start(Promise startPromise) throws Exception { this.encoder, this.clock, this.identityScope, - this.saltRetrievalResponseHandler + this.saltRetrievalResponseHandler, + identityV3Enabled ); final Router router = createRoutesSetup(); From cfd6a6a21b2a3fbd25d24aa2f1c4935c9096690d Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 11:57:33 +1100 Subject: [PATCH 06/29] Added ConfigValidatorUtil, Added configValidationHandler method to ConfigService Moved validation logic from UIDOperatorVerticle and UIDOperatorService to ConfigValidatorUtil configValidationHandler is set as a config processor for configRetriever in ConfigService --- .../uid2/operator/service/ConfigService.java | 32 ++++++++++++++++- .../operator/service/ConfigValidatorUtil.java | 34 +++++++++++++++++++ .../operator/service/UIDOperatorService.java | 10 ------ .../operator/vertx/UIDOperatorVerticle.java | 5 +-- 4 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index fcdfa7598..4f95b8696 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -1,18 +1,23 @@ package com.uid2.operator.service; +import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; import io.vertx.config.ConfigRetrieverOptions; import io.vertx.config.ConfigStoreOptions; import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.uid2.operator.Const.Config.*; +import static com.uid2.operator.service.ConfigValidatorUtil.*; +import static com.uid2.operator.service.UIDOperatorService.*; public class ConfigService implements IConfigService { private static volatile ConfigService instance; private ConfigRetriever configRetriever; - + private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); private ConfigService(Vertx vertx, JsonObject bootstrapConfig) { this.initialiseConfigRetriever(vertx, bootstrapConfig); @@ -58,6 +63,8 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) this.configRetriever = ConfigRetriever.create(vertx, retrieverOptions); + this.configRetriever.setConfigurationProcessor(this::configValidationHandler); + this.configRetriever.getConfig(ar -> { if (ar.succeeded()) { System.out.println("Successfully loaded config"); @@ -67,4 +74,27 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) }); } + + private JsonObject configValidationHandler(JsonObject config) { + boolean isValid = true; + Integer identityExpiresAfter = config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + Integer refreshExpiresAfter = config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); + Integer refreshIdentityAfter = config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp); + + isValid &= validateIdentityRefreshTokens(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter); + + isValid &= validateBidstreamLifetime(maxBidstreamLifetimeSeconds, identityExpiresAfter); + + if (!isValid) { + logger.error("Failed to update config"); + JsonObject lastConfig = this.getConfig(); + if (lastConfig == null || lastConfig.isEmpty()) { + throw new RuntimeException("Invalid config retrieved and no previous config to revert to"); + } + return lastConfig; + } + + return config; + } } diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java new file mode 100644 index 000000000..4f1c0999c --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -0,0 +1,34 @@ +package com.uid2.operator.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.uid2.operator.service.UIDOperatorService.*; + +public class ConfigValidatorUtil { + private static final Logger logger = LoggerFactory.getLogger(ConfigValidatorUtil.class); + + public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter, Integer refreshExpiresAfter, Integer refreshIdentityAfter) { + boolean isValid = true; + if (identityExpiresAfter > refreshExpiresAfter) { + logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + isValid = false; + } + if (refreshIdentityAfter > identityExpiresAfter) { + logger.error(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + isValid = false; + } + if (refreshIdentityAfter > refreshExpiresAfter) { + logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + } + return isValid; + } + + public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { + if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { + logger.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); + return false; + } + return true; + } +} diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 80a1a5735..846e4a475 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -79,16 +79,6 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv this.refreshExpiresAfter = Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); this.refreshIdentityAfter = Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); - if (this.identityExpiresAfter.compareTo(this.refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - } - if (this.refreshIdentityAfter.compareTo(this.identityExpiresAfter) > 0) { - throw new IllegalStateException(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - } - if (this.refreshIdentityAfter.compareTo(this.refreshExpiresAfter) > 0) { - throw new IllegalStateException(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - } - this.refreshTokenVersion = TokenVersion.V3; this.identityV3Enabled = identityV3Enabled; } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4cf97d9d5..ed67f71a7 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -673,10 +673,7 @@ public void handleKeysBidstream(RoutingContext rc) { JsonObject config = this.getConfigFromRc(rc); Integer identityTokenExpiresAfterSeconds = config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityTokenExpiresAfterSeconds); - if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { - LOGGER.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); - throw new RuntimeException("Max bidstream lifetime seconds is less than identity token lifetime seconds"); - } + addBidstreamHeaderFields(resp, maxBidstreamLifetimeSeconds); resp.put("keys", keysJson); From 975f6ab89e6813bc446226b22efe84056ab582d4 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 14:54:42 +1100 Subject: [PATCH 07/29] Updated UIDOperatorService to take config as method parameters, Update UIDOperatorServiceTest --- src/main/java/com/uid2/operator/Const.java | 1 + .../operator/service/IUIDOperatorService.java | 6 +- .../operator/service/UIDOperatorService.java | 33 +-- .../operator/vertx/UIDOperatorVerticle.java | 82 ++++++-- .../uid2/operator/UIDOperatorServiceTest.java | 189 ++++++++++++++---- .../operator/benchmark/BenchmarkCommon.java | 29 ++- .../benchmark/TokenEndecBenchmark.java | 28 ++- 7 files changed, 272 insertions(+), 96 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 8326db6ce..084e8f3bb 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -33,5 +33,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; + public static final String identityV3 = "identity_v3"; } } diff --git a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java index c64e499fe..567f17eb4 100644 --- a/src/main/java/com/uid2/operator/service/IUIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/IUIDOperatorService.java @@ -11,9 +11,9 @@ public interface IUIDOperatorService { - IdentityTokens generateIdentity(IdentityRequest request); + IdentityTokens generateIdentity(IdentityRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter); - RefreshResponse refreshIdentity(RefreshToken refreshToken); + RefreshResponse refreshIdentity(RefreshToken token, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter); MappedIdentity mapIdentity(MapRequest request); @@ -27,6 +27,4 @@ public interface IUIDOperatorService { boolean advertisingTokenMatches(String advertisingToken, UserIdentity userIdentity, Instant asOf); Instant getLatestOptoutEntry(UserIdentity userIdentity, Instant asOf); - - Duration getIdentityExpiryDuration(); } diff --git a/src/main/java/com/uid2/operator/service/UIDOperatorService.java b/src/main/java/com/uid2/operator/service/UIDOperatorService.java index 846e4a475..2c20782fe 100644 --- a/src/main/java/com/uid2/operator/service/UIDOperatorService.java +++ b/src/main/java/com/uid2/operator/service/UIDOperatorService.java @@ -9,7 +9,6 @@ import io.vertx.core.AsyncResult; import io.vertx.core.Future; import io.vertx.core.Handler; -import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,9 +40,6 @@ public class UIDOperatorService implements IUIDOperatorService { private final UserIdentity testValidateIdentityForPhone; private final UserIdentity testRefreshOptOutIdentityForEmail; private final UserIdentity testRefreshOptOutIdentityForPhone; - private final Duration identityExpiresAfter; - private final Duration refreshExpiresAfter; - private final Duration refreshIdentityAfter; private final OperatorIdentity operatorIdentity; private final TokenVersion refreshTokenVersion; @@ -51,7 +47,7 @@ public class UIDOperatorService implements IUIDOperatorService { private final Handler saltRetrievalResponseHandler; - public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, + public UIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { this.saltProvider = saltProvider; this.encoder = encoder; @@ -75,16 +71,12 @@ public UIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProv this.operatorIdentity = new OperatorIdentity(0, OperatorType.Service, 0, 0); - this.identityExpiresAfter = Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); - this.refreshExpiresAfter = Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); - this.refreshIdentityAfter = Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); - this.refreshTokenVersion = TokenVersion.V3; this.identityV3Enabled = identityV3Enabled; } @Override - public IdentityTokens generateIdentity(IdentityRequest request) { + public IdentityTokens generateIdentity(IdentityRequest request, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { final Instant now = EncodingUtils.NowUTCMillis(this.clock); final byte[] firstLevelHash = getFirstLevelHash(request.userIdentity.id, now); final UserIdentity firstLevelHashIdentity = new UserIdentity( @@ -94,12 +86,12 @@ public IdentityTokens generateIdentity(IdentityRequest request) { if (request.shouldCheckOptOut() && getGlobalOptOutResult(firstLevelHashIdentity, false).isOptedOut()) { return IdentityTokens.LogoutToken; } else { - return generateIdentity(request.publisherIdentity, firstLevelHashIdentity); + return this.generateIdentity(request.publisherIdentity, firstLevelHashIdentity, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); } } @Override - public RefreshResponse refreshIdentity(RefreshToken token) { + public RefreshResponse refreshIdentity(RefreshToken token, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { // should not be possible as different scopes should be using different keys, but just in case if (token.userIdentity.identityScope != this.identityScope) { return RefreshResponse.Invalid; @@ -125,7 +117,7 @@ public RefreshResponse refreshIdentity(RefreshToken token) { final Duration durationSinceLastRefresh = Duration.between(token.createdAt, now); if (!optedOut) { - IdentityTokens identityTokens = this.generateIdentity(token.publisherIdentity, token.userIdentity); + IdentityTokens identityTokens = this.generateIdentity(token.publisherIdentity, token.userIdentity, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); return RefreshResponse.createRefreshedResponse(identityTokens, durationSinceLastRefresh, isCstg); } else { @@ -198,11 +190,6 @@ public Instant getLatestOptoutEntry(UserIdentity userIdentity, Instant asOf) { return this.optOutStore.getLatestEntry(firstLevelHashIdentity); } - @Override - public Duration getIdentityExpiryDuration() { - return this.identityExpiresAfter; - } - private UserIdentity getFirstLevelHashIdentity(UserIdentity userIdentity, Instant asOf) { return getFirstLevelHashIdentity(userIdentity.identityScope, userIdentity.identityType, userIdentity.id, asOf); } @@ -226,7 +213,7 @@ private MappedIdentity getAdvertisingId(UserIdentity firstLevelHashIdentity, Ins rotatingSalt.getHashedId()); } - private IdentityTokens generateIdentity(PublisherIdentity publisherIdentity, UserIdentity firstLevelHashIdentity) { + private IdentityTokens generateIdentity(PublisherIdentity publisherIdentity, UserIdentity firstLevelHashIdentity, Duration refreshIdentityAfter, Duration refreshExpiresAfter, Duration identityExpiresAfter) { final Instant nowUtc = EncodingUtils.NowUTCMillis(this.clock); final MappedIdentity mappedIdentity = getAdvertisingId(firstLevelHashIdentity, nowUtc); @@ -234,14 +221,14 @@ private IdentityTokens generateIdentity(PublisherIdentity publisherIdentity, Use mappedIdentity.advertisingId, firstLevelHashIdentity.privacyBits, firstLevelHashIdentity.establishedAt, nowUtc); return this.encoder.encode( - this.createAdvertisingToken(publisherIdentity, advertisingIdentity, nowUtc), - this.createRefreshToken(publisherIdentity, firstLevelHashIdentity, nowUtc), + this.createAdvertisingToken(publisherIdentity, advertisingIdentity, nowUtc, identityExpiresAfter), + this.createRefreshToken(publisherIdentity, firstLevelHashIdentity, nowUtc, refreshExpiresAfter), nowUtc.plusMillis(refreshIdentityAfter.toMillis()), nowUtc ); } - private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { + private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now, Duration refreshExpiresAfter) { return new RefreshToken( this.refreshTokenVersion, now, @@ -251,7 +238,7 @@ private RefreshToken createRefreshToken(PublisherIdentity publisherIdentity, Use userIdentity); } - private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now) { + private AdvertisingToken createAdvertisingToken(PublisherIdentity publisherIdentity, UserIdentity userIdentity, Instant now, Duration identityExpiresAfter) { return new AdvertisingToken(TokenVersion.V4, now, now.plusMillis(identityExpiresAfter.toMillis()), this.operatorIdentity, publisherIdentity, userIdentity); } diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index ed67f71a7..2c2c275ff 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -182,10 +182,9 @@ public UIDOperatorVerticle(IConfigService configService, @Override public void start(Promise startPromise) throws Exception { - Boolean identityV3Enabled = this.configService.getConfig().getBoolean("identity_v3", false); + Boolean identityV3Enabled = this.configService.getConfig().getBoolean(identityV3, false); this.healthComponent.setHealthStatus(false, "still starting"); this.idService = new UIDOperatorService( - this.configService.getConfig(), //TODO this.optOutStore, this.saltProvider, this.encoder, @@ -337,6 +336,13 @@ private Set getAppNames(ClientSideKeypair keypair) { private void handleClientSideTokenGenerateImpl(RoutingContext rc) throws NoSuchAlgorithmException, InvalidKeyException { final JsonObject body; + + JsonObject config = this.getConfigFromRc(rc); + + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; try { body = rc.body().asJsonObject(); @@ -474,7 +480,10 @@ else if(emailHash != null) { new IdentityRequest( new PublisherIdentity(clientSideKeypair.getSiteId(), 0, 0), input.toUserIdentity(this.identityScope, privacyBits.getAsInt(), Instant.now()), - OptoutCheckPolicy.RespectOptOut)); + OptoutCheckPolicy.RespectOptOut), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); } catch (KeyManager.NoActiveKeyException e){ SendServerErrorResponseAndRecordStats(rc, "No active encryption key available", clientSideKeypair.getSiteId(), TokenResponseStatsCollector.Endpoint.ClientSideTokenGenerateV2, TokenResponseStatsCollector.ResponseStatus.NoActiveKey, siteProvider, e, platformType); return; @@ -825,6 +834,10 @@ private void handleTokenRefreshV1(RoutingContext rc) { } } + JsonObject config = this.getConfigFromRc(rc); + + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { final RefreshResponse r = this.refreshIdentity(rc, refreshToken); siteId = rc.get(Const.RoutingContextData.SiteId); @@ -843,7 +856,7 @@ private void handleTokenRefreshV1(RoutingContext rc) { } } else { ResponseUtil.Success(rc, toJsonV1(r.getTokens())); - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV1, r, platformType); @@ -855,6 +868,9 @@ private void handleTokenRefreshV1(RoutingContext rc) { private void handleTokenRefreshV2(RoutingContext rc) { Integer siteId = null; TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + JsonObject config = this.getConfigFromRc(rc); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); try { platformType = getPlatformType(rc); String tokenStr = (String) rc.data().get("request"); @@ -877,7 +893,7 @@ private void handleTokenRefreshV2(RoutingContext rc) { } } else { ResponseUtil.SuccessV2(rc, toJsonV1(r.getTokens())); - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV2, r, platformType); } catch (Exception e) { @@ -948,6 +964,12 @@ private void handleTokenValidateV2(RoutingContext rc) { private void handleTokenGenerateV1(RoutingContext rc) { final int siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { final InputUtil.InputVal input = this.phoneSupport ? this.getTokenInputV1(rc) : this.getTokenInput(rc); platformType = getPlatformType(rc); @@ -956,7 +978,10 @@ private void handleTokenGenerateV1(RoutingContext rc) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); + OptoutCheckPolicy.defaultPolicy()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); ResponseUtil.Success(rc, toJsonV1(t)); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV1, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), platformType); @@ -969,6 +994,12 @@ private void handleTokenGenerateV1(RoutingContext rc) { private void handleTokenGenerateV2(RoutingContext rc) { final Integer siteId = AuthMiddleware.getAuthClient(rc).getSiteId(); TokenResponseStatsCollector.PlatformType platformType = TokenResponseStatsCollector.PlatformType.Other; + + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { JsonObject req = (JsonObject) rc.data().get("request"); platformType = getPlatformType(rc); @@ -1009,7 +1040,10 @@ private void handleTokenGenerateV2(RoutingContext rc) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.respectOptOut())); + OptoutCheckPolicy.respectOptOut()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); if (t.isEmptyToken()) { if (optoutCheckPolicy.getItem1() == OptoutCheckPolicy.DoNotRespect) { // only legacy can use this policy @@ -1025,7 +1059,10 @@ private void handleTokenGenerateV2(RoutingContext rc) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), optOutTokenInput.toUserIdentity(this.identityScope, pb.getAsInt(), Instant.now()), - OptoutCheckPolicy.DoNotRespect)); + OptoutCheckPolicy.DoNotRespect), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); ResponseUtil.SuccessV2(rc, toJsonV1(optOutTokens)); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV2, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, optOutTokens.getAdvertisingTokenVersion(), platformType); @@ -1050,6 +1087,13 @@ private void handleTokenGenerateV2(RoutingContext rc) { private void handleTokenGenerate(RoutingContext rc) { final InputUtil.InputVal input = this.getTokenInput(rc); Integer siteId = null; + + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + + if (input == null) { SendClientErrorResponseAndRecordStats(ResponseStatus.ClientError, 400, rc, ERROR_INVALID_INPUT_EMAIL_MISSING, siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.BadPayload, siteProvider, TokenResponseStatsCollector.PlatformType.Other); return; @@ -1065,7 +1109,10 @@ else if (!input.isValid()) { new IdentityRequest( new PublisherIdentity(siteId, 0, 0), input.toUserIdentity(this.identityScope, 1, Instant.now()), - OptoutCheckPolicy.defaultPolicy())); + OptoutCheckPolicy.defaultPolicy()), + refreshIdentityAfter, + refreshExpiresAfter, + identityExpiresAfter); recordTokenResponseStats(siteId, TokenResponseStatsCollector.Endpoint.GenerateV0, TokenResponseStatsCollector.ResponseStatus.Success, siteProvider, t.getAdvertisingTokenVersion(), TokenResponseStatsCollector.PlatformType.Other); sendJsonResponse(rc, toJson(t)); @@ -1083,6 +1130,10 @@ private void handleTokenRefresh(RoutingContext rc) { return; } + JsonObject config = this.getConfigFromRc(rc); + + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + try { final RefreshResponse r = this.refreshIdentity(rc, tokenList.get(0)); @@ -1090,7 +1141,7 @@ private void handleTokenRefresh(RoutingContext rc) { siteId = rc.get(Const.RoutingContextData.SiteId); if (r.isRefreshed()) { - this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER)); + this.recordRefreshDurationStats(siteId, getApiContact(rc), r.getDurationSinceLastRefresh(), rc.request().headers().contains(ORIGIN_HEADER), identityExpiresAfter); } TokenResponseStatsCollector.recordRefresh(siteProvider, siteId, TokenResponseStatsCollector.Endpoint.RefreshV0, r, TokenResponseStatsCollector.PlatformType.Other); } catch (Exception e) { @@ -1797,7 +1848,12 @@ private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) { } recordRefreshTokenVersionCount(String.valueOf(rc.data().get(Const.RoutingContextData.SiteId)), this.getRefreshTokenVersion(tokenStr)); - return this.idService.refreshIdentity(refreshToken); + JsonObject config = this.getConfigFromRc(rc); + Duration refreshIdentityAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)); + Duration refreshExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)); + Duration identityExpiresAfter = Duration.ofSeconds(config.getInteger(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); + + return this.idService.refreshIdentity(refreshToken, refreshIdentityAfter, refreshExpiresAfter, identityExpiresAfter); } public static String getSiteName(ISiteStore siteStore, Integer siteId) { @@ -1823,7 +1879,7 @@ private TokenResponseStatsCollector.PlatformType getPlatformType(RoutingContext return origin != null ? TokenResponseStatsCollector.PlatformType.HasOriginHeader : TokenResponseStatsCollector.PlatformType.Other; } - private void recordRefreshDurationStats(Integer siteId, String apiContact, Duration durationSinceLastRefresh, boolean hasOriginHeader) { + private void recordRefreshDurationStats(Integer siteId, String apiContact, Duration durationSinceLastRefresh, boolean hasOriginHeader, Duration identityExpiresAfter) { DistributionSummary ds = _refreshDurationMetricSummaries.computeIfAbsent(new Tuple.Tuple2<>(apiContact, hasOriginHeader), k -> DistributionSummary .builder("uid2.token_refresh_duration_seconds") @@ -1836,7 +1892,7 @@ private void recordRefreshDurationStats(Integer siteId, String apiContact, Durat ); ds.record(durationSinceLastRefresh.getSeconds()); - boolean isExpired = durationSinceLastRefresh.compareTo(this.idService.getIdentityExpiryDuration()) > 0; + boolean isExpired = durationSinceLastRefresh.compareTo(identityExpiresAfter) > 0; Counter c = _advertisingTokenExpiryStatus.computeIfAbsent(new Tuple.Tuple3<>(String.valueOf(siteId), hasOriginHeader, isExpired), k -> Counter .builder("uid2.advertising_token_expired_on_refresh") diff --git a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java index 37eeef36f..a818d29df 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorServiceTest.java @@ -21,6 +21,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + +import static com.uid2.operator.Const.Config.identityV3; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.params.ParameterizedTest; @@ -31,6 +33,7 @@ import java.nio.charset.StandardCharsets; import java.security.Security; import java.time.Clock; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -54,8 +57,8 @@ public class UIDOperatorServiceTest { final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; class ExtendedUIDOperatorService extends UIDOperatorService { - public ExtendedUIDOperatorService(JsonObject config, IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler) { - super(config, optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler); + public ExtendedUIDOperatorService(IOptOutStore optOutStore, ISaltProvider saltProvider, ITokenEncoder encoder, Clock clock, IdentityScope identityScope, Handler saltRetrievalResponseHandler, Boolean identityV3Enabled) { + super(optOutStore, saltProvider, encoder, clock, identityScope, saltRetrievalResponseHandler, identityV3Enabled); } } @@ -88,32 +91,32 @@ void setup() throws Exception { uid2Config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); uid2Config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - uid2Config.put("identity_v3", false); + uid2Config.put(identityV3, false); uid2Service = new ExtendedUIDOperatorService( - uid2Config, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.UID2, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(identityV3) ); euidConfig = new JsonObject(); euidConfig.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); euidConfig.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - euidConfig.put("identity_v3", true); + euidConfig.put(identityV3, true); euidService = new ExtendedUIDOperatorService( - euidConfig, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.EUID, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + euidConfig.getBoolean(identityV3) ); } @@ -157,7 +160,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { createUserIdentity("test-email-hash", IdentityScope.UID2, IdentityType.Email), OptoutCheckPolicy.DoNotRespect ); - final IdentityTokens tokens = uid2Service.generateIdentity(identityRequest); + final IdentityTokens tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokens); @@ -176,7 +183,11 @@ public void testGenerateAndRefresh(int siteId, TokenVersion tokenVersion) { setNow(Instant.now().plusSeconds(200)); reset(shutdownHandler); - final RefreshResponse refreshResponse = uid2Service.refreshIdentity(refreshToken); + final RefreshResponse refreshResponse = uid2Service.refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(refreshResponse); @@ -207,14 +218,22 @@ public void testTestOptOutKey_DoNotRespectOptout() { inputVal.toUserIdentity(IdentityScope.UID2, 0, this.now), OptoutCheckPolicy.DoNotRespect ); - final IdentityTokens tokens = uid2Service.generateIdentity(identityRequest); + final IdentityTokens tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokens); assertFalse(tokens.isEmptyToken()); final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); - assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Optout, uid2Service.refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } @Test @@ -226,7 +245,11 @@ public void testTestOptOutKey_RespectOptout() { inputVal.toUserIdentity(IdentityScope.UID2, 0, this.now), OptoutCheckPolicy.RespectOptOut ); - final IdentityTokens tokens = uid2Service.generateIdentity(identityRequest); + final IdentityTokens tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); assertTrue(tokens.isEmptyToken()); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -242,14 +265,22 @@ public void testTestOptOutKeyIdentityScopeMismatch() { inputVal.toUserIdentity(IdentityScope.EUID, 0, this.now), OptoutCheckPolicy.DoNotRespect ); - final IdentityTokens tokens = euidService.generateIdentity(identityRequest); + final IdentityTokens tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); assertNotNull(tokens); final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Invalid, uid2Service.refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -279,20 +310,36 @@ public void testGenerateTokenForOptOutUser(IdentityType type, String identity, I final AdvertisingToken advertisingToken; final IdentityTokens tokensAfterOptOut; if (scope == IdentityScope.UID2) { - tokens = uid2Service.generateIdentity(identityRequestForceGenerate); + tokens = uid2Service.generateIdentity( + identityRequestForceGenerate, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.UID2, userIdentity.identityType, identityRequestRespectOptOut.publisherIdentity.siteId); reset(shutdownHandler); - tokensAfterOptOut = uid2Service.generateIdentity(identityRequestRespectOptOut); + tokensAfterOptOut = uid2Service.generateIdentity( + identityRequestRespectOptOut, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = euidService.generateIdentity(identityRequestForceGenerate); + tokens = euidService.generateIdentity( + identityRequestForceGenerate, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.EUID, userIdentity.identityType, identityRequestRespectOptOut.publisherIdentity.siteId); reset(shutdownHandler); - tokensAfterOptOut = euidService.generateIdentity(identityRequestRespectOptOut); + tokensAfterOptOut = euidService.generateIdentity( + identityRequestRespectOptOut, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -403,10 +450,18 @@ void testSpecialIdentityOptOutTokenGenerate(TestIdentityInputType type, String i IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -466,10 +521,18 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -481,7 +544,11 @@ void testSpecialIdentityOptOutTokenRefresh(TestIdentityInputType type, String id final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -508,10 +575,18 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -523,7 +598,11 @@ void testSpecialIdentityRefreshOptOutGenerate(TestIdentityInputType type, String final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken)); + assertEquals(RefreshResponse.Optout, (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); verify(shutdownHandler, never()).handleSaltRetrievalResponse(anyBoolean()); } @@ -584,10 +663,18 @@ void testSpecialIdentityValidateGenerate(TestIdentityInputType type, String id, IdentityTokens tokens; AdvertisingToken advertisingToken; if (scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), scope, identityRequest.userIdentity.identityType, identityRequest.publisherIdentity.siteId); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); @@ -647,10 +734,18 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop ); IdentityTokens tokens; if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(false); verify(shutdownHandler, never()).handleSaltRetrievalResponse(true); @@ -658,7 +753,11 @@ void testNormalIdentityOptIn(TestIdentityInputType type, String id, IdentityScop assertNotNull(tokens); final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken); + RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); assertTrue(refreshResponse.isRefreshed()); assertNotNull(refreshResponse.getTokens()); assertNotEquals(RefreshResponse.Optout, refreshResponse); @@ -678,23 +777,23 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String saltProvider.loadContent(); UIDOperatorService uid2Service = new UIDOperatorService( - uid2Config, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.UID2, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + uid2Config.getBoolean(identityV3) ); UIDOperatorService euidService = new UIDOperatorService( - euidConfig, optOutStore, saltProvider, tokenEncoder, this.clock, IdentityScope.EUID, - this.shutdownHandler::handleSaltRetrievalResponse + this.shutdownHandler::handleSaltRetrievalResponse, + euidConfig.getBoolean(identityV3) ); when(this.optOutStore.getLatestEntry(any())).thenReturn(null); @@ -710,11 +809,19 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String AdvertisingToken advertisingToken; reset(shutdownHandler); if(scope == IdentityScope.EUID) { - tokens = euidService.generateIdentity(identityRequest); + tokens = euidService.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.EUID, identityRequest.userIdentity.identityType, identityRequest.publisherIdentity.siteId); } else { - tokens = uid2Service.generateIdentity(identityRequest); + tokens = uid2Service.generateIdentity( + identityRequest, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); advertisingToken = validateAndGetToken(tokenEncoder, tokens.getAdvertisingToken(), IdentityScope.UID2, identityRequest.userIdentity.identityType, identityRequest.publisherIdentity.siteId); } verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); @@ -725,7 +832,11 @@ void testExpiredSaltsNotifiesShutdownHandler(TestIdentityInputType type, String final RefreshToken refreshToken = this.tokenEncoder.decodeRefreshToken(tokens.getRefreshToken()); reset(shutdownHandler); - RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity(refreshToken); + RefreshResponse refreshResponse = (scope == IdentityScope.EUID? euidService: uid2Service).refreshIdentity( + refreshToken, + Duration.ofSeconds(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS), + Duration.ofSeconds(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS), + Duration.ofSeconds(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)); verify(shutdownHandler, atLeastOnce()).handleSaltRetrievalResponse(true); verify(shutdownHandler, never()).handleSaltRetrievalResponse(false); assertTrue(refreshResponse.isRefreshed()); diff --git a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java index 4cc327e9f..11b0f4a3b 100644 --- a/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java +++ b/src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java @@ -41,6 +41,8 @@ import java.util.List; import java.util.Random; +import static com.uid2.operator.Const.Config.identityV3; + public class BenchmarkCommon { static IUIDOperatorService createUidOperatorService() throws Exception { @@ -59,16 +61,9 @@ static IUIDOperatorService createUidOperatorService() throws Exception { "/com.uid2.core/test/salts/metadata.json"); saltProvider.loadContent(); - final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; - final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; - final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; - - final JsonObject config = new JsonObject(); - config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); - config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + final JsonObject config = getConfig(); - final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); + final EncryptedTokenEncoder tokenEncoder = new EncryptedTokenEncoder(new KeyManager(keysetKeyStore, keysetProvider)); final List optOutPartitionFiles = new ArrayList<>(); final ICloudStorage optOutLocalStorage = make1mOptOutEntryStorage( saltProvider.getSnapshot(Instant.now()).getFirstLevelSalt(), @@ -76,16 +71,28 @@ static IUIDOperatorService createUidOperatorService() throws Exception { final IOptOutStore optOutStore = new StaticOptOutStore(optOutLocalStorage, make1mOptOutEntryConfig(), optOutPartitionFiles); return new UIDOperatorService( - config, optOutStore, saltProvider, tokenEncoder, Clock.systemUTC(), IdentityScope.UID2, - null + null, + config.getBoolean(identityV3) ); } + public static JsonObject getConfig() { + final int IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS = 600; + final int REFRESH_TOKEN_EXPIRES_AFTER_SECONDS = 900; + final int REFRESH_IDENTITY_TOKEN_AFTER_SECONDS = 300; + + final JsonObject config = new JsonObject(); + config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); + config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); + config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); + return config; + } + static EncryptedTokenEncoder createTokenEncoder() throws Exception { RotatingKeysetKeyStore keysetKeyStore = new RotatingKeysetKeyStore( new EmbeddedResourceStorage(Main.class), diff --git a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java index aaa821db9..d1fba74be 100644 --- a/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java +++ b/src/test/java/com/uid2/operator/benchmark/TokenEndecBenchmark.java @@ -3,13 +3,17 @@ import com.uid2.operator.model.*; import com.uid2.operator.service.EncryptedTokenEncoder; import com.uid2.operator.service.IUIDOperatorService; +import io.vertx.core.json.JsonObject; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; +import java.time.Duration; import java.util.ArrayList; import java.util.List; +import static com.uid2.operator.service.UIDOperatorService.*; + public class TokenEndecBenchmark { private static final IUIDOperatorService uidService; @@ -18,6 +22,7 @@ public class TokenEndecBenchmark { private static final EncryptedTokenEncoder encoder; private static final IdentityTokens[] generatedTokens; private static int idx = 0; + private static final JsonObject config; static { try { @@ -29,6 +34,7 @@ public class TokenEndecBenchmark { if (generatedTokens.length < 65536 || userIdentities.length < 65536) { throw new IllegalStateException("must create more than 65535 test candidates."); } + config = BenchmarkCommon.getConfig(); } catch (Exception e) { throw new RuntimeException(e); } @@ -38,10 +44,14 @@ static IdentityTokens[] createAdvertisingTokens() { List tokens = new ArrayList<>(); for (int i = 0; i < userIdentities.length; i++) { tokens.add( - uidService.generateIdentity(new IdentityRequest( - publisher, - userIdentities[i], - OptoutCheckPolicy.DoNotRespect))); + uidService.generateIdentity( + new IdentityRequest( + publisher, + userIdentities[i], + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS)))); } return tokens.toArray(new IdentityTokens[tokens.size()]); } @@ -52,7 +62,10 @@ public IdentityTokens TokenGenerationBenchmark() { return uidService.generateIdentity(new IdentityRequest( publisher, userIdentities[(idx++) & 65535], - OptoutCheckPolicy.DoNotRespect)); + OptoutCheckPolicy.DoNotRespect), + Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } @Benchmark @@ -60,6 +73,9 @@ public IdentityTokens TokenGenerationBenchmark() { public RefreshResponse TokenRefreshBenchmark() { return uidService.refreshIdentity( encoder.decodeRefreshToken( - generatedTokens[(idx++) & 65535].getRefreshToken())); + generatedTokens[(idx++) & 65535].getRefreshToken()), + Duration.ofSeconds(config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS)), + Duration.ofSeconds(config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS))); } } From 491025bc54f3f53da3aebb2ae6012e19e85f964f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 15:15:17 +1100 Subject: [PATCH 08/29] Removed setMaxSharingLifetimeSeconds metho from ExtendedUIDOperatorVerticle, Updated UIDOperatorVerticleTest --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 1 - .../java/com/uid2/operator/ExtendedUIDOperatorVerticle.java | 4 ---- src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 2c2c275ff..a8a4cbdcf 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -118,7 +118,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static long OPT_OUT_CHECK_CUTOFF_DATE = Instant.parse("2023-09-01T00:00:00.00Z").getEpochSecond(); private final Handler saltRetrievalResponseHandler; private final int allowClockSkewSeconds; - protected int maxSharingLifetimeSeconds; protected Map> siteIdToInvalidOriginsAndAppNames = new HashMap<>(); protected boolean keySharingEndpointProvideAppNames; protected Instant lastInvalidOriginProcessTime = Instant.now(); diff --git a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java index cfb90630b..379d39719 100644 --- a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java +++ b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java @@ -40,10 +40,6 @@ public void setKeySharingEndpointProvideAppNames(boolean enable) { this.keySharingEndpointProvideAppNames = enable; } - public void setMaxSharingLifetimeSeconds(int maxSharingLifetimeSeconds) { - this.maxSharingLifetimeSeconds = maxSharingLifetimeSeconds; - } - public void setLastInvalidOriginProcessTime(Instant lastInvalidOriginProcessTime) { this.lastInvalidOriginProcessTime = lastInvalidOriginProcessTime; } diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 8a2888d61..830b7521b 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -4745,7 +4745,7 @@ void keyDownloadEndpointKeysets_IDREADER(boolean provideAppNames, KeyDownloadEnd @Test void keySharingKeysets_SHARER_CustomMaxSharingLifetimeSeconds(Vertx vertx, VertxTestContext testContext) { - this.uidOperatorVerticle.setMaxSharingLifetimeSeconds(999999); + this.config.put(Const.Config.MaxSharingLifetimeProp, 999999); keySharingKeysets_SHARER(true, true, vertx, testContext, 999999); } From 8018196a3e5f2e3432105fb7432eb8ed740ed25d Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 18 Dec 2024 15:47:15 +1100 Subject: [PATCH 09/29] Added ConfigValidatorUtilTest, Removed unused imports in UIDOperatorVerticleTest --- .../operator/UIDOperatorVerticleTest.java | 3 -- .../service/ConfigValidatorUtilTest.java | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index 830b7521b..bfcaa4451 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -13,7 +13,6 @@ import com.uid2.operator.util.Tuple; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.vertx.UIDOperatorVerticle; -import com.uid2.operator.vertx.ClientInputValidationException; import com.uid2.shared.Utils; import com.uid2.shared.auth.ClientKey; import com.uid2.shared.auth.Keyset; @@ -27,9 +26,7 @@ import com.uid2.shared.secret.KeyHasher; import com.uid2.shared.store.*; import com.uid2.shared.store.reader.RotatingKeysetProvider; -import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.search.MeterNotFoundException; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.vertx.core.AsyncResult; import io.vertx.core.Future; diff --git a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java new file mode 100644 index 000000000..9d71bb376 --- /dev/null +++ b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java @@ -0,0 +1,32 @@ +package com.uid2.operator.service; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class ConfigValidatorUtilTest { + @Test + void testValidateIdentityRefreshTokens() { + // identityExpiresAfter is greater than refreshExpiresAfter + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(10, 5, 3)); + + // refreshIdentityAfter is greater than identityExpiresAfter + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(5, 10, 6)); + + // refreshIdentityAfter is greater than refreshExpiresAfter + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(5, 10, 11)); + + // all conditions are valid + assertTrue(ConfigValidatorUtil.validateIdentityRefreshTokens(5, 10, 3)); + } + + @Test + void testValidateBidstreamLifetime() { + // maxBidstreamLifetimeSeconds is less than identityTokenExpiresAfterSeconds + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(5, 10)); + + // maxBidstreamLifetimeSeconds is greater than or equal to identityTokenExpiresAfterSeconds + assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 5)); + assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 10)); + } +} From cd23998333ec6c9c9168464025d9fc85567914d7 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 19 Dec 2024 15:34:56 +1100 Subject: [PATCH 10/29] Updated ConfigValidatorUtil to handle null values, Updated ConfigValidatorUtilTest to test handling null values --- .../operator/service/ConfigValidatorUtil.java | 13 +++++++++ .../service/ConfigValidatorUtilTest.java | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java index 4f1c0999c..c7643d707 100644 --- a/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java +++ b/src/main/java/com/uid2/operator/service/ConfigValidatorUtil.java @@ -7,9 +7,17 @@ public class ConfigValidatorUtil { private static final Logger logger = LoggerFactory.getLogger(ConfigValidatorUtil.class); + public static final String VALUES_ARE_NULL = "Required config values are null"; public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter, Integer refreshExpiresAfter, Integer refreshIdentityAfter) { boolean isValid = true; + + if (identityExpiresAfter == null || refreshExpiresAfter == null || refreshIdentityAfter == null) { + logger.error(VALUES_ARE_NULL); + return false; + } + + if (identityExpiresAfter > refreshExpiresAfter) { logger.error(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS + " must be >= " + IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); isValid = false; @@ -25,6 +33,11 @@ public static Boolean validateIdentityRefreshTokens(Integer identityExpiresAfter } public static Boolean validateBidstreamLifetime(Integer maxBidstreamLifetimeSeconds, Integer identityTokenExpiresAfterSeconds) { + if (maxBidstreamLifetimeSeconds == null || identityTokenExpiresAfterSeconds == null) { + logger.error(VALUES_ARE_NULL); + return false; + } + if (maxBidstreamLifetimeSeconds < identityTokenExpiresAfterSeconds) { logger.error("Max bidstream lifetime seconds ({} seconds) is less than identity token lifetime ({} seconds)", maxBidstreamLifetimeSeconds, identityTokenExpiresAfterSeconds); return false; diff --git a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java index 9d71bb376..dde9b6080 100644 --- a/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java +++ b/src/test/java/com/uid2/operator/service/ConfigValidatorUtilTest.java @@ -29,4 +29,31 @@ void testValidateBidstreamLifetime() { assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 5)); assertTrue(ConfigValidatorUtil.validateBidstreamLifetime(10, 10)); } + + @Test + void testValidateIdentityRefreshTokensWithNullValues() { + // identityExpiresAfter is null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(null, 10, 5)); + + // refreshExpiresAfter is null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(10, null, 5)); + + // refreshIdentityAfter is null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(10, 5, null)); + + // all values are null + assertFalse(ConfigValidatorUtil.validateIdentityRefreshTokens(null, null, null)); + } + + @Test + void testValidateBidstreamLifetimeWithNullValues() { + // maxBidstreamLifetimeSeconds is null + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(null, 10)); + + // identityTokenExpiresAfterSeconds is null + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(10, null)); + + // both values are null + assertFalse(ConfigValidatorUtil.validateBidstreamLifetime(null, null)); + } } From bde83a5362e2eaee5dc1633c57bc0ff4d84be77f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 19 Dec 2024 15:52:35 +1100 Subject: [PATCH 11/29] Fixed httpStore in ConfigService --- conf/default-config.json | 2 +- src/main/java/com/uid2/operator/Const.java | 2 +- .../java/com/uid2/operator/service/ConfigService.java | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/conf/default-config.json b/conf/default-config.json index 849501309..d160858de 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -36,6 +36,6 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", - "core_config_url": "http://localhost:8088/config", + "core_config_path": "/config", "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 084e8f3bb..2b70c1cb0 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,7 +30,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigUrl = "core_config_url"; //TODO: update when endpoint name finalised + public static final String CoreConfigPath = "core_config_path"; //TODO: update when endpoint name finalised public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 4f95b8696..44de433d3 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -44,13 +44,15 @@ public JsonObject getConfig() { } private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { - String configUrl = bootstrapConfig.getString(CoreConfigUrl); + String configPath = bootstrapConfig.getString(CoreConfigPath); + ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") .setConfig(new JsonObject() - .put("url", configUrl) - .put("method", "GET")); + .put("host", "127.0.0.1") + .put("port", Const.Port.ServicePortForCore) + .put("path", configPath)); ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() .setType("json") @@ -70,6 +72,7 @@ private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) System.out.println("Successfully loaded config"); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); + logger.error("Failed to load config"); } }); From 8478fd208d9668137f35c6e691f8b3cc5566f1e5 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Fri, 20 Dec 2024 10:31:27 +1100 Subject: [PATCH 12/29] Added ConfigServiceTest --- .../com/uid2/operator/ConfigServiceTest.java | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 src/test/java/com/uid2/operator/ConfigServiceTest.java diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java new file mode 100644 index 000000000..e41954ee5 --- /dev/null +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -0,0 +1,161 @@ +package com.uid2.operator; + +import com.uid2.operator.service.ConfigService; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import io.vertx.config.ConfigRetriever; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.handler.BodyHandler; +import org.junit.jupiter.api.*; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.lang.reflect.*; +import java.util.concurrent.TimeUnit; + +import static com.uid2.operator.Const.Config.*; +import static com.uid2.operator.service.UIDOperatorService.*; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +@ExtendWith(VertxExtension.class) +class ConfigServiceTest { + private Vertx vertx; + private JsonObject bootstrapConfig; + + @BeforeEach + void setUp() { + vertx = Vertx.vertx(); + bootstrapConfig = new JsonObject() + .put(CoreConfigPath, "/config") + .put(ConfigScanPeriodMs, 300000) + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) + .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) + .put(MaxBidstreamLifetimeSecondsProp, 7200); + } + + @AfterEach + void tearDown() { + vertx.close(); + } + + void startMockServer(VertxTestContext testContext, JsonObject config) throws InterruptedException { + Router router = Router.router(vertx); + router.route().handler(BodyHandler.create()); + router.get("/config").handler(ctx -> ctx.response() + .putHeader("content-type", "application/json") + .end(config.encode())); + + vertx.createHttpServer() + .requestHandler(router) + .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { + if (!http.succeeded()) { + testContext.failNow(http.cause()); + } + }); + + testContext.awaitCompletion(5, TimeUnit.SECONDS); + } + + @Test + void testSingletonBehavior() { + ConfigService instance1 = ConfigService.getInstance(vertx, bootstrapConfig); + ConfigService instance2 = ConfigService.getInstance(vertx, bootstrapConfig); + assertSame(instance1, instance2, "getInstance should return the same instance"); + } + + @Test + void testGetConfig() { + ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + JsonObject cachedConfig = new JsonObject().put("key", "value"); + when(mockConfigRetriever.getCachedConfig()).thenReturn(cachedConfig); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + // Reflection to inject mocked ConfigRetriever + try { + Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); + configRetrieverField.setAccessible(true); + configRetrieverField.set(configService, mockConfigRetriever); + } catch (Exception e) { + fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); + } + JsonObject result = configService.getConfig(); + assertEquals(cachedConfig, result, "getConfig should return the cached configuration"); + } + + @Test + void testInitialiseConfigRetriever(VertxTestContext testContext) throws InterruptedException { + JsonObject httpStoreConfig = new JsonObject().put("http", "value"); + this.startMockServer(testContext, httpStoreConfig); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + // Wait for the initialisation to finish - alternative is to have getInstance return a future + testContext.awaitCompletion(1, TimeUnit.SECONDS); + JsonObject retrievedConfig = configService.getConfig(); + assertNotNull(retrievedConfig, "Config retriever should initialise without error"); + assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); + assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + testContext.completeNow(); + } + + @Test + void testInvalidConfigRevertsToPrevious() { + ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + JsonObject lastConfig = new JsonObject().put("previous", "config"); + JsonObject invalidConfig = new JsonObject() + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); + when(mockConfigRetriever.getCachedConfig()).thenReturn(lastConfig); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + try { + Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); + configRetrieverField.setAccessible(true); + configRetrieverField.set(configService, mockConfigRetriever); + } catch (Exception e) { + fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); + } + + try { + Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); + configValidationHandlerMethod.setAccessible(true); + JsonObject validatedConfig = (JsonObject) configValidationHandlerMethod.invoke(configService, invalidConfig); + assertEquals(lastConfig, validatedConfig, "Invalid config not reverted to previous config"); + } catch (Exception e) { + fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); + } + } + + @Test + void testFirstInvalidConfigThrowsRuntimeException() { + ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + JsonObject invalidConfig = new JsonObject() + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); + when(mockConfigRetriever.getCachedConfig()).thenReturn(null); + ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); + try { + Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); + configRetrieverField.setAccessible(true); + configRetrieverField.set(configService, mockConfigRetriever); + } catch (Exception e) { + fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); + } + + + + try { + Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); + configValidationHandlerMethod.setAccessible(true); + assertThrows(RuntimeException.class, () -> { + try { + configValidationHandlerMethod.invoke(configService, invalidConfig); + } catch (InvocationTargetException e) { + // Throw cause as InvocationTargetException wraps actual exception thrown by configValidationHandler + throw e.getCause(); + } + }); + } catch (Exception e) { + fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); + } + } +} \ No newline at end of file From 8aee5a37ba79698199bfd73b8c8e940962ab28ea Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 31 Dec 2024 16:30:29 +1100 Subject: [PATCH 13/29] Added ConfigRetrieverFactory to create vert.x ConfigRetriever --- .../service/ConfigRetrieverFactory.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java new file mode 100644 index 000000000..5b18cf3b1 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -0,0 +1,36 @@ +package com.uid2.operator.service; + +import com.uid2.operator.Const; +import io.vertx.config.ConfigRetriever; +import io.vertx.config.ConfigRetrieverOptions; +import io.vertx.config.ConfigStoreOptions; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; + +import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; +import static com.uid2.operator.Const.Config.CoreConfigPath; + +public class ConfigRetrieverFactory { + public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { + String configPath = bootstrapConfig.getString(CoreConfigPath); + + + ConfigStoreOptions httpStore = new ConfigStoreOptions() + .setType("http") + .setConfig(new JsonObject() + .put("host", "127.0.0.1") + .put("port", Const.Port.ServicePortForCore) + .put("path", configPath)); + + ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() + .setType("json") + .setConfig(bootstrapConfig); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) + .addStore(bootstrapStore) + .addStore(httpStore); + + return ConfigRetriever.create(vertx, retrieverOptions); + } +} From 8f40ba2a9b2e88733a2d2e92d5a6c75ccba37646 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 31 Dec 2024 16:33:29 +1100 Subject: [PATCH 14/29] Updated ConfigService, Main ConfigService returns a Future when created and takes ConfigRetriever in "create" method "run()" method in Main waits for ConfigService Future to complete before creating UIDOperatorVerticle --- src/main/java/com/uid2/operator/Main.java | 71 ++++++++++--------- .../uid2/operator/service/ConfigService.java | 69 +++++------------- 2 files changed, 57 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 7dbf64153..589f64bd3 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -8,9 +8,7 @@ import com.uid2.operator.monitoring.IStatsCollectorQueue; import com.uid2.operator.monitoring.OperatorMetrics; import com.uid2.operator.monitoring.StatsCollectorVerticle; -import com.uid2.operator.service.ConfigService; -import com.uid2.operator.service.SecureLinkValidatorService; -import com.uid2.operator.service.ShutdownService; +import com.uid2.operator.service.*; import com.uid2.operator.vertx.Endpoints; import com.uid2.operator.vertx.OperatorShutdownHandler; import com.uid2.operator.store.CloudSyncOptOutStore; @@ -38,6 +36,7 @@ import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.micrometer.prometheus.PrometheusRenameFilter; +import io.vertx.config.ConfigRetriever; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; import io.vertx.core.http.impl.HttpUtils; @@ -266,41 +265,49 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } private void run() throws Exception { - ConfigService configService = ConfigService.getInstance(vertx, config); + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, config); - Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); - return verticle; - }; + ConfigService.create(configRetriever).compose(configService -> { - DeploymentOptions options = new DeploymentOptions(); - int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); - options.setInstances(svcInstances); + Supplier operatorVerticleSupplier = () -> { + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; + }; - Promise compositePromise = Promise.promise(); - List fs = new ArrayList<>(); - fs.add(createAndDeployStatsCollector()); - fs.add(createStoreVerticles()); + DeploymentOptions options = new DeploymentOptions(); + int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); + options.setInstances(svcInstances); - CompositeFuture.all(fs).onComplete(ar -> { - if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); - else compositePromise.complete(); - }); + Promise compositePromise = Promise.promise(); + List fs = new ArrayList<>(); + fs.add(createAndDeployStatsCollector()); + try { + fs.add(createStoreVerticles()); + } catch (Exception e) { + throw new RuntimeException(e); + } - compositePromise.future() - .compose(v -> { - metrics.setup(); - vertx.setPeriodic(60000, id -> metrics.update()); - - Promise promise = Promise.promise(); - vertx.deployVerticle(operatorVerticleSupplier, options, promise); - return promise.future(); - }) - .onFailure(t -> { - LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); - vertx.close(); - System.exit(1); + CompositeFuture.all(fs).onComplete(ar -> { + if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); + else compositePromise.complete(); }); + + return compositePromise.future() + .compose(v -> { + metrics.setup(); + vertx.setPeriodic(60000, id -> metrics.update()); + + Promise promise = Promise.promise(); + vertx.deployVerticle(operatorVerticleSupplier, options, promise); + return promise.future(); + }) + .onFailure(t -> { + LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); + vertx.close(); + System.exit(1); + }); + }); } private Future createStoreVerticles() throws Exception { diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index 44de433d3..c58e59fc7 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -2,80 +2,47 @@ import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; -import io.vertx.config.ConfigRetrieverOptions; -import io.vertx.config.ConfigStoreOptions; -import io.vertx.core.Vertx; +import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.ConfigValidatorUtil.*; import static com.uid2.operator.service.UIDOperatorService.*; public class ConfigService implements IConfigService { - private static volatile ConfigService instance; private ConfigRetriever configRetriever; private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); - private ConfigService(Vertx vertx, JsonObject bootstrapConfig) { - this.initialiseConfigRetriever(vertx, bootstrapConfig); - } - - public static ConfigService getInstance(Vertx vertx, JsonObject bootstrapConfig) { - ConfigService configService = instance; - - if (configService == null) { - synchronized (ConfigService.class) { - configService = instance; - if (configService == null) { - instance = configService = new ConfigService(vertx, bootstrapConfig); - } - } - } - - return configService; - } - - @Override - public JsonObject getConfig() { - return configRetriever.getCachedConfig(); + private ConfigService(ConfigRetriever configRetriever) { + this.configRetriever = configRetriever; + this.configRetriever.setConfigurationProcessor(this::configValidationHandler); } - private void initialiseConfigRetriever(Vertx vertx, JsonObject bootstrapConfig) { - String configPath = bootstrapConfig.getString(CoreConfigPath); + public static Future create(ConfigRetriever configRetriever) { + Promise promise = Promise.promise(); + ConfigService instance = new ConfigService(configRetriever); - ConfigStoreOptions httpStore = new ConfigStoreOptions() - .setType("http") - .setConfig(new JsonObject() - .put("host", "127.0.0.1") - .put("port", Const.Port.ServicePortForCore) - .put("path", configPath)); - - ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() - .setType("json") - .setConfig(bootstrapConfig); - - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() - .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) - .addStore(bootstrapStore) - .addStore(httpStore); - - this.configRetriever = ConfigRetriever.create(vertx, retrieverOptions); - - this.configRetriever.setConfigurationProcessor(this::configValidationHandler); - - this.configRetriever.getConfig(ar -> { + configRetriever.getConfig(ar -> { if (ar.succeeded()) { System.out.println("Successfully loaded config"); + promise.complete(instance); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); - logger.error("Failed to load config"); + logger.error("Failed to load config{}", ar.cause().getMessage()); + promise.fail(ar.cause()); } }); + return promise.future(); + } + + @Override + public JsonObject getConfig() { + return configRetriever.getCachedConfig(); } private JsonObject configValidationHandler(JsonObject config) { From c1fd6dfbc470f819a60984135fa2ef0e165c0953 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 13:32:36 +1100 Subject: [PATCH 15/29] Updated ConfigServiceTest Removed use of reflection to access private attributes/methods Removed unnecessary tests Updated remaining tests to inject ConfigRetriever into ConfigService --- .../com/uid2/operator/ConfigServiceTest.java | 144 +++++++----------- 1 file changed, 55 insertions(+), 89 deletions(-) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index e41954ee5..f00b2e900 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -1,7 +1,9 @@ package com.uid2.operator; +import com.uid2.operator.service.ConfigRetrieverFactory; import com.uid2.operator.service.ConfigService; import io.vertx.core.Vertx; +import io.vertx.core.http.HttpServer; import io.vertx.core.json.JsonObject; import io.vertx.config.ConfigRetriever; import io.vertx.ext.web.Router; @@ -11,9 +13,6 @@ import io.vertx.junit5.VertxTestContext; import org.junit.jupiter.api.extension.ExtendWith; -import java.lang.reflect.*; -import java.util.concurrent.TimeUnit; - import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.UIDOperatorService.*; import static org.junit.jupiter.api.Assertions.*; @@ -23,6 +22,8 @@ class ConfigServiceTest { private Vertx vertx; private JsonObject bootstrapConfig; + private ConfigRetriever configRetriever; + private HttpServer server; @BeforeEach void setUp() { @@ -34,6 +35,9 @@ void setUp() { .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) .put(MaxBidstreamLifetimeSecondsProp, 7200); + + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); } @AfterEach @@ -41,121 +45,83 @@ void tearDown() { vertx.close(); } - void startMockServer(VertxTestContext testContext, JsonObject config) throws InterruptedException { + void startMockServer(VertxTestContext testContext, JsonObject config) { + if (server != null) { + server.close(); + } + Router router = Router.router(vertx); router.route().handler(BodyHandler.create()); router.get("/config").handler(ctx -> ctx.response() .putHeader("content-type", "application/json") .end(config.encode())); - vertx.createHttpServer() + server = vertx.createHttpServer() .requestHandler(router) .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { if (!http.succeeded()) { testContext.failNow(http.cause()); } }); - - testContext.awaitCompletion(5, TimeUnit.SECONDS); - } - - @Test - void testSingletonBehavior() { - ConfigService instance1 = ConfigService.getInstance(vertx, bootstrapConfig); - ConfigService instance2 = ConfigService.getInstance(vertx, bootstrapConfig); - assertSame(instance1, instance2, "getInstance should return the same instance"); - } - - @Test - void testGetConfig() { - ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); - JsonObject cachedConfig = new JsonObject().put("key", "value"); - when(mockConfigRetriever.getCachedConfig()).thenReturn(cachedConfig); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - // Reflection to inject mocked ConfigRetriever - try { - Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); - configRetrieverField.setAccessible(true); - configRetrieverField.set(configService, mockConfigRetriever); - } catch (Exception e) { - fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); - } - JsonObject result = configService.getConfig(); - assertEquals(cachedConfig, result, "getConfig should return the cached configuration"); } @Test - void testInitialiseConfigRetriever(VertxTestContext testContext) throws InterruptedException { + void testGetConfig(VertxTestContext testContext) { JsonObject httpStoreConfig = new JsonObject().put("http", "value"); this.startMockServer(testContext, httpStoreConfig); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - // Wait for the initialisation to finish - alternative is to have getInstance return a future - testContext.awaitCompletion(1, TimeUnit.SECONDS); - JsonObject retrievedConfig = configService.getConfig(); - assertNotNull(retrievedConfig, "Config retriever should initialise without error"); - assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); - assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); - testContext.completeNow(); + ConfigService.create(configRetriever).onComplete(ar -> { + if (ar.succeeded()) { + ConfigService configService = ar.result(); + JsonObject retrievedConfig = configService.getConfig(); + assertNotNull(retrievedConfig, "Config retriever should initialise without error"); + assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); + assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + testContext.completeNow(); + } else { + testContext.failNow(ar.cause()); + } + }); + } @Test - void testInvalidConfigRevertsToPrevious() { - ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject lastConfig = new JsonObject().put("previous", "config"); + ConfigRetriever spyRetriever = spy(configRetriever); + when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - when(mockConfigRetriever.getCachedConfig()).thenReturn(lastConfig); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - try { - Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); - configRetrieverField.setAccessible(true); - configRetrieverField.set(configService, mockConfigRetriever); - } catch (Exception e) { - fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); - } - - try { - Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); - configValidationHandlerMethod.setAccessible(true); - JsonObject validatedConfig = (JsonObject) configValidationHandlerMethod.invoke(configService, invalidConfig); - assertEquals(lastConfig, validatedConfig, "Invalid config not reverted to previous config"); - } catch (Exception e) { - fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); - } + this.startMockServer(testContext, invalidConfig); + ConfigService.create(spyRetriever).onComplete(ar -> { + if (ar.succeeded()) { + reset(spyRetriever); + ConfigService configService = ar.result(); + assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); + testContext.completeNow(); + } + else { + testContext.failNow(ar.cause()); + } + }); } @Test - void testFirstInvalidConfigThrowsRuntimeException() { - ConfigRetriever mockConfigRetriever = mock(ConfigRetriever.class); + void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) { JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - when(mockConfigRetriever.getCachedConfig()).thenReturn(null); - ConfigService configService = ConfigService.getInstance(vertx, bootstrapConfig); - try { - Field configRetrieverField = ConfigService.class.getDeclaredField("configRetriever"); - configRetrieverField.setAccessible(true); - configRetrieverField.set(configService, mockConfigRetriever); - } catch (Exception e) { - fail("Failed to inject mock ConfigRetriever: " + e.getMessage()); - } - - - - try { - Method configValidationHandlerMethod = ConfigService.class.getDeclaredMethod("configValidationHandler", JsonObject.class); - configValidationHandlerMethod.setAccessible(true); - assertThrows(RuntimeException.class, () -> { - try { - configValidationHandlerMethod.invoke(configService, invalidConfig); - } catch (InvocationTargetException e) { - // Throw cause as InvocationTargetException wraps actual exception thrown by configValidationHandler - throw e.getCause(); - } - }); - } catch (Exception e) { - fail("Failed to access and invoke the configValidationHandler method: " + e.getMessage()); - } + this.startMockServer(testContext, invalidConfig); + ConfigService.create(configRetriever).onComplete(ar -> { + if (ar.succeeded()) { + testContext.failNow(new RuntimeException("Expected a RuntimeException but the creation succeeded")); + } + else { + assertThrows(RuntimeException.class, () -> { + throw ar.cause(); + }); + testContext.completeNow(); + } + }); } } \ No newline at end of file From ab3005459319c09253a7fcd1804187207f66f58c Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 14:13:40 +1100 Subject: [PATCH 16/29] Updated ConfigRetrieverFactory Set http store to optional --- .../java/com/uid2/operator/service/ConfigRetrieverFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 5b18cf3b1..a35b63416 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -17,6 +17,7 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") + .setOptional(true) .setConfig(new JsonObject() .put("host", "127.0.0.1") .put("port", Const.Port.ServicePortForCore) From 7d2a16dd979dad327003d74c4fe5f0621481276c Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 15:09:50 +1100 Subject: [PATCH 17/29] Updated ConfigService Added default value for max_bidstream_lifetime_seconds --- src/main/java/com/uid2/operator/service/ConfigService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigService.java b/src/main/java/com/uid2/operator/service/ConfigService.java index c58e59fc7..cd5b8b9bc 100644 --- a/src/main/java/com/uid2/operator/service/ConfigService.java +++ b/src/main/java/com/uid2/operator/service/ConfigService.java @@ -13,7 +13,7 @@ public class ConfigService implements IConfigService { - private ConfigRetriever configRetriever; + private final ConfigRetriever configRetriever; private static final Logger logger = LoggerFactory.getLogger(ConfigService.class); private ConfigService(ConfigRetriever configRetriever) { @@ -32,7 +32,7 @@ public static Future create(ConfigRetriever configRetriever) { promise.complete(instance); } else { System.err.println("Failed to load config: " + ar.cause().getMessage()); - logger.error("Failed to load config{}", ar.cause().getMessage()); + logger.error("Failed to load config: {}", ar.cause().getMessage()); promise.fail(ar.cause()); } }); @@ -50,7 +50,7 @@ private JsonObject configValidationHandler(JsonObject config) { Integer identityExpiresAfter = config.getInteger(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS); Integer refreshExpiresAfter = config.getInteger(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS); Integer refreshIdentityAfter = config.getInteger(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS); - Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp); + Integer maxBidstreamLifetimeSeconds = config.getInteger(Const.Config.MaxBidstreamLifetimeSecondsProp, identityExpiresAfter); isValid &= validateIdentityRefreshTokens(identityExpiresAfter, refreshExpiresAfter, refreshIdentityAfter); From 67b05b182e718217816774ca4a87c3af00442e03 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 16:34:12 +1100 Subject: [PATCH 18/29] Added ConfigServiceManager, StaticConfigService ConfigServiceManager handles switching between dynamic and static config StaticConfigService implements IConfigService, serving config from static files --- .../service/ConfigServiceManager.java | 58 +++++++++++++++++++ .../operator/service/StaticConfigService.java | 17 ++++++ 2 files changed, 75 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/ConfigServiceManager.java create mode 100644 src/main/java/com/uid2/operator/service/StaticConfigService.java diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java new file mode 100644 index 000000000..fc69c79fc --- /dev/null +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -0,0 +1,58 @@ +package com.uid2.operator.service; + +import io.vertx.config.ConfigRetriever; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConfigServiceManager { + private IConfigService currentConfigService; + private final ConfigService dynamicConfigService; + private final StaticConfigService staticConfigService; + private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); + + private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigService staticConfigService, boolean useDynamicConfig) { + this.dynamicConfigService = dynamicConfigService; + this.staticConfigService = staticConfigService; + this.currentConfigService = useDynamicConfig ? dynamicConfigService : staticConfigService; + } + + public Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { + Promise promise = Promise.promise(); + + StaticConfigService staticConfigService = new StaticConfigService(bootstrapConfig); + + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); + + ConfigService.create(configRetriever).onComplete(ar -> { + if (ar.succeeded()) { + ConfigService dynamicConfigService = ar.result(); + ConfigServiceManager instance = new ConfigServiceManager(dynamicConfigService, staticConfigService, useDynamicConfig); + promise.complete(instance); + } + else { + promise.fail(ar.cause()); + } + }); + + return promise.future(); + } + + public void updateConfigService(boolean useDynamicConfig) { + if (useDynamicConfig) { + logger.info("Switching to DynamicConfigService"); + this.currentConfigService = dynamicConfigService; + } else { + logger.info("Switching to StaticConfigService"); + this.currentConfigService = staticConfigService; + } + } + + public IConfigService getConfigService() { + return currentConfigService; + } +} diff --git a/src/main/java/com/uid2/operator/service/StaticConfigService.java b/src/main/java/com/uid2/operator/service/StaticConfigService.java new file mode 100644 index 000000000..e44440fc9 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/StaticConfigService.java @@ -0,0 +1,17 @@ +package com.uid2.operator.service; + +import io.vertx.core.json.JsonObject; + + +public class StaticConfigService implements IConfigService { + private final JsonObject staticConfig; + + public StaticConfigService(JsonObject staticConfig) { + this.staticConfig = staticConfig; + } + + @Override + public JsonObject getConfig() { + return staticConfig; + } +} From 25ad265fa9035b035e426e2e3cfca91a32217898 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 2 Jan 2025 17:40:48 +1100 Subject: [PATCH 19/29] Update UIDOperatorVerticle, Main Main creates new ConfigServiceManager instance and injects it into UIDOperatorVerticle Also Added feature flag config value to local-config.json, Made create method static in ConfigServiceManager --- conf/local-config.json | 3 ++- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 7 ++----- .../com/uid2/operator/service/ConfigServiceManager.java | 2 +- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 6 ++++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conf/local-config.json b/conf/local-config.json index 6a357dba1..c73940613 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -36,5 +36,6 @@ "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12, - "operator_type": "public" + "operator_type": "public", + "remote_config_feature_flag": true } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 2b70c1cb0..b1c7175b0 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -34,5 +34,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; + public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 85fcdef9c..5ef898c85 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -268,12 +268,9 @@ private void run() throws Exception { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, config); - - ConfigService.create(configRetriever).compose(configService -> { + ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configServiceManager, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java index fc69c79fc..e21fca62f 100644 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -20,7 +20,7 @@ private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigSer this.currentConfigService = useDynamicConfig ? dynamicConfigService : staticConfigService; } - public Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { + public static Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { Promise promise = Promise.promise(); StaticConfigService staticConfigService = new StaticConfigService(bootstrapConfig); diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f1512e10..f3bd23937 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -96,6 +96,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; + private final ConfigServiceManager configServiceManager; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); @@ -135,7 +136,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(IConfigService configService, + public UIDOperatorVerticle(ConfigServiceManager configServiceManager, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -154,7 +155,8 @@ public UIDOperatorVerticle(IConfigService configService, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } - this.configService = configService; + this.configServiceManager = configServiceManager; + this.configService = configServiceManager.getConfigService(); this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); From 6bd4a350776194b2c233ac01f72ea9eabdc83d2e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 6 Jan 2025 11:02:28 +1100 Subject: [PATCH 20/29] Update UIDOperatorVerticle Removed ConfigServiceManager from constructor --- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index f3bd23937..4f1512e10 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -96,7 +96,6 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final IOptOutStore optOutStore; private final IClientKeyProvider clientKeyProvider; private final Clock clock; - private final ConfigServiceManager configServiceManager; protected IUIDOperatorService idService; private final Map _identityMapMetricSummaries = new HashMap<>(); private final Map, DistributionSummary> _refreshDurationMetricSummaries = new HashMap<>(); @@ -136,7 +135,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { private static final String ERROR_INVALID_INPUT_EMAIL_TWICE = "Only one of email or email_hash can be specified"; public final static String ORIGIN_HEADER = "Origin"; - public UIDOperatorVerticle(ConfigServiceManager configServiceManager, + public UIDOperatorVerticle(IConfigService configService, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -155,8 +154,7 @@ public UIDOperatorVerticle(ConfigServiceManager configServiceManager, } catch (NoSuchAlgorithmException | NoSuchPaddingException e) { throw new RuntimeException(e); } - this.configServiceManager = configServiceManager; - this.configService = configServiceManager.getConfigService(); + this.configService = configService; this.clientSideTokenGenerate = clientSideTokenGenerate; this.healthComponent.setHealthStatus(false, "not started"); this.auth = new AuthMiddleware(clientKeyProvider); From 80b6b199053451fcb5790d400d5f3530629aa49f Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 6 Jan 2025 11:03:52 +1100 Subject: [PATCH 21/29] Add DelegatingConfigService Allows for switches between static and dynamic config services without having to change UIDOperatorVerticle --- .../service/DelegatingConfigService.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/main/java/com/uid2/operator/service/DelegatingConfigService.java diff --git a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java new file mode 100644 index 000000000..36fd77da2 --- /dev/null +++ b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java @@ -0,0 +1,22 @@ +package com.uid2.operator.service; + +import io.vertx.core.json.JsonObject; + +import java.util.concurrent.atomic.AtomicReference; + +public class DelegatingConfigService implements IConfigService{ + private AtomicReference activeConfigService; + + public DelegatingConfigService(IConfigService initialConfigService) { + this.activeConfigService = new AtomicReference<>(initialConfigService); + } + + public void updateConfigService(IConfigService newConfigService) { + this.activeConfigService = new AtomicReference<>(newConfigService); + } + + @Override + public JsonObject getConfig() { + return activeConfigService.get().getConfig(); + } +} From 7ef6af6e65923df6404dcf6d3aa0750f520c16bb Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Mon, 6 Jan 2025 11:07:59 +1100 Subject: [PATCH 22/29] Update ConfigServiceManager, Main ConfigRetriever in Main listens for feature flag changes and publishes to eventbus ConfigServiceManager listens to eventbus and updates DelegatingConfigService active ConfigService --- src/main/java/com/uid2/operator/Const.java | 1 + src/main/java/com/uid2/operator/Main.java | 23 +++++++++++++------ .../service/ConfigServiceManager.java | 23 ++++++++++++++----- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index b1c7175b0..271b36170 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -35,5 +35,6 @@ public class Config extends com.uid2.shared.Const.Config { public static final String Config = "config"; public static final String identityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; + public static final String RemoteConfigFlagEventBus = "RemoteConfigFlag"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 5ef898c85..1eb7f3763 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -203,7 +203,16 @@ else if (!Utils.isProductionEnvironment()) { } Vertx vertx = createVertx(); - VertxUtils.createConfigRetriever(vertx).getConfig(ar -> { + ConfigRetriever configRetriever = VertxUtils.createConfigRetriever(vertx); + + configRetriever.listen(configChange -> { + JsonObject newConfig = configChange.getNewConfiguration(); + boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + + vertx.eventBus().publish(Const.Config.RemoteConfigFlagEventBus, useDynamicConfig); + }); + + configRetriever.getConfig(ar -> { if (ar.failed()) { LOGGER.error("Unable to read config: " + ar.cause().getMessage(), ar.cause()); return; @@ -264,15 +273,16 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private void run() throws Exception { + private void run() { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { - Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configServiceManager, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); - return verticle; - }; + IConfigService configService = configServiceManager.getDelegatingConfigService(); + Supplier operatorVerticleSupplier = () -> { + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; + }; DeploymentOptions options = new DeploymentOptions(); int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); @@ -296,7 +306,6 @@ private void run() throws Exception { .compose(v -> { metrics.setup(); vertx.setPeriodic(60000, id -> metrics.update()); - Promise promise = Promise.promise(); vertx.deployVerticle(operatorVerticleSupplier, options, promise); return promise.future(); diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java index e21fca62f..1002fbecf 100644 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -1,5 +1,6 @@ package com.uid2.operator.service; +import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -9,7 +10,7 @@ import org.slf4j.LoggerFactory; public class ConfigServiceManager { - private IConfigService currentConfigService; + private final DelegatingConfigService delegatingConfigService; private final ConfigService dynamicConfigService; private final StaticConfigService staticConfigService; private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); @@ -17,7 +18,7 @@ public class ConfigServiceManager { private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigService staticConfigService, boolean useDynamicConfig) { this.dynamicConfigService = dynamicConfigService; this.staticConfigService = staticConfigService; - this.currentConfigService = useDynamicConfig ? dynamicConfigService : staticConfigService; + this.delegatingConfigService = new DelegatingConfigService(useDynamicConfig ? dynamicConfigService : staticConfigService); } public static Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { @@ -32,6 +33,7 @@ public static Future create(Vertx vertx, JsonObject bootst if (ar.succeeded()) { ConfigService dynamicConfigService = ar.result(); ConfigServiceManager instance = new ConfigServiceManager(dynamicConfigService, staticConfigService, useDynamicConfig); + instance.initialiseListener(vertx); promise.complete(instance); } else { @@ -42,17 +44,26 @@ public static Future create(Vertx vertx, JsonObject bootst return promise.future(); } + private void initialiseListener(Vertx vertx) { + vertx.eventBus().consumer(Const.Config.RemoteConfigFlagEventBus, message -> { + boolean useDynamicConfig = Boolean.parseBoolean(message.body().toString()); + + this.updateConfigService(useDynamicConfig); + }); + } + public void updateConfigService(boolean useDynamicConfig) { if (useDynamicConfig) { logger.info("Switching to DynamicConfigService"); - this.currentConfigService = dynamicConfigService; + this.delegatingConfigService.updateConfigService(dynamicConfigService); } else { logger.info("Switching to StaticConfigService"); - this.currentConfigService = staticConfigService; + this.delegatingConfigService.updateConfigService(staticConfigService); } } - public IConfigService getConfigService() { - return currentConfigService; + public IConfigService getDelegatingConfigService() { + return delegatingConfigService; } + } From ed3141a7ae2a218bd63e9c49f98c94e8b00a0b9c Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 8 Jan 2025 11:16:16 +1100 Subject: [PATCH 23/29] Update UIDOperatorVerticleTest Added tests to verify config changes are reflected in endpoints --- .../operator/UIDOperatorVerticleTest.java | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index bcb9bfdb3..de105ab48 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -5090,4 +5090,101 @@ void secureLinkValidationFailsReturnsIdentityError(Vertx vertx, VertxTestContext testContext.completeNow(); }); } + + @Test + void tokenGenerateRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + final int clientSiteId = 201; + final String emailAddress = "test@uid2.com"; + fakeAuth(clientSiteId, Role.GENERATOR); + setupSalts(); + setupKeys(); + + String v1Param = "email=" + emailAddress; + JsonObject v2Payload = new JsonObject(); + v2Payload.put("email", emailAddress); + + Duration newIdentityExpiresAfter = Duration.ofMinutes(20); + Duration newRefreshExpiresAfter = Duration.ofMinutes(30); + Duration newRefreshIdentityAfter = Duration.ofMinutes(10); + + config.put(UIDOperatorService.IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, newIdentityExpiresAfter.toMillis() / 1000); + config.put(UIDOperatorService.REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, newRefreshExpiresAfter.toMillis() / 1000); + config.put(UIDOperatorService.REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, newRefreshIdentityAfter.toMillis() / 1000); + when(configService.getConfig()).thenReturn(config); + + try { + sendTokenGenerate("v2", vertx, + v1Param, v2Payload, 200, + respJson -> { + JsonObject body = respJson.getJsonObject("body"); + testContext.verify(() -> { + assertNotNull(body); + assertEquals(now.plusMillis(newIdentityExpiresAfter.toMillis()).toEpochMilli(), body.getLong("identity_expires")); + assertEquals(now.plusMillis(newRefreshExpiresAfter.toMillis()).toEpochMilli(), body.getLong("refresh_expires")); + assertEquals(now.plusMillis(newRefreshIdentityAfter.toMillis()).toEpochMilli(), body.getLong("refresh_from")); + }); + testContext.completeNow(); + }); + } catch (Exception e) { + testContext.failNow(e); + } + } + + @Test + void keySharingRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + int newSharingTokenExpiry = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + int newMaxSharingLifetimeSeconds = config.getInteger(Const.Config.SharingTokenExpiryProp) + 1; + + config.put(Const.Config.SharingTokenExpiryProp, newSharingTokenExpiry); + config.put(Const.Config.MaxSharingLifetimeProp, newMaxSharingLifetimeSeconds); + + String apiVersion = "v2"; + int siteId = 5; + fakeAuth(siteId, Role.SHARER); + Keyset[] keysets = { + new Keyset(MasterKeysetId, MasterKeySiteId, "test", null, now.getEpochSecond(), true, true), + new Keyset(10, 5, "siteKeyset", null, now.getEpochSecond(), true, true), + }; + KeysetKey[] encryptionKeys = { + new KeysetKey(101, "master key".getBytes(), now, now, now.plusSeconds(10), MasterKeysetId), + new KeysetKey(102, "site key".getBytes(), now, now, now.plusSeconds(10), 10), + }; + MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); + setupSiteDomainAndAppNameMock(true, false, 101, 102, 103, 104, 105); + send(apiVersion, vertx, apiVersion + "/key/sharing", true, null, null, 200, respJson -> { + + JsonObject body = respJson.getJsonObject("body"); + testContext.verify(() -> { + assertNotNull(body); + assertEquals(newSharingTokenExpiry, Integer.parseInt(body.getString("token_expiry_seconds"))); + assertEquals(newMaxSharingLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxSharingLifetimeProp)); + }); + testContext.completeNow(); + }); + } + + @Test + void keyBidstreamRespectsConfigValues(Vertx vertx, VertxTestContext testContext) { + int newMaxBidstreamLifetimeSeconds = 999999; + config.put(Const.Config.MaxBidstreamLifetimeSecondsProp, newMaxBidstreamLifetimeSeconds); + + final String apiVersion = "v2"; + final KeyDownloadEndpoint endpoint = KeyDownloadEndpoint.BIDSTREAM; + + final int clientSiteId = 101; + fakeAuth(clientSiteId, Role.ID_READER); + + // Required, sets up mock keys. + new MultipleKeysetsTests(); + + send(apiVersion, vertx, apiVersion + endpoint.getPath(), true, null, null, 200, respJson -> { + JsonObject body = respJson.getJsonObject("body"); + testContext.verify(() -> { + assertNotNull(body); + assertEquals(newMaxBidstreamLifetimeSeconds + TOKEN_LIFETIME_TOLERANCE.toSeconds(), body.getLong(Const.Config.MaxBidstreamLifetimeSecondsProp)); + }); + testContext.completeNow(); + }); + } + } From 5cde2b27a36d4a5879bc9acc91621c381e74feda Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 8 Jan 2025 11:47:39 +1100 Subject: [PATCH 24/29] Add static config to UIDOperatorVerticle constructor static config values are set from main config rather than ConfigService --- src/main/java/com/uid2/operator/Main.java | 2 +- .../java/com/uid2/operator/vertx/UIDOperatorVerticle.java | 2 +- .../java/com/uid2/operator/ExtendedUIDOperatorVerticle.java | 4 +++- src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 1eb7f3763..a0546b25a 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -280,7 +280,7 @@ private void run() { ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { IConfigService configService = configServiceManager.getDelegatingConfigService(); Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); return verticle; }; diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index 4f1512e10..c0e6af231 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -136,6 +136,7 @@ public class UIDOperatorVerticle extends AbstractVerticle { public final static String ORIGIN_HEADER = "Origin"; public UIDOperatorVerticle(IConfigService configService, + JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -164,7 +165,6 @@ public UIDOperatorVerticle(IConfigService configService, this.saltProvider = saltProvider; this.optOutStore = optOutStore; this.clock = clock; - JsonObject config = configService.getConfig(); this.identityScope = IdentityScope.fromString(config.getString("identity_scope", "uid2")); this.v2PayloadHandler = new V2PayloadHandler(keyManager, config.getBoolean("enable_v2_encryption", true), this.identityScope, siteProvider); this.phoneSupport = config.getBoolean("enable_phone_support", true); diff --git a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java index 379d39719..24353eaf1 100644 --- a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java +++ b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java @@ -9,6 +9,7 @@ import com.uid2.operator.vertx.UIDOperatorVerticle; import com.uid2.shared.store.*; import io.vertx.core.Handler; +import io.vertx.core.json.JsonObject; import java.time.Clock; import java.time.Instant; @@ -18,6 +19,7 @@ //An extended UIDOperatorVerticle to expose classes for testing purposes public class ExtendedUIDOperatorVerticle extends UIDOperatorVerticle { public ExtendedUIDOperatorVerticle(IConfigService configService, + JsonObject config, boolean clientSideTokenGenerate, ISiteStore siteProvider, IClientKeyProvider clientKeyProvider, @@ -29,7 +31,7 @@ public ExtendedUIDOperatorVerticle(IConfigService configService, IStatsCollectorQueue statsCollectorQueue, SecureLinkValidatorService secureLinkValidationService, Handler saltRetrievalResponseHandler) { - super(configService, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); + super(configService, config, clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, keyManager, saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidationService, saltRetrievalResponseHandler); } public IUIDOperatorService getIdService() { diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index de105ab48..2af239977 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -134,7 +134,7 @@ public void deployVerticle(Vertx vertx, VertxTestContext testContext, TestInfo t } when(configService.getConfig()).thenReturn(config); - this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); + this.uidOperatorVerticle = new ExtendedUIDOperatorVerticle(configService, config, config.getBoolean("client_side_token_generate"), siteProvider, clientKeyProvider, clientSideKeypairProvider, new KeyManager(keysetKeyStore, keysetProvider), saltProvider, optOutStore, clock, statsCollectorQueue, secureLinkValidatorService, shutdownHandler::handleSaltRetrievalResponse); vertx.deployVerticle(uidOperatorVerticle, testContext.succeeding(id -> testContext.completeNow())); From 91ecc1d313047ab5b7dce0bcf50b596d0edc190e Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Thu, 9 Jan 2025 11:43:59 +1100 Subject: [PATCH 25/29] Update ConfigServiceTest to combine futures using compose - removed bootstrap config as a store from ConfigRetrieverFactory --- .../service/ConfigRetrieverFactory.java | 5 -- .../com/uid2/operator/ConfigServiceTest.java | 84 +++++++++---------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index a35b63416..215b58482 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -23,13 +23,8 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { .put("port", Const.Port.ServicePortForCore) .put("path", configPath)); - ConfigStoreOptions bootstrapStore = new ConfigStoreOptions() - .setType("json") - .setConfig(bootstrapConfig); - ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) - .addStore(bootstrapStore) .addStore(httpStore); return ConfigRetriever.create(vertx, retrieverOptions); diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index f00b2e900..cf2b8fc6b 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -2,7 +2,10 @@ import com.uid2.operator.service.ConfigRetrieverFactory; import com.uid2.operator.service.ConfigService; +import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.Vertx; +import io.vertx.core.http.HttpHeaders; import io.vertx.core.http.HttpServer; import io.vertx.core.json.JsonObject; import io.vertx.config.ConfigRetriever; @@ -45,43 +48,44 @@ void tearDown() { vertx.close(); } - void startMockServer(VertxTestContext testContext, JsonObject config) { + private Future startMockServer(JsonObject config) { if (server != null) { server.close(); } + Promise promise = Promise.promise(); + Router router = Router.router(vertx); router.route().handler(BodyHandler.create()); router.get("/config").handler(ctx -> ctx.response() - .putHeader("content-type", "application/json") + .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(config.encode())); server = vertx.createHttpServer() .requestHandler(router) .listen(Const.Port.ServicePortForCore,"127.0.0.1", http -> { - if (!http.succeeded()) { - testContext.failNow(http.cause()); + if (http.succeeded()) { + promise.complete(); + } else { + promise.fail(http.cause()); } }); + + return promise.future(); } @Test void testGetConfig(VertxTestContext testContext) { - JsonObject httpStoreConfig = new JsonObject().put("http", "value"); - this.startMockServer(testContext, httpStoreConfig); - ConfigService.create(configRetriever).onComplete(ar -> { - if (ar.succeeded()) { - ConfigService configService = ar.result(); - JsonObject retrievedConfig = configService.getConfig(); - assertNotNull(retrievedConfig, "Config retriever should initialise without error"); - assertTrue(retrievedConfig.fieldNames().containsAll(bootstrapConfig.fieldNames()), "Retrieved config should contain all keys in bootstrap config"); - assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); - testContext.completeNow(); - } else { - testContext.failNow(ar.cause()); - } - }); - + JsonObject httpStoreConfig = bootstrapConfig; + startMockServer(httpStoreConfig) + .compose(v -> ConfigService.create(configRetriever)) + .compose(configService -> { + JsonObject retrievedConfig = configService.getConfig(); + assertNotNull(retrievedConfig, "Config retriever should initialise without error"); + assertTrue(retrievedConfig.fieldNames().containsAll(httpStoreConfig.fieldNames()), "Retrieved config should contain all keys in http store config"); + return Future.succeededFuture(); + }) + .onComplete(testContext.succeedingThenComplete()); } @Test @@ -92,18 +96,14 @@ void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - this.startMockServer(testContext, invalidConfig); - ConfigService.create(spyRetriever).onComplete(ar -> { - if (ar.succeeded()) { - reset(spyRetriever); - ConfigService configService = ar.result(); - assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); - testContext.completeNow(); - } - else { - testContext.failNow(ar.cause()); - } - }); + startMockServer(invalidConfig) + .compose(v -> ConfigService.create(spyRetriever)) + .compose(configService -> { + reset(spyRetriever); + assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); + return Future.succeededFuture(); + }) + .onComplete(testContext.succeedingThenComplete()); } @Test @@ -111,17 +111,15 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - this.startMockServer(testContext, invalidConfig); - ConfigService.create(configRetriever).onComplete(ar -> { - if (ar.succeeded()) { - testContext.failNow(new RuntimeException("Expected a RuntimeException but the creation succeeded")); - } - else { - assertThrows(RuntimeException.class, () -> { - throw ar.cause(); - }); - testContext.completeNow(); - } - }); + startMockServer(invalidConfig) + .compose(v -> ConfigService.create(configRetriever)) + .compose(configService -> Future.failedFuture(new RuntimeException("Expected a RuntimeException but the creation succeeded"))) + .recover(throwable -> { + assertThrows(RuntimeException.class, () -> { + throw throwable; + }); + return Future.succeededFuture(); + }) + .onComplete(testContext.succeedingThenComplete()); } } \ No newline at end of file From 41e7489f52b024f2c7e1ae3d3bde44720e70844a Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 14 Jan 2025 17:12:18 +1100 Subject: [PATCH 26/29] Update feature flag functionality - exclusive config retriever to listen to changes in mounted configmap - listener directly interacts with ConfigServiceManager rather than publishing to EventBus --- conf/default-config.json | 2 +- src/main/java/com/uid2/operator/Const.java | 4 +- src/main/java/com/uid2/operator/Main.java | 112 +++++++++++------- .../service/ConfigRetrieverFactory.java | 29 ++++- .../service/ConfigServiceManager.java | 69 +++++------ .../service/DelegatingConfigService.java | 4 +- .../operator/service/StaticConfigService.java | 17 --- 7 files changed, 130 insertions(+), 107 deletions(-) delete mode 100644 src/main/java/com/uid2/operator/service/StaticConfigService.java diff --git a/conf/default-config.json b/conf/default-config.json index d160858de..932f8dbbd 100644 --- a/conf/default-config.json +++ b/conf/default-config.json @@ -36,6 +36,6 @@ "failure_shutdown_wait_hours": 120, "sharing_token_expiry_seconds": 2592000, "operator_type": "public", - "core_config_path": "/config", + "core_config_path": "/operator/config", "config_scan_period_ms": 300000 } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 271b36170..2be8eb38a 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,11 +30,11 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigPath = "core_config_path"; //TODO: update when endpoint name finalised + public static final String CoreConfigPath = "core_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; public static final String RemoteConfigFeatureFlag = "remote_config_feature_flag"; - public static final String RemoteConfigFlagEventBus = "RemoteConfigFlag"; + public static final String RemoteConfigFlagConfigMapPath = "remote_config_feature_flag_path"; } } diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index a0546b25a..6da6f699f 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -205,13 +205,6 @@ else if (!Utils.isProductionEnvironment()) { Vertx vertx = createVertx(); ConfigRetriever configRetriever = VertxUtils.createConfigRetriever(vertx); - configRetriever.listen(configChange -> { - JsonObject newConfig = configChange.getNewConfiguration(); - boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); - - vertx.eventBus().publish(Const.Config.RemoteConfigFlagEventBus, useDynamicConfig); - }); - configRetriever.getConfig(ar -> { if (ar.failed()) { LOGGER.error("Unable to read config: " + ar.cause().getMessage(), ar.cause()); @@ -277,45 +270,80 @@ private void run() { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); - ConfigServiceManager.create(vertx, config, config.getBoolean(Const.Config.RemoteConfigFeatureFlag, true)).compose(configServiceManager -> { - IConfigService configService = configServiceManager.getDelegatingConfigService(); - Supplier operatorVerticleSupplier = () -> { - UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); - return verticle; - }; + ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); + ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createHttpRetriever(vertx, config); + ConfigRetriever staticConfigRetriever = configRetrieverFactory.createJsonRetriever(vertx, config); - DeploymentOptions options = new DeploymentOptions(); - int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); - options.setInstances(svcInstances); + Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); + Future staticConfigFuture = ConfigService.create(staticConfigRetriever); - Promise compositePromise = Promise.promise(); - List fs = new ArrayList<>(); - fs.add(createAndDeployStatsCollector()); - try { - fs.add(createStoreVerticles()); - } catch (Exception e) { - throw new RuntimeException(e); - } + ConfigRetriever featureFlagConfigRetriever = configRetrieverFactory.createFileRetriever( + vertx, + config.getString(Const.Config.RemoteConfigFlagConfigMapPath, "conf/local-config.json") + ); + + Future featureFlagFuture = featureFlagConfigRetriever.getConfig(); + + Future.all(dynamicConfigFuture, staticConfigFuture, featureFlagFuture) + .compose(configServiceManagerCompositeFuture -> { + ConfigService dynamicConfigService = configServiceManagerCompositeFuture.resultAt(0); + ConfigService staticConfigService = configServiceManagerCompositeFuture.resultAt(1); + JsonObject featureFlagConfig = configServiceManagerCompositeFuture.resultAt(2); + + boolean featureFlag = featureFlagConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + + ConfigServiceManager configServiceManager = new ConfigServiceManager(vertx, dynamicConfigService, staticConfigService, featureFlag); - CompositeFuture.all(fs).onComplete(ar -> { - if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); - else compositePromise.complete(); - }); - - return compositePromise.future() - .compose(v -> { - metrics.setup(); - vertx.setPeriodic(60000, id -> metrics.update()); - Promise promise = Promise.promise(); - vertx.deployVerticle(operatorVerticleSupplier, options, promise); - return promise.future(); - }) - .onFailure(t -> { - LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); - vertx.close(); - System.exit(1); + featureFlagConfigRetriever.listen(change -> { + JsonObject newConfig = change.getNewConfiguration(); + boolean useDynamicConfig = newConfig.getBoolean(Const.Config.RemoteConfigFeatureFlag, true); + configServiceManager.updateConfigService(useDynamicConfig).onComplete(update -> { + if (update.succeeded()) { + LOGGER.info("Remote config feature flag toggled successfully"); + } else { + LOGGER.error("Failed to toggle remote config feature flag: " + update.cause()); + } + }); }); - }); + + IConfigService configService = configServiceManager.getDelegatingConfigService(); + Supplier operatorVerticleSupplier = () -> { + UIDOperatorVerticle verticle = new UIDOperatorVerticle(configService, config, this.clientSideTokenGenerate, siteProvider, clientKeyProvider, clientSideKeypairProvider, getKeyManager(), saltProvider, optOutStore, Clock.systemUTC(), _statsCollectorQueue, new SecureLinkValidatorService(this.serviceLinkProvider, this.serviceProvider), this.shutdownHandler::handleSaltRetrievalResponse); + return verticle; + }; + + DeploymentOptions options = new DeploymentOptions(); + int svcInstances = this.config.getInteger(Const.Config.ServiceInstancesProp); + options.setInstances(svcInstances); + + Promise compositePromise = Promise.promise(); + List fs = new ArrayList<>(); + fs.add(createAndDeployStatsCollector()); + try { + fs.add(createStoreVerticles()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + CompositeFuture.all(fs).onComplete(ar -> { + if (ar.failed()) compositePromise.fail(new Exception(ar.cause())); + else compositePromise.complete(); + }); + + return compositePromise.future() + .compose(v -> { + metrics.setup(); + vertx.setPeriodic(60000, id -> metrics.update()); + Promise promise = Promise.promise(); + vertx.deployVerticle(operatorVerticleSupplier, options, promise); + return promise.future(); + }) + .onFailure(t -> { + LOGGER.error("Failed to bootstrap operator: " + t.getMessage(), new Exception(t)); + vertx.close(); + System.exit(1); + }); + }); } private Future createStoreVerticles() throws Exception { diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 215b58482..11957fbe6 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -11,10 +11,9 @@ import static com.uid2.operator.Const.Config.CoreConfigPath; public class ConfigRetrieverFactory { - public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { + public ConfigRetriever createHttpRetriever(Vertx vertx, JsonObject bootstrapConfig) { String configPath = bootstrapConfig.getString(CoreConfigPath); - ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") .setOptional(true) @@ -29,4 +28,30 @@ public ConfigRetriever create(Vertx vertx, JsonObject bootstrapConfig) { return ConfigRetriever.create(vertx, retrieverOptions); } + + public ConfigRetriever createJsonRetriever(Vertx vertx, JsonObject config) { + ConfigStoreOptions jsonStore = new ConfigStoreOptions() + .setType("json") + .setConfig(config); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .setScanPeriod(-1) + .addStore(jsonStore); + + + return ConfigRetriever.create(vertx, retrieverOptions); + } + + public ConfigRetriever createFileRetriever(Vertx vertx, String path) { + ConfigStoreOptions fileStore = new ConfigStoreOptions() + .setType("file") + .setConfig(new JsonObject() + .put("path", path) + .put("format", "json")); + + ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() + .addStore(fileStore); + + return ConfigRetriever.create(vertx, retrieverOptions); + } } diff --git a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java index 1002fbecf..5fa3ffac2 100644 --- a/src/main/java/com/uid2/operator/service/ConfigServiceManager.java +++ b/src/main/java/com/uid2/operator/service/ConfigServiceManager.java @@ -1,67 +1,54 @@ package com.uid2.operator.service; -import com.uid2.operator.Const; -import io.vertx.config.ConfigRetriever; import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; -import io.vertx.core.json.JsonObject; +import io.vertx.core.shareddata.Lock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ConfigServiceManager { + private final Vertx vertx; private final DelegatingConfigService delegatingConfigService; - private final ConfigService dynamicConfigService; - private final StaticConfigService staticConfigService; + private final IConfigService dynamicConfigService; + private final IConfigService staticConfigService; private static final Logger logger = LoggerFactory.getLogger(ConfigServiceManager.class); - private ConfigServiceManager(ConfigService dynamicConfigService, StaticConfigService staticConfigService, boolean useDynamicConfig) { + public ConfigServiceManager(Vertx vertx, IConfigService dynamicConfigService, IConfigService staticConfigService, boolean useDynamicConfig) { + this.vertx = vertx; this.dynamicConfigService = dynamicConfigService; this.staticConfigService = staticConfigService; this.delegatingConfigService = new DelegatingConfigService(useDynamicConfig ? dynamicConfigService : staticConfigService); } - public static Future create(Vertx vertx, JsonObject bootstrapConfig, boolean useDynamicConfig) { - Promise promise = Promise.promise(); - - StaticConfigService staticConfigService = new StaticConfigService(bootstrapConfig); - - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); - - ConfigService.create(configRetriever).onComplete(ar -> { - if (ar.succeeded()) { - ConfigService dynamicConfigService = ar.result(); - ConfigServiceManager instance = new ConfigServiceManager(dynamicConfigService, staticConfigService, useDynamicConfig); - instance.initialiseListener(vertx); - promise.complete(instance); - } - else { - promise.fail(ar.cause()); + public Future updateConfigService(boolean useDynamicConfig) { + Promise promise = Promise.promise(); + vertx.sharedData().getLocalLock("updateConfigServiceLock", lockAsyncResult -> { + if (lockAsyncResult.succeeded()) { + Lock lock = lockAsyncResult.result(); + try { + if (useDynamicConfig) { + logger.info("Switching to DynamicConfigService"); + delegatingConfigService.updateConfigService(dynamicConfigService); + } else { + logger.info("Switching to StaticConfigService"); + delegatingConfigService.updateConfigService(staticConfigService); + } + promise.complete(); + } catch (Exception e) { + promise.fail(e); + } finally { + lock.release(); + } + } else { + logger.error("Failed to acquire lock for updating active ConfigService", lockAsyncResult.cause()); + promise.fail(lockAsyncResult.cause()); } }); return promise.future(); } - private void initialiseListener(Vertx vertx) { - vertx.eventBus().consumer(Const.Config.RemoteConfigFlagEventBus, message -> { - boolean useDynamicConfig = Boolean.parseBoolean(message.body().toString()); - - this.updateConfigService(useDynamicConfig); - }); - } - - public void updateConfigService(boolean useDynamicConfig) { - if (useDynamicConfig) { - logger.info("Switching to DynamicConfigService"); - this.delegatingConfigService.updateConfigService(dynamicConfigService); - } else { - logger.info("Switching to StaticConfigService"); - this.delegatingConfigService.updateConfigService(staticConfigService); - } - } - public IConfigService getDelegatingConfigService() { return delegatingConfigService; } diff --git a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java index 36fd77da2..a34d8b1c7 100644 --- a/src/main/java/com/uid2/operator/service/DelegatingConfigService.java +++ b/src/main/java/com/uid2/operator/service/DelegatingConfigService.java @@ -5,14 +5,14 @@ import java.util.concurrent.atomic.AtomicReference; public class DelegatingConfigService implements IConfigService{ - private AtomicReference activeConfigService; + private final AtomicReference activeConfigService; public DelegatingConfigService(IConfigService initialConfigService) { this.activeConfigService = new AtomicReference<>(initialConfigService); } public void updateConfigService(IConfigService newConfigService) { - this.activeConfigService = new AtomicReference<>(newConfigService); + this.activeConfigService.set(newConfigService); } @Override diff --git a/src/main/java/com/uid2/operator/service/StaticConfigService.java b/src/main/java/com/uid2/operator/service/StaticConfigService.java deleted file mode 100644 index e44440fc9..000000000 --- a/src/main/java/com/uid2/operator/service/StaticConfigService.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.uid2.operator.service; - -import io.vertx.core.json.JsonObject; - - -public class StaticConfigService implements IConfigService { - private final JsonObject staticConfig; - - public StaticConfigService(JsonObject staticConfig) { - this.staticConfig = staticConfig; - } - - @Override - public JsonObject getConfig() { - return staticConfig; - } -} From 272ac08b823210df67c993be57e04a037d1eecd2 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 14 Jan 2025 17:13:25 +1100 Subject: [PATCH 27/29] Add ConfigServiceManager tests --- .../operator/ConfigServiceManagerTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/test/java/com/uid2/operator/ConfigServiceManagerTest.java diff --git a/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java new file mode 100644 index 000000000..188949562 --- /dev/null +++ b/src/test/java/com/uid2/operator/ConfigServiceManagerTest.java @@ -0,0 +1,60 @@ +package com.uid2.operator; + +import com.uid2.operator.service.ConfigServiceManager; +import com.uid2.operator.service.IConfigService; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static com.uid2.operator.Const.Config.*; +import static org.junit.jupiter.api.Assertions.*; +import static com.uid2.operator.service.UIDOperatorService.*; +import static org.mockito.Mockito.*; + +@ExtendWith(VertxExtension.class) +public class ConfigServiceManagerTest { + private JsonObject bootstrapConfig; + private JsonObject staticConfig; + private ConfigServiceManager configServiceManager; + + @BeforeEach + void setUp(Vertx vertx) { + bootstrapConfig = new JsonObject() + .put(CoreConfigPath, "/operator/config") + .put(ConfigScanPeriodMs, 300000) + .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) + .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) + .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) + .put(MaxBidstreamLifetimeSecondsProp, 7200); + staticConfig = new JsonObject(bootstrapConfig.toString()) + .put(MaxBidstreamLifetimeSecondsProp, 7201); + + IConfigService dynamicConfigService = mock(IConfigService.class); + when(dynamicConfigService.getConfig()).thenReturn(bootstrapConfig); + IConfigService staticConfigService = mock(IConfigService.class); + when(staticConfigService.getConfig()).thenReturn(staticConfig); + + configServiceManager = new ConfigServiceManager(vertx, dynamicConfigService, staticConfigService, true); + } + + @Test + void testRemoteFeatureFlag(VertxTestContext testContext) { + IConfigService delegatingConfigService = configServiceManager.getDelegatingConfigService(); + + configServiceManager.updateConfigService(true) + .compose(updateToDynamic -> { + testContext.verify(() -> assertEquals(bootstrapConfig, delegatingConfigService.getConfig())); + + return configServiceManager.updateConfigService(false); + }) + .onSuccess(updateToStatic -> testContext.verify(() -> { + assertEquals(staticConfig, delegatingConfigService.getConfig()); + testContext.completeNow(); + })) + .onFailure(testContext::failNow); + } +} From 68530b4017a97b1c7c9e91f5c56a9c98d5cffe86 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Tue, 14 Jan 2025 17:17:17 +1100 Subject: [PATCH 28/29] Update ConfigServiceTest to remove unnecessary use of mock server --- .../com/uid2/operator/ConfigServiceTest.java | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index cf2b8fc6b..fe943f61e 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -25,26 +25,29 @@ class ConfigServiceTest { private Vertx vertx; private JsonObject bootstrapConfig; - private ConfigRetriever configRetriever; private HttpServer server; + private ConfigRetrieverFactory configRetrieverFactory; @BeforeEach void setUp() { vertx = Vertx.vertx(); bootstrapConfig = new JsonObject() - .put(CoreConfigPath, "/config") + .put(CoreConfigPath, "/operator/config") .put(ConfigScanPeriodMs, 300000) .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) .put(REFRESH_IDENTITY_TOKEN_AFTER_SECONDS, 1800) .put(MaxBidstreamLifetimeSecondsProp, 7200); - ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - configRetriever = configRetrieverFactory.create(vertx, bootstrapConfig); + configRetrieverFactory = new ConfigRetrieverFactory(); + } @AfterEach void tearDown() { + if (server != null) { + server.close(); + } vertx.close(); } @@ -57,7 +60,7 @@ private Future startMockServer(JsonObject config) { Router router = Router.router(vertx); router.route().handler(BodyHandler.create()); - router.get("/config").handler(ctx -> ctx.response() + router.get("/operator/config").handler(ctx -> ctx.response() .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") .end(config.encode())); @@ -76,6 +79,7 @@ private Future startMockServer(JsonObject config) { @Test void testGetConfig(VertxTestContext testContext) { + ConfigRetriever configRetriever = configRetrieverFactory.createHttpRetriever(vertx, bootstrapConfig); JsonObject httpStoreConfig = bootstrapConfig; startMockServer(httpStoreConfig) .compose(v -> ConfigService.create(configRetriever)) @@ -91,13 +95,12 @@ void testGetConfig(VertxTestContext testContext) { @Test void testInvalidConfigRevertsToPrevious(VertxTestContext testContext) { JsonObject lastConfig = new JsonObject().put("previous", "config"); - ConfigRetriever spyRetriever = spy(configRetriever); - when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - startMockServer(invalidConfig) - .compose(v -> ConfigService.create(spyRetriever)) + ConfigRetriever spyRetriever = spy(configRetrieverFactory.createJsonRetriever(vertx, invalidConfig)); + when(spyRetriever.getCachedConfig()).thenReturn(lastConfig); + ConfigService.create(spyRetriever) .compose(configService -> { reset(spyRetriever); assertEquals(lastConfig, configService.getConfig(), "Invalid config not reverted to previous config"); @@ -111,15 +114,13 @@ void testFirstInvalidConfigThrowsRuntimeException(VertxTestContext testContext) JsonObject invalidConfig = new JsonObject() .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 1000) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 2000); - startMockServer(invalidConfig) - .compose(v -> ConfigService.create(configRetriever)) - .compose(configService -> Future.failedFuture(new RuntimeException("Expected a RuntimeException but the creation succeeded"))) - .recover(throwable -> { - assertThrows(RuntimeException.class, () -> { - throw throwable; - }); - return Future.succeededFuture(); - }) - .onComplete(testContext.succeedingThenComplete()); + ConfigRetriever configRetriever = configRetrieverFactory.createJsonRetriever(vertx, invalidConfig); + ConfigService.create(configRetriever) + .onComplete(testContext.failing(throwable -> { + assertThrows(RuntimeException.class, () -> { + throw throwable; + }, "Expected a RuntimeException but the creation succeeded"); + testContext.completeNow(); + })); } } \ No newline at end of file From 1dfdea331deda4fa59d8444b9cfb596d1a0e9e40 Mon Sep 17 00:00:00 2001 From: Behnam Mozafari Date: Wed, 15 Jan 2025 16:19:41 +1100 Subject: [PATCH 29/29] Add authorisation header to config retriever --- conf/integ-config.json | 4 ++-- scripts/aws/conf/integ-uid2-config.json | 3 ++- src/main/java/com/uid2/operator/Const.java | 2 +- src/main/java/com/uid2/operator/Main.java | 4 ++-- .../operator/service/ConfigRetrieverFactory.java | 15 ++++++++++----- .../java/com/uid2/operator/ConfigServiceTest.java | 8 +++++--- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/conf/integ-config.json b/conf/integ-config.json index f1cf90742..4aa307c35 100644 --- a/conf/integ-config.json +++ b/conf/integ-config.json @@ -14,6 +14,6 @@ "optout_api_token": "test-operator-key", "optout_api_uri": "http://localhost:8081/optout/replicate", "salts_expired_shutdown_hours": 12, - "operator_type": "public" - + "operator_type": "public", + "core_operator_config_path": "http://localhost:8088/operator/config" } \ No newline at end of file diff --git a/scripts/aws/conf/integ-uid2-config.json b/scripts/aws/conf/integ-uid2-config.json index a7272a26a..91b53b1f2 100644 --- a/scripts/aws/conf/integ-uid2-config.json +++ b/scripts/aws/conf/integ-uid2-config.json @@ -11,5 +11,6 @@ "core_attest_url": "https://core-integ.uidapi.com/attest", "optout_api_uri": "https://optout-integ.uidapi.com/optout/replicate", "optout_s3_folder": "uid-optout-integ/", - "allow_legacy_api": false + "allow_legacy_api": false, + "core_operator_config_path": "https://core-integ.uidapi.com/operator/config" } diff --git a/src/main/java/com/uid2/operator/Const.java b/src/main/java/com/uid2/operator/Const.java index 2be8eb38a..b1623f5ee 100644 --- a/src/main/java/com/uid2/operator/Const.java +++ b/src/main/java/com/uid2/operator/Const.java @@ -30,7 +30,7 @@ public class Config extends com.uid2.shared.Const.Config { public static final String MaxInvalidPaths = "logging_limit_max_invalid_paths_per_interval"; public static final String MaxVersionBucketsPerSite = "logging_limit_max_version_buckets_per_site"; - public static final String CoreConfigPath = "core_config_path"; + public static final String CoreConfigPath = "core_operator_config_path"; public static final String ConfigScanPeriodMs = "config_scan_period_ms"; public static final String Config = "config"; public static final String identityV3 = "identity_v3"; diff --git a/src/main/java/com/uid2/operator/Main.java b/src/main/java/com/uid2/operator/Main.java index 6da6f699f..470fafd80 100644 --- a/src/main/java/com/uid2/operator/Main.java +++ b/src/main/java/com/uid2/operator/Main.java @@ -266,12 +266,12 @@ private ICloudStorage wrapCloudStorageForOptOut(ICloudStorage cloudStorage) { } } - private void run() { + private void run() throws Exception { this.createVertxInstancesMetric(); this.createVertxEventLoopsMetric(); ConfigRetrieverFactory configRetrieverFactory = new ConfigRetrieverFactory(); - ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createHttpRetriever(vertx, config); + ConfigRetriever dynamicConfigRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, config, this.createOperatorKeyRetriever().retrieve()); ConfigRetriever staticConfigRetriever = configRetrieverFactory.createJsonRetriever(vertx, config); Future dynamicConfigFuture = ConfigService.create(dynamicConfigRetriever); diff --git a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java index 11957fbe6..488c9d69e 100644 --- a/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java +++ b/src/main/java/com/uid2/operator/service/ConfigRetrieverFactory.java @@ -1,26 +1,31 @@ package com.uid2.operator.service; -import com.uid2.operator.Const; import io.vertx.config.ConfigRetriever; import io.vertx.config.ConfigRetrieverOptions; import io.vertx.config.ConfigStoreOptions; import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; +import java.net.URI; +import java.net.URISyntaxException; + import static com.uid2.operator.Const.Config.ConfigScanPeriodMs; import static com.uid2.operator.Const.Config.CoreConfigPath; public class ConfigRetrieverFactory { - public ConfigRetriever createHttpRetriever(Vertx vertx, JsonObject bootstrapConfig) { + public ConfigRetriever createRemoteConfigRetriever(Vertx vertx, JsonObject bootstrapConfig, String operatorKey) throws URISyntaxException { String configPath = bootstrapConfig.getString(CoreConfigPath); + URI uri = new URI(configPath); ConfigStoreOptions httpStore = new ConfigStoreOptions() .setType("http") .setOptional(true) .setConfig(new JsonObject() - .put("host", "127.0.0.1") - .put("port", Const.Port.ServicePortForCore) - .put("path", configPath)); + .put("host", uri.getHost()) + .put("port", uri.getPort()) + .put("path", uri.getPath()) + .put("headers", new JsonObject() + .put("Authorization", "Bearer " + operatorKey))); ConfigRetrieverOptions retrieverOptions = new ConfigRetrieverOptions() .setScanPeriod(bootstrapConfig.getLong(ConfigScanPeriodMs)) diff --git a/src/test/java/com/uid2/operator/ConfigServiceTest.java b/src/test/java/com/uid2/operator/ConfigServiceTest.java index fe943f61e..d7ff8e9e7 100644 --- a/src/test/java/com/uid2/operator/ConfigServiceTest.java +++ b/src/test/java/com/uid2/operator/ConfigServiceTest.java @@ -16,6 +16,8 @@ import io.vertx.junit5.VertxTestContext; import org.junit.jupiter.api.extension.ExtendWith; +import java.net.URISyntaxException; + import static com.uid2.operator.Const.Config.*; import static com.uid2.operator.service.UIDOperatorService.*; import static org.junit.jupiter.api.Assertions.*; @@ -32,7 +34,7 @@ class ConfigServiceTest { void setUp() { vertx = Vertx.vertx(); bootstrapConfig = new JsonObject() - .put(CoreConfigPath, "/operator/config") + .put(CoreConfigPath, "http://localhost:8088/operator/config") .put(ConfigScanPeriodMs, 300000) .put(IDENTITY_TOKEN_EXPIRES_AFTER_SECONDS, 3600) .put(REFRESH_TOKEN_EXPIRES_AFTER_SECONDS, 7200) @@ -78,8 +80,8 @@ private Future startMockServer(JsonObject config) { } @Test - void testGetConfig(VertxTestContext testContext) { - ConfigRetriever configRetriever = configRetrieverFactory.createHttpRetriever(vertx, bootstrapConfig); + void testGetConfig(VertxTestContext testContext) throws URISyntaxException { + ConfigRetriever configRetriever = configRetrieverFactory.createRemoteConfigRetriever(vertx, bootstrapConfig, ""); JsonObject httpStoreConfig = bootstrapConfig; startMockServer(httpStoreConfig) .compose(v -> ConfigService.create(configRetriever))