From 2e0d5acf6f2fcea97ad1d598fd7ce0c3900242d6 Mon Sep 17 00:00:00 2001 From: jon8787 <112368577+jon8787@users.noreply.github.com> Date: Mon, 18 Sep 2023 20:13:36 +1000 Subject: [PATCH 1/9] expose metrics for record counts (ScopedStoreReader) --- .../com/uid2/shared/store/ScopedStoreReader.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java index ecee9432..25479f6c 100644 --- a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java @@ -6,11 +6,14 @@ import com.uid2.shared.store.parser.Parser; import com.uid2.shared.store.parser.ParsingResult; import com.uid2.shared.store.scope.StoreScope; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Metrics; import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.InputStream; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; public class ScopedStoreReader { @@ -22,6 +25,7 @@ public class ScopedStoreReader { private final String dataTypeName; private final DownloadCloudStorage contentStreamProvider; private final AtomicReference latestSnapshot; + private final AtomicLong latestEntryCount = new AtomicLong(-1L); public ScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser parser, String dataTypeName) { this.metadataStreamProvider = fileStreamProvider; @@ -34,6 +38,11 @@ public ScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope sco this.contentStreamProvider = fileStreamProvider; } latestSnapshot = new AtomicReference<>(); + + Gauge.builder("uid2_config_store_entry_count", latestEntryCount::get) + .tag("store", dataTypeName) + .description("gauge for " + dataTypeName + "store total entry count") + .register(Metrics.globalRegistry); } public CloudPath getMetadataPath() { @@ -55,8 +64,11 @@ private long loadContent(String path) throws Exception { try (InputStream inputStream = this.contentStreamProvider.download(path)) { ParsingResult parsed = parser.deserialize(inputStream); latestSnapshot.set(parsed.getData()); - LOGGER.info(String.format("Loaded %d %s", parsed.getCount(), dataTypeName)); - return parsed.getCount(); + + final int count = parsed.getCount(); + latestEntryCount.set(count); + LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); + return count; } } From 4c72a4adf635bf005c02f0e9d9df955c54822945 Mon Sep 17 00:00:00 2001 From: jon8787 <112368577+jon8787@users.noreply.github.com> Date: Tue, 19 Sep 2023 08:39:07 +1000 Subject: [PATCH 2/9] add spaces --- .../java/com/uid2/shared/store/ScopedStoreReader.java | 2 +- .../java/com/uid2/shared/vertx/RotatingStoreVerticle.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java index 25479f6c..acb747a5 100644 --- a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java @@ -41,7 +41,7 @@ public ScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope sco Gauge.builder("uid2_config_store_entry_count", latestEntryCount::get) .tag("store", dataTypeName) - .description("gauge for " + dataTypeName + "store total entry count") + .description("gauge for " + dataTypeName + " store total entry count") .register(Metrics.globalRegistry); } diff --git a/src/main/java/com/uid2/shared/vertx/RotatingStoreVerticle.java b/src/main/java/com/uid2/shared/vertx/RotatingStoreVerticle.java index aa5f1bdc..c7c03511 100644 --- a/src/main/java/com/uid2/shared/vertx/RotatingStoreVerticle.java +++ b/src/main/java/com/uid2/shared/vertx/RotatingStoreVerticle.java @@ -39,22 +39,22 @@ public RotatingStoreVerticle(String storeName, long refreshIntervalMs, IMetadata this.counterStoreRefreshed = Counter .builder("uid2.config_store.refreshed") .tag("store", storeName) - .description("counter for how many times " + storeName + "store is refreshed") + .description("counter for how many times " + storeName + " store is refreshed") .register(Metrics.globalRegistry); this.counterStoreRefreshTimeMs = Counter .builder("uid2.config_store.refreshtime_ms") .tag("store", storeName) - .description("counter for total time (ms) " + storeName + "store spend in refreshing") + .description("counter for total time (ms) " + storeName + " store spend in refreshing") .register(Metrics.globalRegistry); this.gaugeStoreVersion = Gauge .builder("uid2.config_store.version", () -> this.latestVersion.get()) .tag("store", storeName) - .description("gauge for " + storeName + "store version") + .description("gauge for " + storeName + " store version") .register(Metrics.globalRegistry); this.gaugeStoreEntryCount = Gauge .builder("uid2.config_store.entry_count", () -> this.latestEntryCount.get()) .tag("store", storeName) - .description("gauge for " + storeName + "store total entry count") + .description("gauge for " + storeName + " store total entry count") .register(Metrics.globalRegistry); this.gaugeConsecutiveRefreshFailures = Gauge .builder("uid2.config_store.consecutive_refresh_failures", () -> this.storeRefreshIsFailing.get()) From 28851019aeea348062d766c245a1521a63e5b778 Mon Sep 17 00:00:00 2001 From: jon8787 <112368577+jon8787@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:19:54 +1000 Subject: [PATCH 3/9] error handling --- src/main/java/com/uid2/shared/store/ScopedStoreReader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java index acb747a5..42fc0461 100644 --- a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java @@ -69,6 +69,10 @@ private long loadContent(String path) throws Exception { latestEntryCount.set(count); LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); return count; + } catch (Exception e) { + latestEntryCount.set(-1); + LOGGER.error(String.format("Unable to load %s", dataTypeName)); + throw e; } } From 2d694a5831f64c13e407b89aff21b6482919f612 Mon Sep 17 00:00:00 2001 From: jon8787 <112368577+jon8787@users.noreply.github.com> Date: Tue, 19 Sep 2023 11:38:27 +1000 Subject: [PATCH 4/9] remove -1 metric since getSnapshot() will retain previous successful attempt --- src/main/java/com/uid2/shared/store/ScopedStoreReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java index 42fc0461..71407dc6 100644 --- a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java @@ -69,8 +69,8 @@ private long loadContent(String path) throws Exception { latestEntryCount.set(count); LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); return count; - } catch (Exception e) { - latestEntryCount.set(-1); + } + catch (Exception e) { LOGGER.error(String.format("Unable to load %s", dataTypeName)); throw e; } From 436139108a0aa019507faabdbb26060fa445db15 Mon Sep 17 00:00:00 2001 From: Thomas Manson Date: Tue, 19 Sep 2023 12:00:01 +1000 Subject: [PATCH 5/9] Updated tests --- .../shared/store/RotatingServiceLinkStoreTest.java | 10 +++++----- .../uid2/shared/store/RotatingServiceStoreTest.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/uid2/shared/store/RotatingServiceLinkStoreTest.java b/src/test/java/com/uid2/shared/store/RotatingServiceLinkStoreTest.java index bdc46120..d2dc7eac 100644 --- a/src/test/java/com/uid2/shared/store/RotatingServiceLinkStoreTest.java +++ b/src/test/java/com/uid2/shared/store/RotatingServiceLinkStoreTest.java @@ -64,9 +64,9 @@ public void loadContentEmptyArray() throws Exception { @Test public void loadContentMultipleServices() throws Exception { JsonArray content = new JsonArray(); - ServiceLink l1 = addServiceLink(content, "abc123", 1, 123, "AWS Venice"); + ServiceLink l1 = addServiceLink(content, "abc123", 1, 123, "Test Service 1"); ServiceLink l2 = addServiceLink(content, "abc123", 2, 123, "test1"); - ServiceLink l3 = addServiceLink(content, "ghi789", 1, 123, "AWS Venice"); + ServiceLink l3 = addServiceLink(content, "ghi789", 1, 123, "Test Service 1"); ServiceLink l4 = addServiceLink(content, "jkl1011", 3, 124, "test2"); when(cloudStorage.download("locationPath")).thenReturn(makeInputStream(content)); @@ -77,9 +77,9 @@ public void loadContentMultipleServices() throws Exception { @Test public void findServiceLinksMultipleServices() throws Exception { JsonArray content = new JsonArray(); - ServiceLink l1 = addServiceLink(content, "abc123", 1, 123, "AWS Venice"); + ServiceLink l1 = addServiceLink(content, "abc123", 1, 123, "Test Service 1"); ServiceLink l2 = addServiceLink(content, "abc123", 2, 123, "test1"); - ServiceLink l3 = addServiceLink(content, "ghi789", 1, 123, "AWS Venice"); + ServiceLink l3 = addServiceLink(content, "ghi789", 1, 123, "Test Service 1"); ServiceLink l4 = addServiceLink(content, "jkl1011", 3, 124, "test2"); when(cloudStorage.download("locationPath")).thenReturn(makeInputStream(content)); @@ -87,7 +87,7 @@ public void findServiceLinksMultipleServices() throws Exception { ServiceLink sl = serviceLinkStore.getServiceLink(1, "abc123"); assertNotNull(sl); - assertEquals("AWS Venice", sl.getName()); + assertEquals("Test Service 1", sl.getName()); assertEquals(1, sl.getServiceId()); assertEquals(123, sl.getSiteId()); assertEquals("abc123", sl.getLinkId()); diff --git a/src/test/java/com/uid2/shared/store/RotatingServiceStoreTest.java b/src/test/java/com/uid2/shared/store/RotatingServiceStoreTest.java index 3bacc35a..16cccb8a 100644 --- a/src/test/java/com/uid2/shared/store/RotatingServiceStoreTest.java +++ b/src/test/java/com/uid2/shared/store/RotatingServiceStoreTest.java @@ -66,9 +66,9 @@ public void loadContentEmptyArray() throws Exception { @Test public void loadContentMultipleServices() throws Exception { JsonArray content = new JsonArray(); - Service s1 = addService(content, 1, 123, "AWS Venice", Set.of()); + Service s1 = addService(content, 1, 123, "Test Service 1", Set.of()); Service s2 = addService(content, 2, 123, "test1", Set.of(Role.GENERATOR)); - Service s3 = addService(content, 3, 124, "AWS Venice", Set.of(Role.GENERATOR, Role.SHARING_PORTAL)); + Service s3 = addService(content, 3, 124, "Test Service 1", Set.of(Role.GENERATOR, Role.SHARING_PORTAL)); Service s4 = addService(content, 4, 125, "test2", Set.of(Role.CLIENTKEY_ISSUER)); when(cloudStorage.download("locationPath")).thenReturn(makeInputStream(content)); From e4c068510ca964ba082a9a91ec703abc666ed052 Mon Sep 17 00:00:00 2001 From: Katherine Chen Date: Tue, 19 Sep 2023 14:31:31 +1000 Subject: [PATCH 6/9] Add `getOperatorKeyFromHash` for `RotatingOperatorKeyProvider` (#134) --- .../com/uid2/shared/auth/RotatingOperatorKeyProvider.java | 5 +++++ .../java/com/uid2/shared/store/IOperatorKeyProvider.java | 1 + 2 files changed, 6 insertions(+) diff --git a/src/main/java/com/uid2/shared/auth/RotatingOperatorKeyProvider.java b/src/main/java/com/uid2/shared/auth/RotatingOperatorKeyProvider.java index 5493c61b..9bf29600 100644 --- a/src/main/java/com/uid2/shared/auth/RotatingOperatorKeyProvider.java +++ b/src/main/java/com/uid2/shared/auth/RotatingOperatorKeyProvider.java @@ -74,6 +74,11 @@ public OperatorKey getOperatorKey(String token) { return operatorKeyStore.getAuthorizableByKey(token); } + @Override + public OperatorKey getOperatorKeyFromHash(String hash) { + return (OperatorKey) this.operatorKeyStore.getAuthorizableByHash(hash); + } + @Override public Collection getAll() { return operatorKeyStore.getAuthorizables(); diff --git a/src/main/java/com/uid2/shared/store/IOperatorKeyProvider.java b/src/main/java/com/uid2/shared/store/IOperatorKeyProvider.java index ed69875b..8af803c8 100644 --- a/src/main/java/com/uid2/shared/store/IOperatorKeyProvider.java +++ b/src/main/java/com/uid2/shared/store/IOperatorKeyProvider.java @@ -7,5 +7,6 @@ public interface IOperatorKeyProvider extends IAuthorizableProvider { OperatorKey getOperatorKey(String token); + OperatorKey getOperatorKeyFromHash(String hash); Collection getAll(); } From 84d46c71b3aa63a06bb30e0016c0e72cd69faa54 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Tue, 19 Sep 2023 04:35:04 +0000 Subject: [PATCH 7/9] Released Patch version: 5.3.6-e4c068510c --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e629f3e8..dff313b2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-shared - 5.3.2-623d4a8784 + 5.3.6-e4c068510c ${project.groupId}:${project.artifactId} Library for all the shared uid2 operations https://github.com/IABTechLab/uid2docs From 6a14e3c1a02066db871c87e6c8bb9028f41f1ad8 Mon Sep 17 00:00:00 2001 From: jon8787 <112368577+jon8787@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:52:44 +1000 Subject: [PATCH 8/9] better logging --- src/main/java/com/uid2/shared/store/ScopedStoreReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java index 71407dc6..b989a394 100644 --- a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java @@ -67,11 +67,11 @@ private long loadContent(String path) throws Exception { final int count = parsed.getCount(); latestEntryCount.set(count); - LOGGER.info(String.format("Loaded %d %s", count, dataTypeName)); + LOGGER.info(String.format("Loaded %d %s from %s", count, dataTypeName, path)); return count; } catch (Exception e) { - LOGGER.error(String.format("Unable to load %s", dataTypeName)); + LOGGER.error(String.format("Unable to load %s from %s", dataTypeName, path)); throw e; } } From c00136ccd39f16ab7ddcb7b4fad7c84c0961b80c Mon Sep 17 00:00:00 2001 From: jon8787 <112368577+jon8787@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:09:43 +1000 Subject: [PATCH 9/9] changed metric name to uid2_scoped_store_entry_count to avoid clashing with RotatingStoreVerticle's uid2.config_store.entry_count --- src/main/java/com/uid2/shared/store/ScopedStoreReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java index b989a394..b3366e20 100644 --- a/src/main/java/com/uid2/shared/store/ScopedStoreReader.java +++ b/src/main/java/com/uid2/shared/store/ScopedStoreReader.java @@ -39,7 +39,7 @@ public ScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope sco } latestSnapshot = new AtomicReference<>(); - Gauge.builder("uid2_config_store_entry_count", latestEntryCount::get) + Gauge.builder("uid2_scoped_store_entry_count", latestEntryCount::get) .tag("store", dataTypeName) .description("gauge for " + dataTypeName + " store total entry count") .register(Metrics.globalRegistry);