diff --git a/conf/local-config.json b/conf/local-config.json index eca081f74..59332c650 100644 --- a/conf/local-config.json +++ b/conf/local-config.json @@ -34,7 +34,7 @@ "optout_partition_interval": 86400, "client_side_token_generate": true, "client_side_token_generate_domain_name_check_enabled": true, - "key_sharing_endpoint_provide_site_domain_names": true, + "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12 } diff --git a/conf/local-e2e-docker-public-config.json b/conf/local-e2e-docker-public-config.json index 8190951fe..af29da6f7 100644 --- a/conf/local-e2e-docker-public-config.json +++ b/conf/local-e2e-docker-public-config.json @@ -24,7 +24,7 @@ "client_side_token_generate": true, "client_side_token_generate_domain_name_check_enabled": true, "client_side_token_generate_log_invalid_http_origins": true, - "key_sharing_endpoint_provide_site_domain_names": true, + "key_sharing_endpoint_provide_app_names": true, "validate_service_links": true, "optout_s3_bucket": "test-optout-bucket", "optout_s3_folder": "optout-v2/", diff --git a/conf/local-e2e-public-config.json b/conf/local-e2e-public-config.json index e8ba64930..d36dc9139 100644 --- a/conf/local-e2e-public-config.json +++ b/conf/local-e2e-public-config.json @@ -37,7 +37,7 @@ "optout_partition_interval": 86400, "client_side_token_generate": true, "client_side_token_generate_domain_name_check_enabled": true, - "key_sharing_endpoint_provide_site_domain_names": true, + "key_sharing_endpoint_provide_app_names": true, "client_side_token_generate_log_invalid_http_origins": true, "salts_expired_shutdown_hours": 12 } diff --git a/conf/validator-latest-e2e-docker-public-config.json b/conf/validator-latest-e2e-docker-public-config.json index 2b94970d2..d6789bd00 100644 --- a/conf/validator-latest-e2e-docker-public-config.json +++ b/conf/validator-latest-e2e-docker-public-config.json @@ -25,7 +25,7 @@ "client_side_token_generate": true, "client_side_token_generate_domain_name_check_enabled": true, "client_side_token_generate_log_invalid_http_origins": true, - "key_sharing_endpoint_provide_site_domain_names": true, + "key_sharing_endpoint_provide_app_names": true, "validate_service_links": true, "optout_s3_bucket": "test-optout-bucket", "optout_s3_folder": "optout-v2/", diff --git a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java index e1af48cc7..8f2e20012 100644 --- a/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java +++ b/src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java @@ -50,6 +50,7 @@ import io.vertx.ext.web.handler.BodyHandler; import io.vertx.ext.web.handler.CorsHandler; import io.vertx.ext.web.handler.StaticHandler; +import org.apache.commons.collections4.CollectionUtils; import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,8 +116,8 @@ public class UIDOperatorVerticle extends AbstractVerticle { private final int maxBidstreamLifetimeSeconds; private final int allowClockSkewSeconds; protected int maxSharingLifetimeSeconds; - protected boolean keySharingEndpointProvideSiteDomainNames; protected Map> siteIdToInvalidOriginsAndAppNames = new HashMap<>(); + protected boolean keySharingEndpointProvideAppNames; protected Instant lastInvalidOriginProcessTime = Instant.now(); public UIDOperatorVerticle(JsonObject config, @@ -153,7 +154,7 @@ public UIDOperatorVerticle(JsonObject config, this.phoneSupport = config.getBoolean("enable_phone_support", true); this.tcfVendorId = config.getInteger("tcf_vendor_id", 21); this.cstgDoDomainNameCheck = config.getBoolean("client_side_token_generate_domain_name_check_enabled", true); - this.keySharingEndpointProvideSiteDomainNames = config.getBoolean("key_sharing_endpoint_provide_site_domain_names", false); + this.keySharingEndpointProvideAppNames = config.getBoolean("key_sharing_endpoint_provide_app_names", false); this._statsCollectorQueue = statsCollectorQueue; this.clientKeyProvider = clientKeyProvider; this.clientSideTokenGenerateLogInvalidHttpOrigin = config.getBoolean("client_side_token_generate_log_invalid_http_origins", false); @@ -670,7 +671,7 @@ private void addBidstreamHeaderFields(JsonObject resp) { } private void addSites(JsonObject resp, List keys, Map keysetMap) { - final List sites = getSitesWithDomainNames(keys, keysetMap); + final List sites = getSitesWithDomainOrAppNames(keys, keysetMap); if (sites != null) { /* The end result will look something like this: @@ -686,14 +687,16 @@ private void addSites(JsonObject resp, List keys, Map sitesJson = sites.stream() - .map(UIDOperatorVerticle::toJson) + .map(site -> UIDOperatorVerticle.toJson(site, keySharingEndpointProvideAppNames)) .collect(Collectors.toList()); resp.put("site_data", sitesJson); } @@ -732,12 +735,9 @@ private void addAllowClockSkewSecondsField(JsonObject resp) { resp.put("allow_clock_skew_seconds", allowClockSkewSeconds); } - private List getSitesWithDomainNames(List keys, Map keysetMap) { + private List getSitesWithDomainOrAppNames(List keys, Map keysetMap) { //without cstg enabled, operator won't have site data and siteProvider could be null - //and adding keySharingEndpointProvideSiteDomainNames in case something goes wrong - //and we can still enable cstg feature but turn off site domain name download in - // key/sharing endpoint - if (!keySharingEndpointProvideSiteDomainNames || !clientSideTokenGenerate) { + if (!clientSideTokenGenerate) { return null; } @@ -747,7 +747,13 @@ private List getSitesWithDomainNames(List keys, Map !site.getDomainNames().isEmpty()) + .filter(site -> { + if (CollectionUtils.isNotEmpty(site.getDomainNames())) { + return true; + } else { + return keySharingEndpointProvideAppNames && CollectionUtils.isNotEmpty(site.getAppNames()); + } + }) .collect(Collectors.toList()); } @@ -755,10 +761,15 @@ private List getSitesWithDomainNames(List keys, Map domainOrAppNames = new HashSet<>(site.getDomainNames()); + + if (includeAppNames) { + domainOrAppNames.addAll(site.getAppNames()); + } + siteObj.put("domain_names", domainOrAppNames.stream().sorted().collect(Collectors.toList())); return siteObj; } diff --git a/src/main/resources/com.uid2.core/test/keyset_keys/keyset_keys.json b/src/main/resources/com.uid2.core/test/keyset_keys/keyset_keys.json index 21262c02c..22b869045 100644 --- a/src/main/resources/com.uid2.core/test/keyset_keys/keyset_keys.json +++ b/src/main/resources/com.uid2.core/test/keyset_keys/keyset_keys.json @@ -78,5 +78,13 @@ "created": 1609459200, "activates": 1609469200, "expires": 4088629662 + }, + { + "id": 11, + "keyset_id": 901, + "secret": "YgyxOX4yX1gYhCINq7O9XxM6jX+etXqSXluZxjB1aG1=", + "created": 1713225363, + "activates": 1713250563, + "expires": 1715756163 } ] diff --git a/src/main/resources/com.uid2.core/test/keysets/keysets.json b/src/main/resources/com.uid2.core/test/keysets/keysets.json index e47eda1e2..9c177732a 100644 --- a/src/main/resources/com.uid2.core/test/keysets/keysets.json +++ b/src/main/resources/com.uid2.core/test/keysets/keysets.json @@ -136,5 +136,14 @@ "keyset_id": 801, "name": "My keyset #5", "site_id": 8 + }, + { + "site_id": 127, + "name": "App Name Test Site Key Set 1", + "keyset_id": 901, + "default": true, + "created": 1713225363, + "enabled": true, + "allowed_sites": [123] } ] diff --git a/src/main/resources/com.uid2.core/test/sites/sites.json b/src/main/resources/com.uid2.core/test/sites/sites.json index 6bb74ce18..6ece32a0e 100644 --- a/src/main/resources/com.uid2.core/test/sites/sites.json +++ b/src/main/resources/com.uid2.core/test/sites/sites.json @@ -20,5 +20,12 @@ "id": 126, "name": "AWS Venice", "enabled": true + }, + { + "id": 127, + "name": "App Name Test Site", + "enabled": true, + "app_names" : ["com.UID2.operator.TEST", "13456789"], + "domain_names" : ["example.com", "unifiedid.com"] } ] \ No newline at end of file diff --git a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java index d30c421bd..c90259fba 100644 --- a/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java +++ b/src/test/java/com/uid2/operator/ExtendedUIDOperatorVerticle.java @@ -36,8 +36,8 @@ public IUIDOperatorService getIdService() { return this.idService; } - public void setKeySharingEndpointProvideSiteDomainNames(boolean enable) { - this.keySharingEndpointProvideSiteDomainNames = enable; + public void setKeySharingEndpointProvideAppNames(boolean enable) { + this.keySharingEndpointProvideAppNames = enable; } public void setMaxSharingLifetimeSeconds(int maxSharingLifetimeSeconds) { diff --git a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java index fffbbae62..1a57827d8 100644 --- a/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java +++ b/src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java @@ -42,6 +42,7 @@ import io.vertx.ext.web.client.WebClient; import io.vertx.junit5.VertxExtension; import io.vertx.junit5.VertxTestContext; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -154,7 +155,7 @@ private void setupConfig(JsonObject config) { config.put("advertising_token_v4_percentage", getTokenVersion() == TokenVersion.V4 ? 100 : 0); config.put("identity_v3", useIdentityV3()); config.put("client_side_token_generate", true); - config.put("key_sharing_endpoint_provide_site_domain_names", true); + config.put("key_sharing_endpoint_provide_app_names", true); config.put("client_side_token_generate_log_invalid_http_origins", true); config.put(Const.Config.AllowClockSkewSecondsProp, 3600); @@ -4313,13 +4314,23 @@ void keySharingKeysets_CorrectFiltering(Vertx vertx, VertxTestContext testContex }); } + private static Site defaultMockSite(int siteId, boolean includeDomainNames, boolean includeAppNames) { + Site site = new Site(siteId, "site" + siteId, true); + if (includeDomainNames) { + site.setDomainNames(Set.of(siteId + ".com", siteId + ".co.uk")); + } + if (includeAppNames) { + site.setAppNames(Set.of(siteId + ".com.UID2.operator", siteId + "bundle123", "12345789")); + } + return site; + } + //set some default domain names for all possible sites for each unit test first - private void setupSiteDomainNameMock(int... siteIds) { + private void setupSiteDomainAndAppNameMock(boolean includeDomainNames, boolean includeAppNames, int... siteIds) { Map sites = new HashMap<>(); for(int siteId : siteIds) { - Site site = new Site(siteId, "site"+siteId, true, new HashSet<>(Arrays.asList(siteId+".com", siteId+".co.uk"))); - sites.put(site.getId(), site); + sites.put(siteId, defaultMockSite(siteId, includeDomainNames, includeAppNames)); } when(siteProvider.getAllSites()).thenReturn(new HashSet<>(sites.values())); @@ -4329,25 +4340,42 @@ private void setupSiteDomainNameMock(int... siteIds) { }); } - public HashMap> setupExpectation(int... siteIds) + private void setupMockSites(Map sites) { + when(siteProvider.getAllSites()).thenReturn(new HashSet<>(sites.values())); + when(siteProvider.getSite(anyInt())).thenAnswer(invocation -> { + int siteId = invocation.getArgument(0); + return sites.get(siteId); + }); + } + + static Map setupExpectation(boolean includeDomainNames, boolean includeAppNames, int... siteIds) { - HashMap> expectedSites = new HashMap(); + Map expectedSites = new HashMap<>(); for (int siteId : siteIds) { - List siteDomains = Arrays.asList(siteId+".co.uk", siteId+".com"); - expectedSites.put(siteId, siteDomains); + if (includeDomainNames || includeAppNames) { + expectedSites.put(siteId, defaultMockSite(siteId, includeDomainNames, includeAppNames)); + } } return expectedSites; } - public void verifyExpectedSiteDetail(HashMap> expectedSites, JsonArray actualResult) { - assertEquals(actualResult.size(), expectedSites.size()); + public void verifyExpectedSiteDetail(Map expectedSites, JsonArray actualResult) { + + assertEquals(expectedSites.size(), actualResult.size()); for(int i = 0; i < actualResult.size(); i++) { JsonObject siteDetail = actualResult.getJsonObject(i); int siteId = siteDetail.getInteger("id"); - assertTrue(expectedSites.get(siteId).containsAll((Collection) siteDetail.getMap().get("domain_names"))); + List actualDomainList = (List) siteDetail.getMap().get("domain_names"); + Site expectedSite = expectedSites.get(siteId); + int size = 0; + assertTrue(actualDomainList.containsAll(expectedSite.getDomainNames())); + size += expectedSite.getDomainNames().size(); + assertTrue(actualDomainList.containsAll(expectedSite.getAppNames())); + size += expectedSite.getAppNames().size(); + assertEquals(size, actualDomainList.size()); } } @@ -4382,13 +4410,40 @@ public void keyBidstreamReturnsCustomMaxBidstreamLifetimeHeader(Vertx vertx, Ver } } + + private static Stream testKeyDownloadEndpointKeysetsData_IDREADER() { + int[] expectedSiteIds = new int [] {101, 102}; + int[] allMockedSiteIds = new int [] {101, 102, 103, 105}; + Map expectedSitesDomainsOnly = setupExpectation(true, false, expectedSiteIds); + Map mockSitesWithDomainsOnly = setupExpectation(true, false, allMockedSiteIds); + + Map expectedSitesWithBoth = setupExpectation(true, true, expectedSiteIds); + Map mockSitesWithBoth = setupExpectation(true, true, allMockedSiteIds); + + Map expectedSitesWithAppNamesOnly = setupExpectation(false, true, expectedSiteIds); + Map mockSitesWithAppNamesOnly = setupExpectation(false, true, allMockedSiteIds); + Map emptySites = new HashMap<>(); + return Stream.of( + // Both domains and app names should be present in response + Arguments.of("true", KeyDownloadEndpoint.SHARING, mockSitesWithBoth, expectedSitesWithBoth), + Arguments.of("true", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithBoth, expectedSitesWithBoth), + + // only domains should be present in response + Arguments.of("false", KeyDownloadEndpoint.SHARING, mockSitesWithDomainsOnly, expectedSitesDomainsOnly), + Arguments.of("false", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithDomainsOnly, expectedSitesDomainsOnly), + + // only app names should be present in response + Arguments.of("true", KeyDownloadEndpoint.SHARING, mockSitesWithAppNamesOnly, expectedSitesWithAppNamesOnly), + Arguments.of("true", KeyDownloadEndpoint.BIDSTREAM, mockSitesWithAppNamesOnly, expectedSitesWithAppNamesOnly), + + // None + Arguments.of("false", KeyDownloadEndpoint.SHARING, emptySites, emptySites), + Arguments.of("false", KeyDownloadEndpoint.BIDSTREAM, emptySites, emptySites) + ); + } + @ParameterizedTest - @CsvSource({ - "true, SHARING", - "false, SHARING", - "true, BIDSTREAM", - "false, BIDSTREAM", - }) + @MethodSource("testKeyDownloadEndpointKeysetsData_IDREADER") // Test the /key/sharing and /key/bidstream endpoints when called with the ID_READER role. // // Tests: @@ -4398,10 +4453,11 @@ public void keyBidstreamReturnsCustomMaxBidstreamLifetimeHeader(Vertx vertx, Ver // ID_READER has no access to a keyset that is disabled - direct reject // ID_READER has no access to a keyset with an empty allowed_sites - reject by sharing // ID_READER has no access to a keyset with an allowed_sites for other sites - reject by sharing - void keyDownloadEndpointKeysets_IDREADER(boolean provideSiteDomainNames, KeyDownloadEndpoint endpoint, Vertx vertx, VertxTestContext testContext) { - - if (!provideSiteDomainNames) { - this.uidOperatorVerticle.setKeySharingEndpointProvideSiteDomainNames(false); + void keyDownloadEndpointKeysets_IDREADER(boolean provideAppNames, KeyDownloadEndpoint endpoint, + Map mockSites, Map expectedSites, + Vertx vertx, VertxTestContext testContext) { + if (!provideAppNames) { + this.uidOperatorVerticle.setKeySharingEndpointProvideAppNames(false); } String apiVersion = "v2"; int clientSiteId = 101; @@ -4435,7 +4491,7 @@ void keyDownloadEndpointKeysets_IDREADER(boolean provideSiteDomainNames, KeyDown createKey(1024, now.minusSeconds(5), now.minusSeconds(2), 9) }; - setupSiteDomainNameMock(101, 102, 103, 105); + setupMockSites(mockSites); //site 104 domain name list will be returned but we will set a blank list for it doReturn(new Site(104, "site104", true, new HashSet<>())).when(siteProvider).getSite(104); @@ -4450,16 +4506,9 @@ void keyDownloadEndpointKeysets_IDREADER(boolean provideSiteDomainNames, KeyDown checkEncryptionKeys(respJson, endpoint, clientSiteId, expectedKeys); - if(provideSiteDomainNames) { - HashMap> expectedSites = setupExpectation(101, 102); - // site 104 has empty domain name list intentionally previously so while site 104 should be included in - // this /key/sharing response, it won't appear in this domain name list - verifyExpectedSiteDetail(expectedSites, body.getJsonArray("site_data")); - } - else { - //otherwise we shouldn't even have a 'sites' field - assertNull(body.getJsonArray("site_data")); - } + // site 104 has empty domain name list intentionally previously so while site 104 should be included in + // this /key/sharing response, it won't appear in this domain name list + verifyExpectedSiteDetail(expectedSites, body.getJsonArray("site_data")); testContext.completeNow(); }); } @@ -4467,12 +4516,18 @@ void keyDownloadEndpointKeysets_IDREADER(boolean provideSiteDomainNames, KeyDown @Test void keySharingKeysets_SHARER_CustomMaxSharingLifetimeSeconds(Vertx vertx, VertxTestContext testContext) { this.uidOperatorVerticle.setMaxSharingLifetimeSeconds(999999); - keySharingKeysets_SHARER(vertx, testContext, 999999); + keySharingKeysets_SHARER(true, true, vertx, testContext, 999999); } - @Test - void keySharingKeysets_SHARER_defaultMaxSharingLifetimeSeconds(Vertx vertx, VertxTestContext testContext) { - keySharingKeysets_SHARER(vertx, testContext, this.config.getInteger(Const.Config.SharingTokenExpiryProp)); + @ParameterizedTest + @CsvSource({ + "true, true", + "true, false", + "false, false", + "true, false" + }) + void keySharingKeysets_SHARER_defaultMaxSharingLifetimeSeconds(boolean provideSiteDomainNames, boolean provideAppNames, Vertx vertx, VertxTestContext testContext) { + keySharingKeysets_SHARER(provideSiteDomainNames, provideAppNames, vertx, testContext, this.config.getInteger(Const.Config.SharingTokenExpiryProp)); } // Tests: @@ -4482,13 +4537,16 @@ void keySharingKeysets_SHARER_defaultMaxSharingLifetimeSeconds(Vertx vertx, Vert // SHARER has no access to a keyset with a missing allowed_sites - reject by sharing // SHARER has no access to a keyset with an empty allowed_sites - reject by sharing // SHARER has no access to a keyset with an allowed_sites for other sites - reject by sharing - void keySharingKeysets_SHARER(Vertx vertx, VertxTestContext testContext, int expectedMaxSharingLifetimeSeconds) { + void keySharingKeysets_SHARER(boolean provideSiteDomainNames, boolean provideAppNames, Vertx vertx, VertxTestContext testContext, int expectedMaxSharingLifetimeSeconds) { + if (!provideAppNames) { + this.uidOperatorVerticle.setKeySharingEndpointProvideAppNames(false); + } String apiVersion = "v2"; int clientSiteId = 101; fakeAuth(clientSiteId, Role.SHARER); MultipleKeysetsTests test = new MultipleKeysetsTests(); //To read these tests, open the MultipleKeysetsTests() constructor in another window so you can see the keyset contents and validate against expectedKeys - setupSiteDomainNameMock(101, 102, 103, 104, 105); + setupSiteDomainAndAppNameMock(provideSiteDomainNames, provideAppNames, 101, 102, 103, 104, 105); //Keys from these keysets are not expected: keyset6 (disabled keyset), keyset7 (sharing with ID_READERs but not SHARERs), keyset8 (not sharing with 101), keyset10 (not sharing with anyone) KeysetKey[] expectedKeys = { createKey(1001, now.minusSeconds(5), now.plusSeconds(3600), MasterKeysetId), @@ -4524,7 +4582,7 @@ void keySharingKeysets_SHARER(Vertx vertx, VertxTestContext testContext, int exp checkEncryptionKeys(respJson, KeyDownloadEndpoint.SHARING, clientSiteId, expectedKeys); - HashMap> expectedSites = setupExpectation(101, 104); + Map expectedSites = setupExpectation(provideSiteDomainNames, provideAppNames, 101, 104); verifyExpectedSiteDetail(expectedSites, respJson.getJsonObject("body").getJsonArray("site_data")); testContext.completeNow(); @@ -4545,7 +4603,7 @@ void keySharingKeysets_ReturnsMasterAndSite(Vertx vertx, VertxTestContext testCo new KeysetKey(102, "site key".getBytes(), now, now, now.plusSeconds(10), 10), }; MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); - setupSiteDomainNameMock(101, 102, 103, 104, 105); + setupSiteDomainAndAppNameMock(true, false, 101, 102, 103, 104, 105); Arrays.sort(encryptionKeys, Comparator.comparing(KeysetKey::getId)); send(apiVersion, vertx, apiVersion + "/key/sharing", true, null, null, 200, respJson -> { System.out.println(respJson); @@ -4578,7 +4636,7 @@ void keySharingKeysets_CorrectIDS(String testRun, Vertx vertx, VertxTestContext new KeysetKey(4, "key4".getBytes(), now, now, now.plusSeconds(10), 7), }; MultipleKeysetsTests test = new MultipleKeysetsTests(Arrays.asList(keysets), Arrays.asList(encryptionKeys)); - setupSiteDomainNameMock(10, 11, 12, 13); + setupSiteDomainAndAppNameMock(true, false, 10, 11, 12, 13); switch (testRun) { case "NoKeyset": siteId = 8; @@ -4620,7 +4678,7 @@ void keySharingKeysets_CorrectIDS(String testRun, Vertx vertx, VertxTestContext case "SharedKey": assertEquals(6, respJson.getJsonObject("body").getInteger("default_keyset_id")); //key 4 returned which has keyset id 7 which in turns has site id 13 - HashMap> expectedSites = setupExpectation(13); + Map expectedSites = setupExpectation(true, false,13); verifyExpectedSiteDetail(expectedSites, siteData); break; }