Merge pull request #546 from IABTechLab/mkc-UID2-2937-optout-api

Add method OptOutStoreSnapshot.getAdIdOptOutTimestamp

conf/default-config.json

@@ -17,6 +17,8 @@
   "optout_partition_interval": 86400,
   "optout_max_partitions": 30,
   "optout_heap_default_capacity": 8192,
+  "optout_status_api_enabled": false,
+  "optout_status_max_request_size": 5000,
   "cloud_download_threads": 8,
   "cloud_upload_threads": 2,
   "cloud_refresh_interval": 60,

conf/local-config.json

@@ -32,6 +32,7 @@
   "optout_heap_default_capacity": 8192,
   "optout_max_partitions": 30,
   "optout_partition_interval": 86400,
+  "optout_status_api_enabled": true,
   "client_side_token_generate": true,
   "client_side_token_generate_domain_name_check_enabled": true,
   "key_sharing_endpoint_provide_app_names": true,

conf/local-e2e-docker-public-config.json

@@ -31,6 +31,7 @@
   "optout_metadata_path": "/optout/refresh",
   "optout_api_uri": "http://optout:8081/optout/replicate",
   "optout_delta_rotate_interval": 60,
+  "optout_status_api_enabled": true,
   "cloud_refresh_interval": 30,
   "salts_expired_shutdown_hours": 12
 }

src/main/java/com/uid2/operator/Const.java

@@ -23,5 +23,7 @@ public class Config extends com.uid2.shared.Const.Config {
         public static final String AzureSecretNameProp = "azure_secret_name";
 
         public static final String GcpSecretVersionNameProp = "gcp_secret_version_name";
+        public static final String OptOutStatusApiEnabled = "optout_status_api_enabled";
+        public static final String OptOutStatusMaxRequestSize = "optout_status_max_request_size";
     }
 }

src/main/java/com/uid2/operator/Main.java

@@ -140,7 +140,7 @@ public Main(Vertx vertx, JsonObject config) throws Exception {
         this.keysetProvider = new RotatingKeysetProvider(fsStores, new GlobalScope(new CloudPath(keysetMdPath)));
         String saltsMdPath = this.config.getString(Const.Config.SaltsMetadataPathProp);
         this.saltProvider = new RotatingSaltProvider(fsStores, saltsMdPath);
-        this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey);
+        this.optOutStore = new CloudSyncOptOutStore(vertx, fsLocal, this.config, operatorKey, Clock.systemUTC());
 
         if (this.validateServiceLinks) {
             String serviceMdPath = this.config.getString(Const.Config.ServiceMetadataPathProp);

src/main/java/com/uid2/operator/store/IOptOutStore.java

@@ -15,5 +15,7 @@ public interface IOptOutStore {
      */
     Instant getLatestEntry(UserIdentity firstLevelHashIdentity);
 
+    long getOptOutTimestampByAdId(String adId);
+
     void addEntry(UserIdentity firstLevelHashIdentity, byte[] advertisingId, Handler<AsyncResult<Instant>> handler);
 }

src/main/java/com/uid2/operator/vertx/UIDOperatorVerticle.java

@@ -100,6 +100,8 @@ public class UIDOperatorVerticle extends AbstractVerticle {
     private final Map<Tuple.Tuple3<String, OptoutCheckPolicy, String>, Counter> _tokenGeneratePolicyCounters = new HashMap<>();
     private final Map<String, Tuple.Tuple2<Counter, Counter>> _identityMapUnmappedIdentifiers = new HashMap<>();
     private final Map<String, Counter> _identityMapRequestWithUnmapped = new HashMap<>();
+
+    private final Map<String, DistributionSummary> optOutStatusCounters = new HashMap<>();
     private final IdentityScope identityScope;
     private final V2PayloadHandler v2PayloadHandler;
     private final boolean phoneSupport;
@@ -120,6 +122,9 @@ public class UIDOperatorVerticle extends AbstractVerticle {
     protected boolean keySharingEndpointProvideAppNames;
     protected Instant lastInvalidOriginProcessTime = Instant.now();
 
+    private final int optOutStatusMaxRequestSize;
+    private final boolean optOutStatusApiEnabled;
+
     public UIDOperatorVerticle(JsonObject config,
                                boolean clientSideTokenGenerate,
                                ISiteStore siteProvider,
@@ -167,6 +172,8 @@ public UIDOperatorVerticle(JsonObject config,
         this.allowClockSkewSeconds = config.getInteger(Const.Config.AllowClockSkewSecondsProp, 1800);
         this.maxSharingLifetimeSeconds = config.getInteger(Const.Config.MaxSharingLifetimeProp, config.getInteger(Const.Config.SharingTokenExpiryProp));
         this.saltRetrievalResponseHandler = saltRetrievalResponseHandler;
+        this.optOutStatusApiEnabled = config.getBoolean(Const.Config.OptOutStatusApiEnabled, false);
+        this.optOutStatusMaxRequestSize = config.getInteger(Const.Config.OptOutStatusMaxRequestSize, 5000);
     }
 
     @Override
@@ -277,7 +284,11 @@ private void setupV2Routes(Router mainRouter, BodyHandler bodyHandler) {
                 rc -> v2PayloadHandler.handle(rc, this::handleKeysBidstream), Role.ID_READER));
         v2Router.post("/token/logout").handler(bodyHandler).handler(auth.handleV1(
                 rc -> v2PayloadHandler.handleAsync(rc, this::handleLogoutAsyncV2), Role.OPTOUT));
-
+        if (this.optOutStatusApiEnabled) {
+            v2Router.post("/optout/status").handler(bodyHandler).handler(auth.handleV1(
+                    rc -> v2PayloadHandler.handle(rc, this::handleOptoutStatus),
+                    Role.MAPPER, Role.SHARER, Role.ID_READER));
+        }
 
         if (this.clientSideTokenGenerate)
             v2Router.post("/token/client-generate").handler(bodyHandler).handler(this::handleClientSideTokenGenerate);
@@ -1679,6 +1690,74 @@ private void recordIdentityMapStatsForServiceLinks(RoutingContext rc, String api
         }
     }
 
+    private List<String> parseOptoutStatusRequestPayload(RoutingContext rc) {
+        final JsonObject requestObj = (JsonObject) rc.data().get("request");
+        if (requestObj == null) {
+            ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Invalid request body");
+            return null;
+        }
+        final JsonArray rawUidsJsonArray = requestObj.getJsonArray("advertising_ids");
+        if (rawUidsJsonArray == null) {
+            ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Required Parameter Missing: advertising_ids");
+            return null;
+        }
+        if (rawUidsJsonArray.size() > optOutStatusMaxRequestSize) {
+            ResponseUtil.Error(ResponseStatus.ClientError, HttpStatus.SC_BAD_REQUEST, rc, "Request payload is too large");
+            return null;
+        }
+        List<String> rawUID2sInputList = new ArrayList<>(rawUidsJsonArray.size());
+        for (int i = 0; i < rawUidsJsonArray.size(); ++i) {
+            rawUID2sInputList.add(rawUidsJsonArray.getString(i));
+        }
+        return rawUID2sInputList;
+    }
+
+    private void handleOptoutStatus(RoutingContext rc) {
+        try {
+            // Parse request to get list of raw UID2 strings
+            List<String> rawUID2sInput = parseOptoutStatusRequestPayload(rc);
+            if (rawUID2sInput == null) {
+                return;
+            }
+            final JsonArray optedOutJsonArray = new JsonArray();
+            for (String rawUId : rawUID2sInput) {
+                // Call opt out service to get timestamp of opted out identities
+                long timestamp = optOutStore.getOptOutTimestampByAdId(rawUId);
+                if (timestamp != -1) {
+                    JsonObject optOutJsonObj = new JsonObject();
+                    optOutJsonObj.put("advertising_id", rawUId);
+                    optOutJsonObj.put("opted_out_since", timestamp);
+                    optedOutJsonArray.add(optOutJsonObj);
+                }
+            }
+            // Create response and return
+            final JsonObject bodyJsonObj = new JsonObject();
+            bodyJsonObj.put("opted_out", optedOutJsonArray);
+            ResponseUtil.SuccessV2(rc, bodyJsonObj);
+            recordOptOutStatusEndpointStats(rc, rawUID2sInput.size(), optedOutJsonArray.size());
+        } catch (Exception e) {
+            ResponseUtil.Error(ResponseStatus.UnknownError, 500, rc,
+                    "Unknown error while getting optout status", e);
+        }
+    }
+
+    private void recordOptOutStatusEndpointStats(RoutingContext rc, int inputCount, int optOutCount) {
+        String apiContact = getApiContact(rc);
+        DistributionSummary inputDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary
+                .builder("uid2.operator.optout.status.input_size")
+                .description("number of UIDs received in request")
+                .tags("api_contact", apiContact)
+                .register(Metrics.globalRegistry));
+        inputDistSummary.record(inputCount);
+
+        DistributionSummary optOutDistSummary = optOutStatusCounters.computeIfAbsent(apiContact, k -> DistributionSummary
+                .builder("uid2.operator.optout.status.optout_size")
+                .description("number of UIDs that have opted out")
+                .tags("api_contact", apiContact)
+                .register(Metrics.globalRegistry));
+        optOutDistSummary.record(optOutCount);
+    }
+
     private RefreshResponse refreshIdentity(RoutingContext rc, String tokenStr) {
         final RefreshToken refreshToken;
         try {

src/test/java/com/uid2/operator/UIDOperatorVerticleTest.java

@@ -42,7 +42,6 @@
 import io.vertx.ext.web.client.WebClient;
 import io.vertx.junit5.VertxExtension;
 import io.vertx.junit5.VertxTestContext;
-import org.apache.commons.collections4.CollectionUtils;
 import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -96,6 +95,8 @@ public class UIDOperatorVerticleTest {
     private static final String clientSideTokenGeneratePrivateKey = "UID2-Y-L-MEECAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcEJzAlAgEBBCBop1Dw/IwDcstgicr/3tDoyR3OIpgAWgw8mD6oTO+1ug==";
     private static final int clientSideTokenGenerateSiteId = 123;
 
+    private static final int optOutStatusMaxRequestSize = 1000;
+
     private AutoCloseable mocks;
     @Mock private ISiteStore siteProvider;
     @Mock private IClientKeyProvider clientKeyProvider;
@@ -159,6 +160,8 @@ private void setupConfig(JsonObject config) {
         config.put("client_side_token_generate_log_invalid_http_origins", true);
 
         config.put(Const.Config.AllowClockSkewSecondsProp, 3600);
+        config.put(Const.Config.OptOutStatusApiEnabled, true);
+        config.put(Const.Config.OptOutStatusMaxRequestSize, optOutStatusMaxRequestSize);
     }
 
     private static byte[] makeAesKey(String prefix) {
@@ -2115,6 +2118,103 @@ void identityMapBatchRequestTooLarge(String apiVersion, Vertx vertx, VertxTestCo
         send(apiVersion, vertx, apiVersion + "/identity/map", false, null, req, 413, json -> testContext.completeNow());
     }
 
+    private static Stream<Arguments> optOutStatusRequestData() {
+        List<String> rawUIDS = Arrays.asList("RUQbFozFwnmPVjDx8VMkk9vJoNXUJImKnz2h9RfzzM24",
+            "qAmIGxqLk_RhOtm4f1nLlqYewqSma8fgvjEXYnQ3Jr0K",
+            "r3wW2uvJkwmeFcbUwSeM6BIpGF8tX38wtPfVc4wYyo71",
+            "e6SA-JVAXnvk8F1MUtzsMOyWuy5Xqe15rLAgqzSGiAbz");
+        Map<String, Long> optedOutIdsCase1 = new HashMap<>();
+
+        optedOutIdsCase1.put(rawUIDS.get(0), Instant.now().minus(1, ChronoUnit.DAYS).getEpochSecond());
+        optedOutIdsCase1.put(rawUIDS.get(1), Instant.now().minus(2, ChronoUnit.DAYS).getEpochSecond());
+        optedOutIdsCase1.put(rawUIDS.get(2), -1L);
+        optedOutIdsCase1.put(rawUIDS.get(3), -1L);
+
+        Map<String, Long> optedOutIdsCase2 = new HashMap<>();
+        optedOutIdsCase2.put(rawUIDS.get(2), -1L);
+        optedOutIdsCase2.put(rawUIDS.get(3), -1L);
+        return Stream.of(
+            Arguments.arguments(optedOutIdsCase1, 2, Role.MAPPER),
+            Arguments.arguments(optedOutIdsCase1, 2, Role.ID_READER),
+            Arguments.arguments(optedOutIdsCase1, 2, Role.SHARER),
+            Arguments.arguments(optedOutIdsCase2, 0, Role.MAPPER)
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("optOutStatusRequestData")
+    void optOutStatusRequest(Map<String, Long> optedOutIds, int optedOutCount, Role role, Vertx vertx, VertxTestContext testContext) {
+        fakeAuth(126, role);
+        setupSalts();
+        setupKeys();
+
+        JsonArray rawUIDs = new JsonArray();
+        for (String rawUID2 : optedOutIds.keySet()) {
+            when(this.optOutStore.getOptOutTimestampByAdId(rawUID2)).thenReturn(optedOutIds.get(rawUID2));
+            rawUIDs.add(rawUID2);
+        }
+        JsonObject requestJson = new JsonObject();
+        requestJson.put("advertising_ids", rawUIDs);
+
+        send("v2", vertx, "v2/optout/status", false, null, requestJson, 200, respJson -> {
+            assertEquals("success", respJson.getString("status"));
+            JsonArray optOutJsonArray = respJson.getJsonObject("body").getJsonArray("opted_out");
+            assertEquals(optedOutCount, optOutJsonArray.size());
+            for (int i = 0; i < optOutJsonArray.size(); ++i) {
+                JsonObject optOutObject = optOutJsonArray.getJsonObject(i);
+                assertEquals(optedOutIds.get(optOutObject.getString("advertising_id")),
+                        optOutObject.getLong("opted_out_since"));
+            }
+            testContext.completeNow();
+        });
+    }
+
+    private static Stream<Arguments> optOutStatusValidationErrorData() {
+        // Test case 1
+        JsonArray rawUIDs = new JsonArray();
+
+        for (int i = 0; i <= optOutStatusMaxRequestSize; ++i) {
+            byte[] rawUid2Bytes = Random.getBytes(32);
+            rawUIDs.add(Utils.toBase64String(rawUid2Bytes));
+        }
+
+        JsonObject requestJson1 = new JsonObject();
+        requestJson1.put("advertising_ids", rawUIDs);
+        // Test case 2
+        JsonObject requestJson2 = new JsonObject();
+        requestJson2.put("advertising", rawUIDs);
+        return Stream.of(
+                Arguments.arguments(requestJson1, "Request payload is too large"),
+                Arguments.arguments(requestJson2, "Required Parameter Missing: advertising_ids")
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("optOutStatusValidationErrorData")
+    void optOutStatusValidationError(JsonObject requestJson, String errorMsg, Vertx vertx, VertxTestContext testContext) {
+        fakeAuth(126, Role.MAPPER);
+        setupSalts();
+        setupKeys();
+
+        send("v2", vertx, "v2/optout/status", false, null, requestJson, 400, respJson -> {
+            assertEquals(com.uid2.shared.Const.ResponseStatus.ClientError, respJson.getString("status"));
+            assertEquals(errorMsg, respJson.getString("message"));
+            testContext.completeNow();
+        });
+    }
+
+    @Test
+    void optOutStatusUnauthorized(Vertx vertx, VertxTestContext testContext) {
+        fakeAuth(126, Role.GENERATOR);
+        setupSalts();
+        setupKeys();
+
+        send("v2", vertx, "v2/optout/status", false, null, new JsonObject(), 401, respJson -> {
+            assertEquals(com.uid2.shared.Const.ResponseStatus.Unauthorized, respJson.getString("status"));
+            testContext.completeNow();
+        });
+    }
+
     @Test
     void LogoutV2(Vertx vertx, VertxTestContext testContext) {
         final int clientSiteId = 201;

src/test/java/com/uid2/operator/benchmark/BenchmarkCommon.java

@@ -181,7 +181,7 @@ static class StaticOptOutStore implements IOptOutStore {
         private CloudSyncOptOutStore.OptOutStoreSnapshot snapshot;
 
         public StaticOptOutStore(ICloudStorage storage, JsonObject jsonConfig, Collection<String> partitions) throws CloudStorageException, IOException {
-            snapshot = new CloudSyncOptOutStore.OptOutStoreSnapshot(storage, jsonConfig);
+            snapshot = new CloudSyncOptOutStore.OptOutStoreSnapshot(storage, jsonConfig, Clock.systemUTC());
             snapshot = snapshot.updateIndex(partitions);
             System.out.println(snapshot.size());
         }
@@ -197,6 +197,11 @@ public Instant getLatestEntry(UserIdentity firstLevelHashIdentity) {
         public void addEntry(UserIdentity firstLevelHashIdentity, byte[] advertisingId, Handler<AsyncResult<Instant>> handler) {
             // noop
         }
+
+        @Override
+        public long getOptOutTimestampByAdId(String adId) {
+            return -1;
+        }
     }
 
 }