Merge pull request #334 from IABTechLab/cbc-UID2-4379-rename-s3-encry…

…ption-cloud-encryption

Starting with the rename of shared

pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-shared</artifactId>
-    <version>7.21.7</version>
+    <version>7.21.12-alpha-165-SNAPSHOT</version>
     <name>${project.groupId}:${project.artifactId}</name>
     <description>Library for all the shared uid2 operations</description>
     <url>https://github.com/IABTechLab/uid2docs</url>
@@ -68,7 +68,7 @@
             <dependency>
                 <groupId>com.google.cloud</groupId>
                 <artifactId>libraries-bom</artifactId>
-                <version>26.26.0</version>
+                <version>26.50.0</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -186,12 +186,22 @@
         <dependency>
             <groupId>com.google.auth</groupId>
             <artifactId>google-auth-library-oauth2-http</artifactId>
-            <version>1.23.0</version>
+            <version>1.30.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.auth</groupId>
+            <artifactId>google-auth-library-credentials</artifactId>
+            <version>1.30.0</version>
         </dependency>
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-logging</artifactId>
-            <version>3.20.6</version>
+            <version>3.15.12</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.25.5</version>
         </dependency>
         <dependency>
             <groupId>com.azure</groupId>

src/main/java/com/uid2/shared/Const.java

@@ -63,7 +63,7 @@ public static class Config {
         public static final String ServiceLinkMetadataPathProp = "service_links_metadata_path";
         public static final String SitesMetadataPathProp = "sites_metadata_path";
         public static final String OperatorsMetadataPathProp = "operators_metadata_path";
-        public static final String S3keysMetadataPathProp = "s3_keys_metadata_path";
+        public static final String CloudEncryptionKeysMetadataPathProp = "cloud_encryption_keys_metadata_path";
         public static final String SaltsMetadataPathProp = "salts_metadata_path";
         public static final String OptOutMetadataPathProp = "optout_metadata_path";
         public static final String CoreAttestUrlProp = "core_attest_url";

src/main/java/com/uid2/shared/model/S3Key.java → src/main/java/com/uid2/shared/model/CloudEncryptionKey.java

@@ -7,15 +7,15 @@
 import java.util.Objects;
 
 @JsonPropertyOrder({ "id", "siteId", "activates", "created", "secret" })
-public class S3Key {
+public class CloudEncryptionKey {
     private final int id;
     private final int siteId;
     private final long activates;
     private final long created;
     private final String secret;
 
     @JsonCreator
-    public S3Key(
+    public CloudEncryptionKey(
             @JsonProperty("id") int id,
             @JsonProperty("site_id") int siteId,
             @JsonProperty("activates") long activates,
@@ -52,12 +52,12 @@ public String getSecret() {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        S3Key s3Key = (S3Key) o;
-        return id == s3Key.id &&
-                siteId == s3Key.siteId &&
-                activates == s3Key.activates &&
-                created == s3Key.created &&
-                Objects.equals(secret, s3Key.secret);
+        CloudEncryptionKey cloudEncryptionKey = (CloudEncryptionKey) o;
+        return id == cloudEncryptionKey.id &&
+                siteId == cloudEncryptionKey.siteId &&
+                activates == cloudEncryptionKey.activates &&
+                created == cloudEncryptionKey.created &&
+                Objects.equals(secret, cloudEncryptionKey.secret);
     }
 
     @Override

src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java

@@ -1,12 +1,11 @@
 package com.uid2.shared.store;
 
 import com.uid2.shared.cloud.DownloadCloudStorage;
-import com.uid2.shared.model.S3Key;
+import com.uid2.shared.model.CloudEncryptionKey;
 import com.uid2.shared.store.parser.Parser;
 import com.uid2.shared.store.parser.ParsingResult;
-import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
-import com.uid2.shared.store.reader.RotatingS3KeyProvider;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
 import io.vertx.core.json.JsonObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -15,19 +14,18 @@
 
 import com.uid2.shared.encryption.AesGcm;
 
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.Map;
 
 public class EncryptedScopedStoreReader<T> extends ScopedStoreReader<T> {
     private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class);
 
-    private final RotatingS3KeyProvider s3KeyProvider;
+    private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
 
-    public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser<T> parser, String dataTypeName, RotatingS3KeyProvider s3KeyProvider) {
+    public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser<T> parser, String dataTypeName, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
         super(fileStreamProvider, scope, parser, dataTypeName);
-        this.s3KeyProvider = s3KeyProvider;
+        this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
     }
 
     @Override
@@ -52,9 +50,9 @@ protected String getDecryptedContent(String encryptedContent) throws Exception {
         JsonObject json = new JsonObject(encryptedContent);
         int keyId = json.getInteger("key_id");
         String encryptedPayload = json.getString("encrypted_payload");
-        Map<Integer, S3Key> s3Keys = s3KeyProvider.getAll();
-        S3Key decryptionKey = null;
-        for (S3Key key : s3Keys.values()) {
+        Map<Integer, CloudEncryptionKey> cloudEncryptionKeys = cloudEncryptionKeyProvider.getAll();
+        CloudEncryptionKey decryptionKey = null;
+        for (CloudEncryptionKey key : cloudEncryptionKeys.values()) {
             if (key.getId() == keyId) {
                 decryptionKey = key;
                 break;

src/main/java/com/uid2/shared/store/parser/CloudEncryptionKeyParser.java

@@ -0,0 +1,23 @@
+package com.uid2.shared.store.parser;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.uid2.shared.model.CloudEncryptionKey;
+import com.uid2.shared.util.Mapper;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class CloudEncryptionKeyParser implements Parser<Map<Integer, CloudEncryptionKey>> {
+    private static final ObjectMapper OBJECT_MAPPER = Mapper.getInstance();
+
+    @Override
+    public ParsingResult<Map<Integer, CloudEncryptionKey>> deserialize(InputStream inputStream) throws IOException {
+        CloudEncryptionKey[] cloudEncryptionKeys = OBJECT_MAPPER.readValue(inputStream, CloudEncryptionKey[].class);
+        Map<Integer, CloudEncryptionKey> cloudEncryptionKeysMap = Arrays.stream(cloudEncryptionKeys)
+                .collect(Collectors.toMap(CloudEncryptionKey::getId, s -> s));
+        return new ParsingResult<>(cloudEncryptionKeysMap, cloudEncryptionKeysMap.size());
+    }
+}

src/main/java/com/uid2/shared/store/reader/RotatingClientKeyProvider.java

@@ -9,7 +9,6 @@
 import com.uid2.shared.store.IClientKeyProvider;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.ClientParser;
-import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -49,8 +48,8 @@ public RotatingClientKeyProvider(DownloadCloudStorage fileStreamProvider, StoreS
         this.authorizableStore = new AuthorizableStore<>(ClientKey.class);
     }
 
-    public RotatingClientKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingS3KeyProvider s3KeyProvider) {
-        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new ClientParser(), "auth keys", s3KeyProvider);
+    public RotatingClientKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new ClientParser(), "auth keys", cloudEncryptionKeyProvider);
         this.authorizableStore = new AuthorizableStore<>(ClientKey.class);
     }
 

src/main/java/com/uid2/shared/store/reader/RotatingS3KeyProvider.java → src/main/java/com/uid2/shared/store/reader/RotatingCloudEncryptionKeyProvider.java

@@ -3,9 +3,9 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.ScopedStoreReader;
-import com.uid2.shared.store.parser.S3KeyParser;
+import com.uid2.shared.store.parser.CloudEncryptionKeyParser;
 import com.uid2.shared.store.scope.StoreScope;
-import com.uid2.shared.model.S3Key;
+import com.uid2.shared.model.CloudEncryptionKey;
 import io.vertx.core.json.JsonObject;
 
 import java.util.Set;
@@ -23,14 +23,14 @@
 
 import java.time.Instant;
 
-public class RotatingS3KeyProvider implements StoreReader<Map<Integer, S3Key>> {
-    ScopedStoreReader<Map<Integer, S3Key>> reader;
+public class RotatingCloudEncryptionKeyProvider implements StoreReader<Map<Integer, CloudEncryptionKey>> {
+    ScopedStoreReader<Map<Integer, CloudEncryptionKey>> reader;
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(RotatingS3KeyProvider.class);
-    public Map<Integer, List<S3Key>> siteToKeysMap = new HashMap<>();
+    private static final Logger LOGGER = LoggerFactory.getLogger(RotatingCloudEncryptionKeyProvider.class);
+    public Map<Integer, List<CloudEncryptionKey>> siteToKeysMap = new HashMap<>();
 
-    public RotatingS3KeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) {
-        this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new S3KeyParser(), "s3encryption_keys");
+    public RotatingCloudEncryptionKeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) {
+        this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new CloudEncryptionKeyParser(), "cloud_encryption_keys");
     }
 
     @Override
@@ -50,19 +50,19 @@ public long getVersion(JsonObject metadata) {
 
     @Override
     public long loadContent(JsonObject metadata) throws Exception {
-        long result = reader.loadContent(metadata, "s3encryption_keys");
+        long result = reader.loadContent(metadata, "cloud_encryption_keys");
         updateSiteToKeysMapping();
         return result;
     }
 
     @Override
-    public Map<Integer, S3Key> getAll() {
-        Map<Integer, S3Key> keys = reader.getSnapshot();
+    public Map<Integer, CloudEncryptionKey> getAll() {
+        Map<Integer, CloudEncryptionKey> keys = reader.getSnapshot();
         return keys != null ? keys : new HashMap<>();
     }
 
     public void updateSiteToKeysMapping() {
-        Map<Integer, S3Key> allKeys = getAll();
+        Map<Integer, CloudEncryptionKey> allKeys = getAll();
         siteToKeysMap.clear();
         allKeys.values().forEach(key ->
                 this.siteToKeysMap
@@ -85,28 +85,28 @@ public int getTotalSites() {
         return siteToKeysMap.size();
     }
 
-    public List<S3Key> getKeys(int siteId) {
+    public List<CloudEncryptionKey> getKeys(int siteId) {
         //for s3 encryption keys retrieval
         return siteToKeysMap.getOrDefault(siteId, new ArrayList<>());
     }
 
-    public Collection<S3Key> getKeysForSite(Integer siteId) {
-        Map<Integer, S3Key> allKeys = getAll();
+    public Collection<CloudEncryptionKey> getKeysForSite(Integer siteId) {
+        Map<Integer, CloudEncryptionKey> allKeys = getAll();
         return allKeys.values().stream()
                 .filter(key -> key.getSiteId() == (siteId))
                 .collect(Collectors.toList());
     }
 
-    public S3Key getEncryptionKeyForSite(Integer siteId) {
+    public CloudEncryptionKey getEncryptionKeyForSite(Integer siteId) {
         //get the youngest activated key
-        Collection<S3Key> keys = getKeysForSite(siteId);
+        Collection<CloudEncryptionKey> keys = getKeysForSite(siteId);
         long now = Instant.now().getEpochSecond();
         if (keys.isEmpty()) {
             throw new IllegalStateException("No S3 keys available for encryption for site ID: " + siteId);
         }
         return keys.stream()
                 .filter(key -> key.getActivates() <= now)
-                .max(Comparator.comparingLong(S3Key::getCreated))
+                .max(Comparator.comparingLong(CloudEncryptionKey::getCreated))
                 .orElseThrow(() -> new IllegalStateException("No active keys found for site ID: " + siteId));
     }
 }

src/main/java/com/uid2/shared/store/reader/RotatingKeyAclProvider.java

@@ -3,7 +3,6 @@
 import com.uid2.shared.auth.AclSnapshot;
 import com.uid2.shared.auth.EncryptionKeyAcl;
 import com.uid2.shared.cloud.DownloadCloudStorage;
-import com.uid2.shared.cloud.ICloudStorage;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.IKeyAclProvider;
@@ -23,8 +22,8 @@ public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls");
     }
 
-    public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
-        this.reader =  new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls", s3KeyProvider);
+    public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader =  new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls", cloudEncryptionKeyProvider);
     }
 
     @Override

src/main/java/com/uid2/shared/store/reader/RotatingKeyStore.java

@@ -52,8 +52,8 @@ public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, StoreScope scop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys");
     }
 
-    public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
-        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys", s3KeyProvider);
+    public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys", cloudEncryptionKeyProvider);
     }
 
     @Override

src/main/java/com/uid2/shared/store/reader/RotatingKeysetKeyStore.java

@@ -7,7 +7,6 @@
 import com.uid2.shared.store.KeysetKeyStoreSnapshot;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeysetKeyParser;
-import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import com.uid2.shared.store.EncryptedScopedStoreReader;
 import io.vertx.core.json.JsonObject;
@@ -22,8 +21,8 @@ public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys");
     }
 
-    public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingS3KeyProvider s3KeyProvider) {
-        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys", s3KeyProvider);
+    public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys", cloudEncryptionKeyProvider);
     }
 
     @Override

src/main/java/com/uid2/shared/store/reader/RotatingKeysetProvider.java

@@ -7,7 +7,6 @@
 import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeysetParser;
-import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -21,8 +20,8 @@ public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeysetParser(), "keysets");
     }
 
-    public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingS3KeyProvider s3KeyProvider) {
-        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider,scope,new KeysetParser(),"keysets",s3KeyProvider);
+    public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider,scope,new KeysetParser(),"keysets",cloudEncryptionKeyProvider);
     }
 
     public KeysetSnapshot getSnapshot(Instant asOf) {

src/main/java/com/uid2/shared/store/reader/RotatingSiteStore.java

@@ -7,7 +7,6 @@
 import com.uid2.shared.store.ISiteStore;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.SiteParser;
-import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -23,8 +22,8 @@ public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, StoreScope sco
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites");
     }
 
-    public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingS3KeyProvider s3KeyProvider) {
-        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites", s3KeyProvider);
+    public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites", cloudEncryptionKeyProvider);
     }
 
     @Override

src/main/java/com/uid2/shared/vertx/RotatingStoreVerticle.java

@@ -3,7 +3,6 @@
 import com.uid2.shared.health.HealthComponent;
 import com.uid2.shared.health.HealthManager;
 import com.uid2.shared.store.reader.IMetadataVersionedStore;
-import com.uid2.shared.store.reader.RotatingS3KeyProvider;
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Gauge;
 import io.micrometer.core.instrument.Metrics;