Wzh uid2 3571 change s3 provider for core

1

@@ -0,0 +1,6 @@
+Merge branch 'wzh-uid2-3570-add-encryoted-scope' of github.com:IABTechLab/uid2-shared into wzh-uid2-3570-add-encryoted-scope
+# Please enter a commit message to explain why this merge is necessary,
+# especially if it merges an updated upstream into a topic branch.
+#
+# Lines starting with '#' will be ignored, and an empty message aborts
+# the commit.

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-shared</artifactId>
-    <version>7.13.0</version>
+    <version>7.13.7-alpha-109-SNAPSHOT</version>
     <name>${project.groupId}:${project.artifactId}</name>
     <description>Library for all the shared uid2 operations</description>
     <url>https://github.com/IABTechLab/uid2docs</url>

src/main/java/com/uid2/shared/store/EncryptedScopedStoreReader.java

@@ -0,0 +1,89 @@
+package com.uid2.shared.store;
+
+import com.uid2.shared.cloud.DownloadCloudStorage;
+import com.uid2.shared.model.S3Key;
+import com.uid2.shared.store.parser.Parser;
+import com.uid2.shared.store.parser.ParsingResult;
+import com.uid2.shared.store.scope.EncryptedScope;
+import com.uid2.shared.store.scope.StoreScope;
+import com.uid2.shared.store.reader.RotatingS3KeyProvider;
+import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+
+import com.uid2.shared.encryption.AesGcm;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Map;
+
+public class EncryptedScopedStoreReader<T> extends ScopedStoreReader<T> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedScopedStoreReader.class);
+
+    private final int siteId;
+    private final RotatingS3KeyProvider s3KeyProvider;
+
+    public EncryptedScopedStoreReader(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, Parser<T> parser, String dataTypeName, RotatingS3KeyProvider s3KeyProvider) {
+        super(fileStreamProvider, scope, parser, dataTypeName);
+        this.siteId = scope.getId();
+        this.s3KeyProvider = s3KeyProvider;
+    }
+
+    @Override
+    protected long loadContent(String path) throws Exception {
+        try (InputStream inputStream = this.contentStreamProvider.download(path)) {
+            String encryptedContent = inputStreamToString(inputStream);
+            String decryptedContent = getDecryptedContent(encryptedContent);
+            ParsingResult<T> parsed = this.parser.deserialize(new ByteArrayInputStream(decryptedContent.getBytes(StandardCharsets.UTF_8)));
+            latestSnapshot.set(parsed.getData());
+
+            final int count = parsed.getCount();
+            latestEntryCount.set(count);
+            LOGGER.info(String.format("Loaded %d %s", count, dataTypeName));
+            return count;
+        } catch (Exception e) {
+            LOGGER.error(String.format("Unable to load %s", dataTypeName));
+            throw e;
+        }
+    }
+
+    protected String getDecryptedContent(String encryptedContent) throws Exception {
+        JsonObject json = new JsonObject(encryptedContent);
+        int keyId = json.getInteger("key_id");
+        String encryptedPayload = json.getString("encrypted_payload");
+
+        Map<Integer, S3Key> s3Keys = s3KeyProvider.getAll();
+        S3Key decryptionKey = null;
+
+        for (S3Key key : s3Keys.values()) {
+            if (key.getSiteId() == siteId && key.getId() == keyId) {
+                decryptionKey = key;
+                break;
+            }
+        }
+
+        if (decryptionKey == null) {
+            throw new IllegalStateException("No matching S3 key found for decryption for site ID: " + siteId + " and key ID: " + keyId);
+        }
+
+        byte[] secret = Base64.getDecoder().decode(decryptionKey.getSecret());
+        byte[] encryptedBytes = Base64.getDecoder().decode(encryptedPayload);
+        byte[] decryptedBytes = AesGcm.decrypt(encryptedBytes, 0, secret);
+
+        return new String(decryptedBytes, StandardCharsets.UTF_8);
+    }
+
+    public static String inputStreamToString(InputStream inputStream) throws IOException {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
+            StringBuilder stringBuilder = new StringBuilder();
+            String line;
+            while ((line = reader.readLine()) != null) {
+                stringBuilder.append(line);
+            }
+            return stringBuilder.toString();
+        }
+    }
+}

src/main/java/com/uid2/shared/store/ScopedStoreReader.java

@@ -21,11 +21,11 @@ public class ScopedStoreReader<T> {
 
     private final DownloadCloudStorage metadataStreamProvider;
     private final StoreScope scope;
-    private final Parser<T> parser;
-    private final String dataTypeName;
-    private final DownloadCloudStorage contentStreamProvider;
-    private final AtomicReference<T> latestSnapshot;
-    private final AtomicLong latestEntryCount = new AtomicLong(-1L);
+    protected final Parser<T> parser;
+    protected final String dataTypeName;
+    protected final DownloadCloudStorage contentStreamProvider;
+    protected final AtomicReference<T> latestSnapshot;
+    protected final AtomicLong latestEntryCount = new AtomicLong(-1L);
 
     public ScopedStoreReader(DownloadCloudStorage fileStreamProvider, StoreScope scope, Parser<T> parser, String dataTypeName) {
         this.metadataStreamProvider = fileStreamProvider;
@@ -60,7 +60,7 @@ public JsonObject getMetadata() throws Exception {
         }
     }
 
-    private long loadContent(String path) throws Exception {
+    protected long loadContent(String path) throws Exception {
         try (InputStream inputStream = this.contentStreamProvider.download(path)) {
             ParsingResult<T> parsed = parser.deserialize(inputStream);
             latestSnapshot.set(parsed.getData());

src/main/java/com/uid2/shared/store/reader/RotatingKeyAclProvider.java

@@ -5,9 +5,11 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.cloud.ICloudStorage;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.IKeyAclProvider;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeyAclParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -21,6 +23,10 @@ public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls");
     }
 
+    public RotatingKeyAclProvider(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader =  new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyAclParser(), "key acls", s3KeyProvider);
+    }
+
     @Override
     public CloudPath getMetadataPath() { return reader.getMetadataPath(); }
 

src/main/java/com/uid2/shared/store/reader/RotatingKeyStore.java

@@ -3,9 +3,11 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.EncryptionKey;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.IKeyStore;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeyParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -50,6 +52,10 @@ public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, StoreScope scop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys");
     }
 
+    public RotatingKeyStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeyParser(), "keys", s3KeyProvider);
+    }
+
     @Override
     public CloudPath getMetadataPath() {
         return reader.getMetadataPath();

src/main/java/com/uid2/shared/store/reader/RotatingKeysetKeyStore.java

@@ -2,11 +2,9 @@
 
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.KeysetKey;
-import com.uid2.shared.store.CloudPath;
-import com.uid2.shared.store.IKeysetKeyStore;
-import com.uid2.shared.store.KeysetKeyStoreSnapshot;
-import com.uid2.shared.store.ScopedStoreReader;
+import com.uid2.shared.store.*;
 import com.uid2.shared.store.parser.KeysetKeyParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -20,6 +18,10 @@ public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys");
     }
 
+    public RotatingKeysetKeyStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new KeysetKeyParser(), "keyset_keys", s3KeyProvider);
+    }
+
     @Override
     public long getVersion(JsonObject metadata) {
         return metadata.getLong("version");

src/main/java/com/uid2/shared/store/reader/RotatingKeysetProvider.java

@@ -4,8 +4,10 @@
 import com.uid2.shared.auth.KeysetSnapshot;
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.KeysetParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -19,6 +21,10 @@ public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, StoreScop
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new KeysetParser(), "keysets");
     }
 
+    public RotatingKeysetProvider(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider,scope,new KeysetParser(),"keysers",s3KeyProvider);
+    }
+
     public KeysetSnapshot getSnapshot(Instant asOf) {
         return reader.getSnapshot();
     }

src/main/java/com/uid2/shared/store/reader/RotatingS3KeyProvider.java

@@ -1,6 +1,7 @@
 package com.uid2.shared.store.reader;
 
 import com.uid2.shared.auth.KeysetSnapshot;
+import com.uid2.shared.auth.RotatingOperatorKeyProvider;
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.ScopedStoreReader;
@@ -9,13 +10,19 @@
 import io.vertx.core.json.JsonObject;
 
 import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
 import com.uid2.shared.model.S3Key;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class RotatingS3KeyProvider implements StoreReader<Map<Integer, S3Key>> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(RotatingOperatorKeyProvider.class);
+
     ScopedStoreReader<Map<Integer, S3Key>> reader;
+    private final Map<Integer, List<S3Key>> siteToKeysMap = new HashMap<>();
+
 
     public RotatingS3KeyProvider(DownloadCloudStorage fileStreamProvider, StoreScope scope) {
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new S3KeyParser(), "s3encryption_keys");
@@ -38,7 +45,18 @@ public long getVersion(JsonObject metadata) {
 
     @Override
     public long loadContent(JsonObject metadata) throws Exception {
-        return reader.loadContent(metadata, "s3encryption_keys");
+        long loadedKeysCount = reader.loadContent(metadata, "s3encryption_keys");
+        LOGGER.info("Loaded {} S3 encryption keys", loadedKeysCount);
+        return loadedKeysCount;
+    }
+
+    public void updateSiteToKeysMapping() {
+        Map<Integer, S3Key> allKeys = getAll();
+        siteToKeysMap.clear();
+        for (S3Key key : allKeys.values()) {
+            siteToKeysMap.computeIfAbsent(key.getSiteId(), k -> new ArrayList<>()).add(key);
+        }
+        LOGGER.info("Updated site-to-keys mapping for {} sites", siteToKeysMap.size());
     }
 
     @Override
@@ -47,9 +65,41 @@ public Map<Integer, S3Key> getAll() {
         return keys != null ? keys : new HashMap<>();
     }
 
-
     @Override
     public void loadContent() throws Exception {
         this.loadContent(this.getMetadata());
     }
+
+    public Collection<S3Key> getKeysForSite(Integer siteId) {
+        Map<Integer, S3Key> allKeys = getAll();
+        return allKeys.values().stream()
+                .filter(key -> key.getSiteId()==(siteId))
+                .collect(Collectors.toList());
+    }
+
+    public S3Key getEncryptionKeyForSite(Integer siteId) {
+        Collection<S3Key> keys = getKeysForSite(siteId);
+        if (keys.isEmpty()) {
+            throw new IllegalStateException("No S3 keys available for encryption for site ID: " + siteId);
+        } else {
+            Map<Integer, S3Key> allKeys = getAll();
+            S3Key largestKey = null;
+            for (S3Key key : allKeys.values()) {
+                if (key.getSiteId() == siteId) {
+                    if (largestKey == null || key.getId() > largestKey.getId()) {
+                        largestKey = key;
+                    }
+                }
+            }
+            return largestKey;
+        }
+    }
+
+    public Set<Integer> getAllSiteIds() {
+        return new HashSet<>(siteToKeysMap.keySet());
+    }
+
+    public int getTotalSites() {
+        return siteToKeysMap.size();
+    }
 }

src/main/java/com/uid2/shared/store/reader/RotatingSiteStore.java

@@ -3,9 +3,11 @@
 import com.uid2.shared.cloud.DownloadCloudStorage;
 import com.uid2.shared.model.Site;
 import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedScopedStoreReader;
 import com.uid2.shared.store.ISiteStore;
 import com.uid2.shared.store.ScopedStoreReader;
 import com.uid2.shared.store.parser.SiteParser;
+import com.uid2.shared.store.scope.EncryptedScope;
 import com.uid2.shared.store.scope.StoreScope;
 import io.vertx.core.json.JsonObject;
 
@@ -21,6 +23,10 @@ public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, StoreScope sco
         this.reader = new ScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites");
     }
 
+    public RotatingSiteStore(DownloadCloudStorage fileStreamProvider, EncryptedScope scope, RotatingS3KeyProvider s3KeyProvider) {
+        this.reader = new EncryptedScopedStoreReader<>(fileStreamProvider, scope, new SiteParser(), "sites", s3KeyProvider);
+    }
+
     @Override
     public CloudPath getMetadataPath() { return reader.getMetadataPath(); }
 

src/main/java/com/uid2/shared/store/scope/EncryptedScope.java

@@ -0,0 +1,46 @@
+
+package com.uid2.shared.store.scope;
+
+
+import com.uid2.shared.store.CloudPath;
+
+public class EncryptedScope implements StoreScope {
+
+    public EncryptedScope(CloudPath rootMetadataPath, Integer siteId, boolean isPublic){
+        this.rootMetadataPath = rootMetadataPath;
+        this.siteId = siteId;
+        this.isPublic = isPublic;
+    }
+    private final Integer siteId;
+    private final CloudPath rootMetadataPath;
+    private final boolean isPublic;
+
+    @Override
+    public CloudPath getMetadataPath() {
+        return resolve(rootMetadataPath.getFileName());
+    }
+
+    @Override
+    public CloudPath resolve(CloudPath cloudPath) {
+        CloudPath directory = rootMetadataPath.getParent();
+        String siteType = isPublic ? "public" : "private";
+        return directory.resolve("encrypted").resolve(siteId + "_" + siteType).resolve(cloudPath);
+    }
+
+    @Override
+    public Integer getId() {
+        return siteId;
+    }
+
+    public boolean getPublic() {
+        return isPublic;
+    }
+
+    /* paths when we no longer need to distinguish private & public sites
+    @Override
+    public CloudPath resolve(CloudPath cloudPath) {
+        CloudPath directory = rootMetadataPath.getParent();
+        return directory.resolve("encryption").resolve("site").resolve(getId().toString()).resolve(cloudPath);
+    }
+    */
+}