diff --git a/Dockerfile b/Dockerfile
index d30085c1..aec568a9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -26,4 +26,5 @@ CMD java \
-Djava.security.egd=file:/dev/./urandom \
-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \
-Dlogback.configurationFile=${LOGBACK_CONF} \
+ -Xmx4g \
-jar ${JAR_NAME}-${JAR_VERSION}.jar
diff --git a/pom.xml b/pom.xml
index 479eb475..85f51a9a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
com.uid2
uid2-admin
- 5.16.0
+ 5.16.8-alpha-103-SNAPSHOT
UTF-8
@@ -16,7 +16,7 @@
1.12.2
5.11.2
- 8.0.6
+ 8.0.25
0.5.10
${project.version}
diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java
index 9febe5f7..8e81705c 100644
--- a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java
+++ b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java
@@ -33,7 +33,7 @@ public String getId() {
public void execute() throws Exception {
PrivateSiteDataMap desiredPrivateState = PrivateSiteUtil.getClientKeys(globalOperators, globalClientKeys);
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
- PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys,globalOperators);
+ PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys, globalOperators);
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
}
}
diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java
new file mode 100644
index 00000000..98d43dd0
--- /dev/null
+++ b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java
@@ -0,0 +1,36 @@
+package com.uid2.admin.job.EncryptionJob;
+
+import com.uid2.admin.job.model.Job;
+import com.uid2.admin.model.PrivateSiteDataMap;
+import com.uid2.admin.store.MultiScopeStoreWriter;
+import com.uid2.admin.util.PublicSiteUtil;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.ClientSideKeypair;
+
+import java.util.Collection;
+
+public class ClientSideKeypairEncryptionJob extends Job {
+ private final Collection globalOperators;
+ private final Collection globalClientSideKeypairs;
+
+ private final MultiScopeStoreWriter> multiScopeStoreWriter;
+
+ public ClientSideKeypairEncryptionJob(Collection globalOperators, Collection globalClientSideKeypairs,
+ MultiScopeStoreWriter> multiScopeStoreWriter) {
+ this.globalOperators = globalOperators;
+ this.globalClientSideKeypairs = globalClientSideKeypairs;
+ this.multiScopeStoreWriter = multiScopeStoreWriter;
+ }
+
+ @Override
+ public String getId() {
+ return "cloud-encryption-sync-clientside-keypair";
+ }
+
+ @Override
+ public void execute() throws Exception {
+ // Only public operators support clientside keypair
+ PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators);
+ multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
+ }
+}
diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java
new file mode 100644
index 00000000..4ffb7b11
--- /dev/null
+++ b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java
@@ -0,0 +1,41 @@
+package com.uid2.admin.job.EncryptionJob;
+
+import com.uid2.admin.job.model.Job;
+import com.uid2.admin.model.PrivateSiteDataMap;
+import com.uid2.admin.store.MultiScopeStoreWriter;
+import com.uid2.admin.util.PrivateSiteUtil;
+import com.uid2.admin.util.PublicSiteUtil;
+import com.uid2.shared.auth.OperatorKey;
+import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.RotatingSaltProvider;
+
+import java.util.Collection;
+import java.util.List;
+
+public class SaltEncryptionJob extends Job {
+ private final Collection globalOperators;
+ private final Collection saltEntries;
+ private final MultiScopeStoreWriter> multiScopeStoreWriter;
+
+ public SaltEncryptionJob(Collection globalOperators,
+ Collection saltEntries,
+ MultiScopeStoreWriter> multiScopeStoreWriter) {
+ this.globalOperators = globalOperators;
+ this.saltEntries = saltEntries;
+ this.multiScopeStoreWriter = multiScopeStoreWriter;
+ }
+
+
+ @Override
+ public String getId() {
+ return "cloud-encryption-sync-salts";
+ }
+
+ @Override
+ public void execute() throws Exception {
+ List desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
+ multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
+ List desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
+ multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
+ }
+}
diff --git a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
index 14077ac3..186809d4 100644
--- a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
+++ b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
@@ -2,7 +2,6 @@
import com.fasterxml.jackson.databind.ObjectWriter;
import com.uid2.admin.job.EncryptionJob.*;
-import com.uid2.admin.job.EncryptionJob.ClientKeyEncryptionJob;
import com.uid2.admin.job.model.Job;
import com.uid2.admin.store.*;
import com.uid2.admin.store.factory.*;
@@ -17,12 +16,15 @@
import com.uid2.shared.auth.RotatingOperatorKeyProvider;
import com.uid2.shared.cloud.CloudUtils;
import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.cloud.TaggableCloudStorage;
+import com.uid2.shared.model.ClientSideKeypair;
import com.uid2.shared.model.EncryptionKey;
import com.uid2.shared.model.KeysetKey;
import com.uid2.shared.model.Site;
import com.uid2.shared.store.CloudPath;
import com.uid2.admin.legacy.LegacyClientKey;
-import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.EncryptedRotatingSaltProvider;
+import com.uid2.shared.store.RotatingSaltProvider;
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
import com.uid2.shared.store.scope.GlobalScope;
import io.vertx.core.json.JsonObject;
@@ -34,12 +36,12 @@
public class EncryptedFilesSyncJob extends Job {
private final JsonObject config;
private final WriteLock writeLock;
- private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
+ private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
this.config = config;
this.writeLock = writeLock;
- this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
+ this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
}
@Override
@@ -49,20 +51,22 @@ public String getId() {
@Override
public void execute() throws Exception {
- ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
+ TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
FileStorage fileStorage = new TmpFileStorage();
ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
Clock clock = new InstantClock();
VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
FileManager fileManager = new FileManager(cloudStorage, fileStorage);
+ RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));
+
SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
cloudStorage,
new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
jsonWriter,
versionGenerator,
clock,
- RotatingCloudEncryptionKeyProvider,
+ rotatingCloudEncryptionKeyProvider,
fileManager);
ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
@@ -71,7 +75,7 @@ public void execute() throws Exception {
jsonWriter,
versionGenerator,
clock,
- RotatingCloudEncryptionKeyProvider,
+ rotatingCloudEncryptionKeyProvider,
fileManager);
EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
@@ -79,7 +83,7 @@ public void execute() throws Exception {
new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
versionGenerator,
clock,
- RotatingCloudEncryptionKeyProvider,
+ rotatingCloudEncryptionKeyProvider,
fileManager);
KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
@@ -88,7 +92,7 @@ public void execute() throws Exception {
jsonWriter,
versionGenerator,
clock,
- RotatingCloudEncryptionKeyProvider,
+ rotatingCloudEncryptionKeyProvider,
fileManager);
KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
@@ -98,7 +102,7 @@ public void execute() throws Exception {
versionGenerator,
clock,
fileManager,
- RotatingCloudEncryptionKeyProvider,
+ rotatingCloudEncryptionKeyProvider,
config.getBoolean(enableKeysetConfigProp));
KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
@@ -107,15 +111,33 @@ public void execute() throws Exception {
versionGenerator,
clock,
fileManager,
- RotatingCloudEncryptionKeyProvider,
+ rotatingCloudEncryptionKeyProvider,
config.getBoolean(enableKeysetConfigProp));
+ SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
+ config,
+ new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
+ fileManager,
+ cloudStorage,
+ versionGenerator,
+ rotatingCloudEncryptionKeyProvider
+ );
+
+ ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory(
+ cloudStorage,
+ new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)),
+ versionGenerator,
+ clock,
+ rotatingCloudEncryptionKeyProvider,
+ fileManager
+ );
+
CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
synchronized (writeLock) {
- RotatingCloudEncryptionKeyProvider.loadContent();
+ rotatingCloudEncryptionKeyProvider.loadContent();
operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
clientKeyStoreFactory.getGlobalReader().loadContent();
@@ -125,13 +147,18 @@ public void execute() throws Exception {
keysetStoreFactory.getGlobalReader().loadContent();
keysetKeyStoreFactory.getGlobalReader().loadContent();
}
+ saltProvider.loadContent();
+ clientSideKeypairStoreFactory.getGlobalReader().loadContent();
}
+
Collection globalOperators = operatorKeyProvider.getAll();
Collection globalSites = siteStoreFactory.getGlobalReader().getAllSites();
Collection globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
Collection globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet();
Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id");
Map globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls();
+ Collection globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll();
+
MultiScopeStoreWriter> siteWriter = new MultiScopeStoreWriter<>(
fileManager,
siteStoreFactory,
@@ -148,6 +175,14 @@ public void execute() throws Exception {
fileManager,
keyAclStoreFactory,
MultiScopeStoreWriter::areMapsEqual);
+ MultiScopeStoreWriter> saltWriter = new MultiScopeStoreWriter<>(
+ fileManager,
+ saltStoreFactory,
+ MultiScopeStoreWriter::areCollectionsEqual);
+ MultiScopeStoreWriter> clientSideKeypairWriter = new MultiScopeStoreWriter<>(
+ fileManager,
+ clientSideKeypairStoreFactory,
+ MultiScopeStoreWriter::areCollectionsEqual);
SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
@@ -160,10 +195,15 @@ public void execute() throws Exception {
encryptionKeyWriter
);
KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
+ SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
+ ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);
+
siteEncryptionSyncJob.execute();
clientEncryptionSyncJob.execute();
encryptionKeyEncryptionSyncJob.execute();
keyAclEncryptionSyncJob.execute();
+ saltEncryptionJob.execute();
+ clientSideKeypairEncryptionJob.execute();
if(config.getBoolean(enableKeysetConfigProp)) {
Map globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();
diff --git a/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java b/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java
index 879b4e12..7c2205ca 100644
--- a/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java
+++ b/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java
@@ -69,6 +69,13 @@ public void uploadPrivateWithEncryption(Map desiredState, JsonObject
}
}
+ public void uploadPrivateWithEncryption(List siteIds, T desiredState, JsonObject extraMeta) throws Exception {
+ EncryptedStoreFactory encryptedFactory = (EncryptedStoreFactory) factory;
+ for (Integer siteId : siteIds) {
+ encryptedFactory.getEncryptedWriter(siteId,false).upload(desiredState, extraMeta);
+ }
+ }
+
public void uploadPublicWithEncryption(Map desiredPublicState, JsonObject extraMeta) throws Exception {
EncryptedStoreFactory encryptedFactory = (EncryptedStoreFactory) factory;
for (Map.Entry entry : desiredPublicState.entrySet()) {
@@ -77,6 +84,13 @@ public void uploadPublicWithEncryption(Map desiredPublicState, JsonO
}
}
+ public void uploadPublicWithEncryption(List siteIds, T desiredState, JsonObject extraMeta) throws Exception {
+ EncryptedStoreFactory encryptedFactory = (EncryptedStoreFactory) factory;
+ for (Integer siteId : siteIds) {
+ encryptedFactory.getEncryptedWriter(siteId,true).upload(desiredState, extraMeta);
+ }
+ }
+
public static boolean areMapsEqual(Map a, Map b) {
return a.size() == b.size() && a.entrySet().stream().allMatch(b.entrySet()::contains);
}
diff --git a/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java
new file mode 100644
index 00000000..e3998741
--- /dev/null
+++ b/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java
@@ -0,0 +1,80 @@
+package com.uid2.admin.store.factory;
+
+import com.uid2.admin.store.writer.ClientSideKeypairStoreWriter;
+import com.uid2.admin.store.writer.StoreWriter;
+import com.uid2.shared.model.ClientSideKeypair;
+import com.uid2.shared.store.reader.RotatingClientSideKeypairStore;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.reader.StoreReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.uid2.admin.store.Clock;
+import com.uid2.admin.store.FileManager;
+import com.uid2.admin.store.version.VersionGenerator;
+import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.scope.EncryptedScope;
+import com.uid2.shared.store.scope.GlobalScope;
+import com.uid2.shared.store.scope.SiteScope;
+
+import java.util.Collection;
+
+public class ClientSideKeypairStoreFactory implements EncryptedStoreFactory> {
+ private final ICloudStorage fileStreamProvider;
+ private final CloudPath rootMetadataPath;
+ private final VersionGenerator versionGenerator;
+ private final Clock clock;
+ private final FileManager fileManager;
+ private final RotatingClientSideKeypairStore globalReader;
+ private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
+
+ public ClientSideKeypairStoreFactory(
+ ICloudStorage fileStreamProvider,
+ CloudPath rootMetadataPath,
+ VersionGenerator versionGenerator,
+ Clock clock,
+ RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider,
+ FileManager fileManager) {
+ this.fileStreamProvider = fileStreamProvider;
+ this.rootMetadataPath = rootMetadataPath;
+ this.versionGenerator = versionGenerator;
+ this.clock = clock;
+ this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
+ this.fileManager = fileManager;
+ GlobalScope globalScope = new GlobalScope(rootMetadataPath);
+ globalReader = new RotatingClientSideKeypairStore(fileStreamProvider, globalScope);
+ }
+
+ public RotatingClientSideKeypairStore getGlobalReader() {
+ return globalReader;
+ }
+
+ @Override
+ public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) {
+ return new ClientSideKeypairStoreWriter(getEncryptedReader(siteId, isPublic),
+ fileManager,
+ versionGenerator,
+ clock,
+ new EncryptedScope(rootMetadataPath, siteId, isPublic),
+ cloudEncryptionKeyProvider);
+ }
+
+ @Override
+ public StoreReader> getEncryptedReader(Integer siteId, boolean isPublic) {
+ return new RotatingClientSideKeypairStore(fileStreamProvider, new EncryptedScope(rootMetadataPath, siteId, isPublic), cloudEncryptionKeyProvider);
+ }
+
+ @Override
+ public RotatingCloudEncryptionKeyProvider getCloudEncryptionProvider() {
+ return cloudEncryptionKeyProvider;
+ }
+
+ @Override
+ public StoreReader> getReader(Integer siteId) {
+ return new RotatingClientSideKeypairStore(fileStreamProvider, new SiteScope(rootMetadataPath, siteId));
+ }
+
+ @Override
+ public StoreWriter> getWriter(Integer siteId) {
+ return new ClientSideKeypairStoreWriter(getReader(siteId), fileManager, versionGenerator, clock, new SiteScope(rootMetadataPath, siteId));
+ }
+}
diff --git a/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java
new file mode 100644
index 00000000..8261e942
--- /dev/null
+++ b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java
@@ -0,0 +1,71 @@
+package com.uid2.admin.store.factory;
+
+import com.uid2.admin.store.FileManager;
+import com.uid2.admin.store.version.VersionGenerator;
+import com.uid2.admin.store.writer.EncryptedSaltStoreWriter;
+import com.uid2.admin.store.writer.StoreWriter;
+import com.uid2.shared.Const;
+import com.uid2.shared.cloud.TaggableCloudStorage;
+import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.EncryptedRotatingSaltProvider;
+import com.uid2.shared.store.RotatingSaltProvider;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.reader.StoreReader;
+import com.uid2.shared.store.scope.EncryptedScope;
+import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+public class SaltStoreFactory implements EncryptedStoreFactory> {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SaltStoreFactory.class);
+
+ JsonObject config;
+ CloudPath rootMetadatapath;
+ FileManager fileManager;
+ TaggableCloudStorage taggableCloudStorage;
+ VersionGenerator versionGenerator;
+ RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
+
+ public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, FileManager fileManager,
+ TaggableCloudStorage taggableCloudStorage, VersionGenerator versionGenerator,
+ RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+ this.config = config;
+ this.rootMetadatapath = rootMetadataPath;
+ this.fileManager = fileManager;
+ this.taggableCloudStorage = taggableCloudStorage;
+ this.versionGenerator = versionGenerator;
+ this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
+ }
+
+ @Override
+ public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) {
+ EncryptedScope scope = new EncryptedScope(rootMetadatapath, siteId, isPublic);
+ EncryptedRotatingSaltProvider saltProvider = new EncryptedRotatingSaltProvider(taggableCloudStorage, cloudEncryptionKeyProvider, scope);
+ return new EncryptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, scope, cloudEncryptionKeyProvider, siteId);
+ }
+
+ @Override
+ public StoreReader> getEncryptedReader(Integer siteId, boolean isPublic) {
+ LOGGER.warn("getEncryptedReader called on SaltStoreFactory. This method is not implemented.");
+ return null;
+ }
+
+ @Override
+ public RotatingCloudEncryptionKeyProvider getCloudEncryptionProvider() {
+ return cloudEncryptionKeyProvider;
+ }
+
+ @Override
+ public StoreReader> getReader(Integer siteId) {
+ LOGGER.warn("getReader called on SaltStoreFactory. This method is not implemented.");
+ return null;
+ }
+
+ @Override
+ public StoreWriter> getWriter(Integer siteId) {
+ LOGGER.warn("getWriter called on SaltStoreFactory. This method is not implemented.");
+ return null;
+ }
+}
diff --git a/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java
index fe48e533..78c0a959 100644
--- a/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java
+++ b/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java
@@ -6,6 +6,8 @@
import com.uid2.admin.store.version.VersionGenerator;
import com.uid2.shared.model.ClientSideKeypair;
import com.uid2.shared.store.reader.RotatingClientSideKeypairStore;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.reader.StoreReader;
import com.uid2.shared.store.scope.StoreScope;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
@@ -17,12 +19,19 @@ public class ClientSideKeypairStoreWriter implements StoreWriter> store, FileManager fileManager, VersionGenerator versionGenerator, Clock clock, StoreScope scope) {
FileName dataFile = new FileName("client_side_keypairs", ".json");
String dataType = "client_side_keypairs";
writer = new ScopedStoreWriter(store, fileManager, versionGenerator, clock, scope, dataFile, dataType);
}
+ public ClientSideKeypairStoreWriter(StoreReader> store, FileManager fileManager,
+ VersionGenerator versionGenerator, Clock clock, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
+ FileName dataFile = new FileName("client_side_keypairs", ".json");
+ String dataType = "client_side_keypairs";
+ writer = new EncryptedScopedStoreWriter(store, fileManager, versionGenerator, clock, scope, dataFile, dataType, cloudEncryptionKeyProvider, scope.getId());
+ }
+
@Override
public void upload(Collection data, JsonObject extraMeta) throws Exception {
JsonArray jsonKeypairs = new JsonArray();
diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java
new file mode 100644
index 00000000..775b29b8
--- /dev/null
+++ b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java
@@ -0,0 +1,105 @@
+package com.uid2.admin.store.writer;
+
+import com.uid2.admin.store.FileManager;
+import com.uid2.admin.store.version.VersionGenerator;
+import com.uid2.shared.cloud.TaggableCloudStorage;
+import com.uid2.shared.encryption.AesGcm;
+import com.uid2.shared.model.CloudEncryptionKey;
+import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.RotatingSaltProvider;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.scope.StoreScope;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import io.vertx.core.json.JsonObject;
+
+import java.io.BufferedWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Base64;
+import java.util.Collection;
+
+public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter {
+ private StoreScope scope;
+ private RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
+ private Integer siteId;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedSaltStoreWriter.class);
+ public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager,
+ TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator, StoreScope scope,
+ RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, Integer siteId) {
+ super(config, provider, fileManager, cloudStorage, versionGenerator);
+ this.scope = scope;
+ this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider;
+ this.siteId = siteId;
+ }
+
+ @Override
+ protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
+ return scope.resolve(new CloudPath("salts.txt." + snapshot.getEffective().toEpochMilli())).toString();
+ }
+
+ @Override
+ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
+ if (siteId == null) {
+ throw new IllegalStateException("Site ID is not set.");
+ }
+
+ if (!cloudStorage.list(location).isEmpty()) {
+ // update the tags on the file to ensure it is still marked as current
+ this.setStatusTagToCurrent(location);
+ return;
+ }
+
+ StringBuilder stringBuilder = new StringBuilder();
+
+ for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
+ stringBuilder.append(entry.getId()).append(",").append(entry.getLastUpdated()).append(",").append(entry.getSalt()).append("\n");
+ }
+
+ String data = stringBuilder.toString();
+
+ CloudEncryptionKey encryptionKey = null;
+ try {
+ encryptionKey = cloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId);
+ } catch (IllegalStateException e) {
+ LOGGER.error("Error: No Cloud Encryption keys available for encryption for site ID: {}", siteId, e);
+ }
+ JsonObject encryptedJson = new JsonObject();
+ if (encryptionKey != null) {
+ byte[] secret = Base64.getDecoder().decode(encryptionKey.getSecret());
+ byte[] encryptedPayload = AesGcm.encrypt(data.getBytes(StandardCharsets.UTF_8), secret);
+ encryptedJson.put("key_id", encryptionKey.getId())
+ .put("encryption_version", "1.0")
+ .put("encrypted_payload", Base64.getEncoder().encodeToString(encryptedPayload));
+ } else {
+ throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId);
+ }
+
+ final Path newSaltsFile = Files.createTempFile("salts", ".txt");
+ try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
+ w.write(encryptedJson.encodePrettily());
+ }
+
+ this.upload(newSaltsFile.toString(), location);
+ }
+
+ @Override
+ protected void refreshProvider() {
+ // we do not need to refresh the provider on encrypted writers
+ }
+
+ @Override
+ public void upload(Object data, JsonObject extraMeta) throws Exception {
+ for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection) data) {
+ super.upload(saltSnapshot);
+ }
+ }
+
+ @Override
+ public void rewriteMeta() throws Exception {
+
+ }
+}
diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java
index c2718c1e..57f82aec 100644
--- a/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java
+++ b/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java
@@ -31,6 +31,7 @@ public EncryptedScopedStoreWriter(IMetadataVersionedStore provider,
this.siteId = siteId;
}
+ @Override
public void upload(String data, JsonObject extraMeta) throws Exception {
if (siteId == null) {
throw new IllegalStateException("Site ID is not set.");
diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
index 76626539..f8bfc2a5 100644
--- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
+++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java
@@ -16,6 +16,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
+import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -26,10 +27,10 @@ public class SaltStoreWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(SaltStoreWriter.class);
private final RotatingSaltProvider provider;
private final FileManager fileManager;
- private final String saltSnapshotLocationPrefix;
+ protected final String saltSnapshotLocationPrefix;
private final VersionGenerator versionGenerator;
- private final TaggableCloudStorage cloudStorage;
+ protected final TaggableCloudStorage cloudStorage;
private final Map currentTags = Map.of("status", "current");
private final Map obsoleteTags = Map.of("status", "obsolete");
@@ -46,7 +47,16 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
final Instant now = Instant.now();
final long generated = now.getEpochSecond();
- final JsonObject metadata = provider.getMetadata();
+ JsonObject metadata = null;
+ try {
+ metadata = provider.getMetadata();
+ } catch (CloudStorageException e) {
+ if (e.getMessage().contains("The specified key does not exist")) {
+ metadata = new JsonObject();
+ } else {
+ throw e;
+ }
+ }
// bump up metadata version
metadata.put("version", versionGenerator.getVersion());
metadata.put("generated", generated);
@@ -54,9 +64,16 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
final JsonArray snapshotsMetadata = new JsonArray();
metadata.put("salts", snapshotsMetadata);
- final List snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data))
- .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective))
- .collect(Collectors.toList());
+ List currentSnapshots = provider.getSnapshots();
+ List snapshots = null;
+
+ if (currentSnapshots != null) {
+ snapshots = Stream.concat(currentSnapshots.stream(), Stream.of(data))
+ .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective))
+ .collect(Collectors.toList());
+ } else {
+ snapshots = List.of(data);
+ }
// of the currently effective snapshots keep only the most recent one
RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = snapshots.stream()
.filter(snapshot -> snapshot.isEffective(now))
@@ -89,6 +106,10 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath()));
// refresh manually
+ refreshProvider();
+ }
+
+ protected void refreshProvider() throws Exception {
provider.loadContent();
}
@@ -111,11 +132,11 @@ public void archiveSaltLocations() throws Exception {
});
}
- private String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
+ protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) {
return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
}
- private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
+ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
// do not overwrite existing files
if (!cloudStorage.list(location).isEmpty()) {
// update the tags on the file to ensure it is still marked as current
@@ -130,10 +151,15 @@ private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, Str
}
}
- cloudStorage.upload(newSaltsFile.toString(), location, this.currentTags);
+ this.upload(newSaltsFile.toString(), location);
+ }
+
+ protected void upload(String data, String location) throws Exception {
+ cloudStorage.upload(data, location, this.currentTags);
+
}
- private void setStatusTagToCurrent(String location) throws CloudStorageException {
+ protected void setStatusTagToCurrent(String location) throws CloudStorageException {
this.cloudStorage.setTags(location, this.currentTags);
}
diff --git a/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java b/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java
index f73541a9..525b9494 100644
--- a/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java
+++ b/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java
@@ -6,7 +6,9 @@
import com.uid2.shared.auth.*;
import com.uid2.shared.model.EncryptionKey;
import com.uid2.shared.model.KeysetKey;
+import com.uid2.shared.model.SaltEntry;
import com.uid2.shared.model.Site;
+import com.uid2.shared.store.RotatingSaltProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -310,4 +312,9 @@ public static PrivateSiteDataMap getKeysetKeys(Collection getPrivateSaltSites(Collection operators) {
+ final PrivateSiteDataMap result = getPrivateSites(operators);
+ return result.keySet().stream().toList();
+ }
}
diff --git a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java
index bc55bc14..815454b4 100644
--- a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java
+++ b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java
@@ -6,16 +6,12 @@
import com.uid2.shared.auth.Keyset;
import com.uid2.shared.auth.OperatorKey;
import com.uid2.shared.auth.OperatorType;
-import com.uid2.shared.model.EncryptionKey;
-import com.uid2.shared.model.KeysetKey;
-import com.uid2.shared.model.Site;
+import com.uid2.shared.model.*;
+import com.uid2.shared.store.RotatingSaltProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.*;
public class PublicSiteUtil {
private static final Logger LOGGER = LoggerFactory.getLogger(PrivateSiteUtil.class);
@@ -128,4 +124,23 @@ public static PrivateSiteDataMap getPublicKeysetKeys(
return result;
}
+
+ public static List getPublicSaltSites(Collection operators) {
+ final PrivateSiteDataMap result = getPublicSitesMap(operators);
+ return result.keySet().stream().toList();
+ }
+
+ public static PrivateSiteDataMap getPublicClientKeypairs(
+ Collection globalClientSideKeypair,
+ Collection operators) {
+ final PrivateSiteDataMap result = getPublicSitesMap(operators);
+
+ globalClientSideKeypair.forEach(clientSideKeypair -> {
+ result.forEach((publicSiteId, publicSiteData) -> {
+ publicSiteData.add(clientSideKeypair);
+ });
+ });
+
+ return result;
+ }
}
diff --git a/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java b/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java
index 8f6adec9..4910c300 100644
--- a/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java
+++ b/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java
@@ -109,6 +109,10 @@ public void overwritesExistingDataWhenChanged() throws Exception {
reader.loadContent();
Long oldVersion = reader.getMetadata().getLong("version");
+ // This test relies on our version generator returning a new timestamp, but the code can execute so fast we don't get a new version
+ // This small sleep makes this test much more stable
+ Thread.sleep(100);
+
Site updatedSite = new Site(scopedSiteId, "site 1 updated", true);
MultiScopeStoreWriter> multiStore = new MultiScopeStoreWriter<>(fileManager, siteStoreFactory, MultiScopeStoreWriter::areCollectionsEqual);
@@ -204,6 +208,7 @@ public void uploadPrivateWithEncryption() throws Exception {
Map allKeys = new HashMap<>();
allKeys.put(1, encryptionKey);
when(cloudEncryptionKeyProvider.getAll()).thenReturn(allKeys);
+ when(cloudEncryptionKeyProvider.getKey(1)).thenReturn(encryptionKey);
SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
cloudStorage,
@@ -241,6 +246,7 @@ public void uploadPublicWithEncryption() throws Exception {
Map allKeys = new HashMap<>();
allKeys.put(1, encryptionKey);
when(cloudEncryptionKeyProvider.getAll()).thenReturn(allKeys);
+ when(cloudEncryptionKeyProvider.getKey(1)).thenReturn(encryptionKey);
SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
cloudStorage,
diff --git a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java
new file mode 100644
index 00000000..f7555106
--- /dev/null
+++ b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java
@@ -0,0 +1,127 @@
+package com.uid2.admin.store.writer;
+
+import com.uid2.admin.store.FileManager;
+import com.uid2.admin.store.version.VersionGenerator;
+import com.uid2.shared.cloud.CloudStorageException;
+import com.uid2.shared.cloud.TaggableCloudStorage;
+import com.uid2.shared.model.CloudEncryptionKey;
+import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.CloudPath;
+import com.uid2.shared.store.RotatingSaltProvider;
+import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
+import com.uid2.shared.store.scope.StoreScope;
+import io.vertx.core.json.JsonObject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.*;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.*;
+import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream;
+
+public class EncryptedSaltStoreWriterTest {
+ private AutoCloseable mocks;
+
+ @Mock
+ private FileManager fileManager;
+
+ @Mock
+ TaggableCloudStorage taggableCloudStorage;
+
+ @Mock
+ RotatingSaltProvider rotatingSaltProvider;
+
+ @Mock
+ RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
+
+ @Mock
+ VersionGenerator versionGenerator;
+
+ @Mock
+ StoreScope storeScope;
+
+ CloudEncryptionKey encryptionKey;
+
+ JsonObject config;
+
+ private final Integer siteId = 1;
+
+ @Captor
+ private ArgumentCaptor pathCaptor;
+ @Captor
+ private ArgumentCaptor cloudPathCaptor;
+
+ @BeforeEach
+ public void setUp() throws Exception {
+ mocks = MockitoAnnotations.openMocks(this);
+ config = new JsonObject();
+ config.put("salt_snapshot_location_prefix", "test");
+
+ when(versionGenerator.getVersion()).thenReturn(1L);
+ when(rotatingSaltProvider.getMetadataPath()).thenReturn("test/path/");
+ when(storeScope.resolve(any())).thenReturn(new CloudPath("test/path/"));
+
+ // Setup Cloud Encryption Keys
+ byte[] keyBytes = new byte[32];
+ new Random().nextBytes(keyBytes);
+ String base64Key = Base64.getEncoder().encodeToString(keyBytes);
+ encryptionKey = new CloudEncryptionKey(1, 1, 0, 0, base64Key);
+
+ Map mockKeyMap = new HashMap<>();
+ mockKeyMap.put(encryptionKey.getId(), encryptionKey);
+ when(rotatingCloudEncryptionKeyProvider.getAll()).thenReturn(mockKeyMap);
+ when(rotatingCloudEncryptionKeyProvider.getKey(1)).thenReturn(mockKeyMap.get(1));
+ when(rotatingCloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId)).thenReturn(encryptionKey);
+ }
+
+ private RotatingSaltProvider.SaltSnapshot makeSnapshot(Instant effective, Instant expires, int nsalts) {
+ SaltEntry[] entries = new SaltEntry[nsalts];
+ for (int i = 0; i < entries.length; ++i) {
+ entries[i] = new SaltEntry(i, "hashed_id", effective.toEpochMilli(), "salt");
+ }
+ return new RotatingSaltProvider.SaltSnapshot(effective, expires, entries, "test_first_level_salt");
+ }
+
+ private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot snapshot) throws IOException {
+ InputStream encoded = Files.newInputStream(Paths.get(filelocation));
+ String contents = decryptInputStream(encoded, rotatingCloudEncryptionKeyProvider);
+ SaltEntry[] entries = snapshot.getAllRotatingSalts();
+ Integer idx = 0;
+ for (String line : contents.split("\n")) {
+ String[] entrySplit = line.split(",");
+ assertEquals(entries[idx].getId(), Long.parseLong(entrySplit[0]));
+ assertEquals(entries[idx].getSalt(), entrySplit[2]);
+ idx++;
+ }
+ }
+
+ @Test
+ public void testUploadNew() throws Exception {
+ RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.now(), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 10000), 1000000);
+
+ when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket"));
+ when(rotatingSaltProvider.getSnapshots()).thenReturn(null);
+
+ when(taggableCloudStorage.list(anyString())).thenReturn(new ArrayList<>());
+
+ EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider,
+ fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId);
+
+ encryptedSaltStoreWriter.upload(snapshot);
+
+ verify(taggableCloudStorage).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any());
+ assertEquals(cloudPathCaptor.getValue(), "test/path");
+
+ verifyFile(pathCaptor.getValue(), snapshot);
+ }
+}