diff --git a/Dockerfile b/Dockerfile index d30085c1..aec568a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,4 +26,5 @@ CMD java \ -Djava.security.egd=file:/dev/./urandom \ -Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \ -Dlogback.configurationFile=${LOGBACK_CONF} \ + -Xmx4g \ -jar ${JAR_NAME}-${JAR_VERSION}.jar diff --git a/pom.xml b/pom.xml index 479eb475..85f51a9a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-admin - 5.16.0 + 5.16.8-alpha-103-SNAPSHOT UTF-8 @@ -16,7 +16,7 @@ 1.12.2 5.11.2 - 8.0.6 + 8.0.25 0.5.10 ${project.version} diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java index 9febe5f7..8e81705c 100644 --- a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java +++ b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientKeyEncryptionJob.java @@ -33,7 +33,7 @@ public String getId() { public void execute() throws Exception { PrivateSiteDataMap desiredPrivateState = PrivateSiteUtil.getClientKeys(globalOperators, globalClientKeys); multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null); - PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys,globalOperators); + PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys, globalOperators); multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null); } } diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java new file mode 100644 index 00000000..98d43dd0 --- /dev/null +++ b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java @@ -0,0 +1,36 @@ +package com.uid2.admin.job.EncryptionJob; + +import com.uid2.admin.job.model.Job; +import com.uid2.admin.model.PrivateSiteDataMap; +import com.uid2.admin.store.MultiScopeStoreWriter; +import com.uid2.admin.util.PublicSiteUtil; +import com.uid2.shared.auth.OperatorKey; +import com.uid2.shared.model.ClientSideKeypair; + +import java.util.Collection; + +public class ClientSideKeypairEncryptionJob extends Job { + private final Collection globalOperators; + private final Collection globalClientSideKeypairs; + + private final MultiScopeStoreWriter> multiScopeStoreWriter; + + public ClientSideKeypairEncryptionJob(Collection globalOperators, Collection globalClientSideKeypairs, + MultiScopeStoreWriter> multiScopeStoreWriter) { + this.globalOperators = globalOperators; + this.globalClientSideKeypairs = globalClientSideKeypairs; + this.multiScopeStoreWriter = multiScopeStoreWriter; + } + + @Override + public String getId() { + return "cloud-encryption-sync-clientside-keypair"; + } + + @Override + public void execute() throws Exception { + // Only public operators support clientside keypair + PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators); + multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null); + } +} diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java new file mode 100644 index 00000000..4ffb7b11 --- /dev/null +++ b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java @@ -0,0 +1,41 @@ +package com.uid2.admin.job.EncryptionJob; + +import com.uid2.admin.job.model.Job; +import com.uid2.admin.model.PrivateSiteDataMap; +import com.uid2.admin.store.MultiScopeStoreWriter; +import com.uid2.admin.util.PrivateSiteUtil; +import com.uid2.admin.util.PublicSiteUtil; +import com.uid2.shared.auth.OperatorKey; +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.store.RotatingSaltProvider; + +import java.util.Collection; +import java.util.List; + +public class SaltEncryptionJob extends Job { + private final Collection globalOperators; + private final Collection saltEntries; + private final MultiScopeStoreWriter> multiScopeStoreWriter; + + public SaltEncryptionJob(Collection globalOperators, + Collection saltEntries, + MultiScopeStoreWriter> multiScopeStoreWriter) { + this.globalOperators = globalOperators; + this.saltEntries = saltEntries; + this.multiScopeStoreWriter = multiScopeStoreWriter; + } + + + @Override + public String getId() { + return "cloud-encryption-sync-salts"; + } + + @Override + public void execute() throws Exception { + List desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators); + multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null); + List desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators); + multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null); + } +} diff --git a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java index 14077ac3..186809d4 100644 --- a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java +++ b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.uid2.admin.job.EncryptionJob.*; -import com.uid2.admin.job.EncryptionJob.ClientKeyEncryptionJob; import com.uid2.admin.job.model.Job; import com.uid2.admin.store.*; import com.uid2.admin.store.factory.*; @@ -17,12 +16,15 @@ import com.uid2.shared.auth.RotatingOperatorKeyProvider; import com.uid2.shared.cloud.CloudUtils; import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.cloud.TaggableCloudStorage; +import com.uid2.shared.model.ClientSideKeypair; import com.uid2.shared.model.EncryptionKey; import com.uid2.shared.model.KeysetKey; import com.uid2.shared.model.Site; import com.uid2.shared.store.CloudPath; import com.uid2.admin.legacy.LegacyClientKey; -import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.EncryptedRotatingSaltProvider; +import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.scope.GlobalScope; import io.vertx.core.json.JsonObject; @@ -34,12 +36,12 @@ public class EncryptedFilesSyncJob extends Job { private final JsonObject config; private final WriteLock writeLock; - private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider; + private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider; public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) { this.config = config; this.writeLock = writeLock; - this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider; + this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider; } @Override @@ -49,20 +51,22 @@ public String getId() { @Override public void execute() throws Exception { - ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config); + TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config); FileStorage fileStorage = new TmpFileStorage(); ObjectWriter jsonWriter = JsonUtil.createJsonWriter(); Clock clock = new InstantClock(); VersionGenerator versionGenerator = new EpochVersionGenerator(clock); FileManager fileManager = new FileManager(cloudStorage, fileStorage); + RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp)); + SiteStoreFactory siteStoreFactory = new SiteStoreFactory( cloudStorage, new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)), jsonWriter, versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory( @@ -71,7 +75,7 @@ public void execute() throws Exception { jsonWriter, versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory( @@ -79,7 +83,7 @@ public void execute() throws Exception { new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)), versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory( @@ -88,7 +92,7 @@ public void execute() throws Exception { jsonWriter, versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory( @@ -98,7 +102,7 @@ public void execute() throws Exception { versionGenerator, clock, fileManager, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, config.getBoolean(enableKeysetConfigProp)); KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory( @@ -107,15 +111,33 @@ public void execute() throws Exception { versionGenerator, clock, fileManager, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, config.getBoolean(enableKeysetConfigProp)); + SaltStoreFactory saltStoreFactory = new SaltStoreFactory( + config, + new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)), + fileManager, + cloudStorage, + versionGenerator, + rotatingCloudEncryptionKeyProvider + ); + + ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory( + cloudStorage, + new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)), + versionGenerator, + clock, + rotatingCloudEncryptionKeyProvider, + fileManager + ); + CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp)); GlobalScope operatorScope = new GlobalScope(operatorMetadataPath); RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope); synchronized (writeLock) { - RotatingCloudEncryptionKeyProvider.loadContent(); + rotatingCloudEncryptionKeyProvider.loadContent(); operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata()); siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata()); clientKeyStoreFactory.getGlobalReader().loadContent(); @@ -125,13 +147,18 @@ public void execute() throws Exception { keysetStoreFactory.getGlobalReader().loadContent(); keysetKeyStoreFactory.getGlobalReader().loadContent(); } + saltProvider.loadContent(); + clientSideKeypairStoreFactory.getGlobalReader().loadContent(); } + Collection globalOperators = operatorKeyProvider.getAll(); Collection globalSites = siteStoreFactory.getGlobalReader().getAllSites(); Collection globalClients = clientKeyStoreFactory.getGlobalReader().getAll(); Collection globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet(); Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id"); Map globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls(); + Collection globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll(); + MultiScopeStoreWriter> siteWriter = new MultiScopeStoreWriter<>( fileManager, siteStoreFactory, @@ -148,6 +175,14 @@ public void execute() throws Exception { fileManager, keyAclStoreFactory, MultiScopeStoreWriter::areMapsEqual); + MultiScopeStoreWriter> saltWriter = new MultiScopeStoreWriter<>( + fileManager, + saltStoreFactory, + MultiScopeStoreWriter::areCollectionsEqual); + MultiScopeStoreWriter> clientSideKeypairWriter = new MultiScopeStoreWriter<>( + fileManager, + clientSideKeypairStoreFactory, + MultiScopeStoreWriter::areCollectionsEqual); SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators); ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators); @@ -160,10 +195,15 @@ public void execute() throws Exception { encryptionKeyWriter ); KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls); + SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter); + ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter); + siteEncryptionSyncJob.execute(); clientEncryptionSyncJob.execute(); encryptionKeyEncryptionSyncJob.execute(); keyAclEncryptionSyncJob.execute(); + saltEncryptionJob.execute(); + clientSideKeypairEncryptionJob.execute(); if(config.getBoolean(enableKeysetConfigProp)) { Map globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets(); diff --git a/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java b/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java index 879b4e12..7c2205ca 100644 --- a/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java @@ -69,6 +69,13 @@ public void uploadPrivateWithEncryption(Map desiredState, JsonObject } } + public void uploadPrivateWithEncryption(List siteIds, T desiredState, JsonObject extraMeta) throws Exception { + EncryptedStoreFactory encryptedFactory = (EncryptedStoreFactory) factory; + for (Integer siteId : siteIds) { + encryptedFactory.getEncryptedWriter(siteId,false).upload(desiredState, extraMeta); + } + } + public void uploadPublicWithEncryption(Map desiredPublicState, JsonObject extraMeta) throws Exception { EncryptedStoreFactory encryptedFactory = (EncryptedStoreFactory) factory; for (Map.Entry entry : desiredPublicState.entrySet()) { @@ -77,6 +84,13 @@ public void uploadPublicWithEncryption(Map desiredPublicState, JsonO } } + public void uploadPublicWithEncryption(List siteIds, T desiredState, JsonObject extraMeta) throws Exception { + EncryptedStoreFactory encryptedFactory = (EncryptedStoreFactory) factory; + for (Integer siteId : siteIds) { + encryptedFactory.getEncryptedWriter(siteId,true).upload(desiredState, extraMeta); + } + } + public static boolean areMapsEqual(Map a, Map b) { return a.size() == b.size() && a.entrySet().stream().allMatch(b.entrySet()::contains); } diff --git a/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java new file mode 100644 index 00000000..e3998741 --- /dev/null +++ b/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java @@ -0,0 +1,80 @@ +package com.uid2.admin.store.factory; + +import com.uid2.admin.store.writer.ClientSideKeypairStoreWriter; +import com.uid2.admin.store.writer.StoreWriter; +import com.uid2.shared.model.ClientSideKeypair; +import com.uid2.shared.store.reader.RotatingClientSideKeypairStore; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.reader.StoreReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.uid2.admin.store.Clock; +import com.uid2.admin.store.FileManager; +import com.uid2.admin.store.version.VersionGenerator; +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.scope.EncryptedScope; +import com.uid2.shared.store.scope.GlobalScope; +import com.uid2.shared.store.scope.SiteScope; + +import java.util.Collection; + +public class ClientSideKeypairStoreFactory implements EncryptedStoreFactory> { + private final ICloudStorage fileStreamProvider; + private final CloudPath rootMetadataPath; + private final VersionGenerator versionGenerator; + private final Clock clock; + private final FileManager fileManager; + private final RotatingClientSideKeypairStore globalReader; + private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + + public ClientSideKeypairStoreFactory( + ICloudStorage fileStreamProvider, + CloudPath rootMetadataPath, + VersionGenerator versionGenerator, + Clock clock, + RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, + FileManager fileManager) { + this.fileStreamProvider = fileStreamProvider; + this.rootMetadataPath = rootMetadataPath; + this.versionGenerator = versionGenerator; + this.clock = clock; + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + this.fileManager = fileManager; + GlobalScope globalScope = new GlobalScope(rootMetadataPath); + globalReader = new RotatingClientSideKeypairStore(fileStreamProvider, globalScope); + } + + public RotatingClientSideKeypairStore getGlobalReader() { + return globalReader; + } + + @Override + public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) { + return new ClientSideKeypairStoreWriter(getEncryptedReader(siteId, isPublic), + fileManager, + versionGenerator, + clock, + new EncryptedScope(rootMetadataPath, siteId, isPublic), + cloudEncryptionKeyProvider); + } + + @Override + public StoreReader> getEncryptedReader(Integer siteId, boolean isPublic) { + return new RotatingClientSideKeypairStore(fileStreamProvider, new EncryptedScope(rootMetadataPath, siteId, isPublic), cloudEncryptionKeyProvider); + } + + @Override + public RotatingCloudEncryptionKeyProvider getCloudEncryptionProvider() { + return cloudEncryptionKeyProvider; + } + + @Override + public StoreReader> getReader(Integer siteId) { + return new RotatingClientSideKeypairStore(fileStreamProvider, new SiteScope(rootMetadataPath, siteId)); + } + + @Override + public StoreWriter> getWriter(Integer siteId) { + return new ClientSideKeypairStoreWriter(getReader(siteId), fileManager, versionGenerator, clock, new SiteScope(rootMetadataPath, siteId)); + } +} diff --git a/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java new file mode 100644 index 00000000..8261e942 --- /dev/null +++ b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java @@ -0,0 +1,71 @@ +package com.uid2.admin.store.factory; + +import com.uid2.admin.store.FileManager; +import com.uid2.admin.store.version.VersionGenerator; +import com.uid2.admin.store.writer.EncryptedSaltStoreWriter; +import com.uid2.admin.store.writer.StoreWriter; +import com.uid2.shared.Const; +import com.uid2.shared.cloud.TaggableCloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.EncryptedRotatingSaltProvider; +import com.uid2.shared.store.RotatingSaltProvider; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.reader.StoreReader; +import com.uid2.shared.store.scope.EncryptedScope; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + +public class SaltStoreFactory implements EncryptedStoreFactory> { + private static final Logger LOGGER = LoggerFactory.getLogger(SaltStoreFactory.class); + + JsonObject config; + CloudPath rootMetadatapath; + FileManager fileManager; + TaggableCloudStorage taggableCloudStorage; + VersionGenerator versionGenerator; + RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + + public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, FileManager fileManager, + TaggableCloudStorage taggableCloudStorage, VersionGenerator versionGenerator, + RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { + this.config = config; + this.rootMetadatapath = rootMetadataPath; + this.fileManager = fileManager; + this.taggableCloudStorage = taggableCloudStorage; + this.versionGenerator = versionGenerator; + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + } + + @Override + public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) { + EncryptedScope scope = new EncryptedScope(rootMetadatapath, siteId, isPublic); + EncryptedRotatingSaltProvider saltProvider = new EncryptedRotatingSaltProvider(taggableCloudStorage, cloudEncryptionKeyProvider, scope); + return new EncryptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, scope, cloudEncryptionKeyProvider, siteId); + } + + @Override + public StoreReader> getEncryptedReader(Integer siteId, boolean isPublic) { + LOGGER.warn("getEncryptedReader called on SaltStoreFactory. This method is not implemented."); + return null; + } + + @Override + public RotatingCloudEncryptionKeyProvider getCloudEncryptionProvider() { + return cloudEncryptionKeyProvider; + } + + @Override + public StoreReader> getReader(Integer siteId) { + LOGGER.warn("getReader called on SaltStoreFactory. This method is not implemented."); + return null; + } + + @Override + public StoreWriter> getWriter(Integer siteId) { + LOGGER.warn("getWriter called on SaltStoreFactory. This method is not implemented."); + return null; + } +} diff --git a/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java index fe48e533..78c0a959 100644 --- a/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java @@ -6,6 +6,8 @@ import com.uid2.admin.store.version.VersionGenerator; import com.uid2.shared.model.ClientSideKeypair; import com.uid2.shared.store.reader.RotatingClientSideKeypairStore; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.reader.StoreReader; import com.uid2.shared.store.scope.StoreScope; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; @@ -17,12 +19,19 @@ public class ClientSideKeypairStoreWriter implements StoreWriter> store, FileManager fileManager, VersionGenerator versionGenerator, Clock clock, StoreScope scope) { FileName dataFile = new FileName("client_side_keypairs", ".json"); String dataType = "client_side_keypairs"; writer = new ScopedStoreWriter(store, fileManager, versionGenerator, clock, scope, dataFile, dataType); } + public ClientSideKeypairStoreWriter(StoreReader> store, FileManager fileManager, + VersionGenerator versionGenerator, Clock clock, StoreScope scope, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { + FileName dataFile = new FileName("client_side_keypairs", ".json"); + String dataType = "client_side_keypairs"; + writer = new EncryptedScopedStoreWriter(store, fileManager, versionGenerator, clock, scope, dataFile, dataType, cloudEncryptionKeyProvider, scope.getId()); + } + @Override public void upload(Collection data, JsonObject extraMeta) throws Exception { JsonArray jsonKeypairs = new JsonArray(); diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java new file mode 100644 index 00000000..775b29b8 --- /dev/null +++ b/src/main/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriter.java @@ -0,0 +1,105 @@ +package com.uid2.admin.store.writer; + +import com.uid2.admin.store.FileManager; +import com.uid2.admin.store.version.VersionGenerator; +import com.uid2.shared.cloud.TaggableCloudStorage; +import com.uid2.shared.encryption.AesGcm; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.RotatingSaltProvider; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.StoreScope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import io.vertx.core.json.JsonObject; + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Collection; + +public class EncryptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter { + private StoreScope scope; + private RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + private Integer siteId; + + private static final Logger LOGGER = LoggerFactory.getLogger(EncryptedSaltStoreWriter.class); + public EncryptedSaltStoreWriter(JsonObject config, RotatingSaltProvider provider, FileManager fileManager, + TaggableCloudStorage cloudStorage, VersionGenerator versionGenerator, StoreScope scope, + RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, Integer siteId) { + super(config, provider, fileManager, cloudStorage, versionGenerator); + this.scope = scope; + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + this.siteId = siteId; + } + + @Override + protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) { + return scope.resolve(new CloudPath("salts.txt." + snapshot.getEffective().toEpochMilli())).toString(); + } + + @Override + protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception { + if (siteId == null) { + throw new IllegalStateException("Site ID is not set."); + } + + if (!cloudStorage.list(location).isEmpty()) { + // update the tags on the file to ensure it is still marked as current + this.setStatusTagToCurrent(location); + return; + } + + StringBuilder stringBuilder = new StringBuilder(); + + for (SaltEntry entry: snapshot.getAllRotatingSalts()) { + stringBuilder.append(entry.getId()).append(",").append(entry.getLastUpdated()).append(",").append(entry.getSalt()).append("\n"); + } + + String data = stringBuilder.toString(); + + CloudEncryptionKey encryptionKey = null; + try { + encryptionKey = cloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId); + } catch (IllegalStateException e) { + LOGGER.error("Error: No Cloud Encryption keys available for encryption for site ID: {}", siteId, e); + } + JsonObject encryptedJson = new JsonObject(); + if (encryptionKey != null) { + byte[] secret = Base64.getDecoder().decode(encryptionKey.getSecret()); + byte[] encryptedPayload = AesGcm.encrypt(data.getBytes(StandardCharsets.UTF_8), secret); + encryptedJson.put("key_id", encryptionKey.getId()) + .put("encryption_version", "1.0") + .put("encrypted_payload", Base64.getEncoder().encodeToString(encryptedPayload)); + } else { + throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId); + } + + final Path newSaltsFile = Files.createTempFile("salts", ".txt"); + try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) { + w.write(encryptedJson.encodePrettily()); + } + + this.upload(newSaltsFile.toString(), location); + } + + @Override + protected void refreshProvider() { + // we do not need to refresh the provider on encrypted writers + } + + @Override + public void upload(Object data, JsonObject extraMeta) throws Exception { + for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection) data) { + super.upload(saltSnapshot); + } + } + + @Override + public void rewriteMeta() throws Exception { + + } +} diff --git a/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java index c2718c1e..57f82aec 100644 --- a/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/EncryptedScopedStoreWriter.java @@ -31,6 +31,7 @@ public EncryptedScopedStoreWriter(IMetadataVersionedStore provider, this.siteId = siteId; } + @Override public void upload(String data, JsonObject extraMeta) throws Exception { if (siteId == null) { throw new IllegalStateException("Site ID is not set."); diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java index 76626539..f8bfc2a5 100644 --- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java @@ -16,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -26,10 +27,10 @@ public class SaltStoreWriter { private static final Logger LOGGER = LoggerFactory.getLogger(SaltStoreWriter.class); private final RotatingSaltProvider provider; private final FileManager fileManager; - private final String saltSnapshotLocationPrefix; + protected final String saltSnapshotLocationPrefix; private final VersionGenerator versionGenerator; - private final TaggableCloudStorage cloudStorage; + protected final TaggableCloudStorage cloudStorage; private final Map currentTags = Map.of("status", "current"); private final Map obsoleteTags = Map.of("status", "obsolete"); @@ -46,7 +47,16 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { final Instant now = Instant.now(); final long generated = now.getEpochSecond(); - final JsonObject metadata = provider.getMetadata(); + JsonObject metadata = null; + try { + metadata = provider.getMetadata(); + } catch (CloudStorageException e) { + if (e.getMessage().contains("The specified key does not exist")) { + metadata = new JsonObject(); + } else { + throw e; + } + } // bump up metadata version metadata.put("version", versionGenerator.getVersion()); metadata.put("generated", generated); @@ -54,9 +64,16 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { final JsonArray snapshotsMetadata = new JsonArray(); metadata.put("salts", snapshotsMetadata); - final List snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data)) - .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) - .collect(Collectors.toList()); + List currentSnapshots = provider.getSnapshots(); + List snapshots = null; + + if (currentSnapshots != null) { + snapshots = Stream.concat(currentSnapshots.stream(), Stream.of(data)) + .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) + .collect(Collectors.toList()); + } else { + snapshots = List.of(data); + } // of the currently effective snapshots keep only the most recent one RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = snapshots.stream() .filter(snapshot -> snapshot.isEffective(now)) @@ -89,6 +106,10 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { fileManager.uploadMetadata(metadata, "salts", new CloudPath(provider.getMetadataPath())); // refresh manually + refreshProvider(); + } + + protected void refreshProvider() throws Exception { provider.loadContent(); } @@ -111,11 +132,11 @@ public void archiveSaltLocations() throws Exception { }); } - private String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) { + protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snapshot) { return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli(); } - private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception { + protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception { // do not overwrite existing files if (!cloudStorage.list(location).isEmpty()) { // update the tags on the file to ensure it is still marked as current @@ -130,10 +151,15 @@ private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, Str } } - cloudStorage.upload(newSaltsFile.toString(), location, this.currentTags); + this.upload(newSaltsFile.toString(), location); + } + + protected void upload(String data, String location) throws Exception { + cloudStorage.upload(data, location, this.currentTags); + } - private void setStatusTagToCurrent(String location) throws CloudStorageException { + protected void setStatusTagToCurrent(String location) throws CloudStorageException { this.cloudStorage.setTags(location, this.currentTags); } diff --git a/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java b/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java index f73541a9..525b9494 100644 --- a/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java +++ b/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java @@ -6,7 +6,9 @@ import com.uid2.shared.auth.*; import com.uid2.shared.model.EncryptionKey; import com.uid2.shared.model.KeysetKey; +import com.uid2.shared.model.SaltEntry; import com.uid2.shared.model.Site; +import com.uid2.shared.store.RotatingSaltProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -310,4 +312,9 @@ public static PrivateSiteDataMap getKeysetKeys(Collection getPrivateSaltSites(Collection operators) { + final PrivateSiteDataMap result = getPrivateSites(operators); + return result.keySet().stream().toList(); + } } diff --git a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java index bc55bc14..815454b4 100644 --- a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java +++ b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java @@ -6,16 +6,12 @@ import com.uid2.shared.auth.Keyset; import com.uid2.shared.auth.OperatorKey; import com.uid2.shared.auth.OperatorType; -import com.uid2.shared.model.EncryptionKey; -import com.uid2.shared.model.KeysetKey; -import com.uid2.shared.model.Site; +import com.uid2.shared.model.*; +import com.uid2.shared.store.RotatingSaltProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; +import java.util.*; public class PublicSiteUtil { private static final Logger LOGGER = LoggerFactory.getLogger(PrivateSiteUtil.class); @@ -128,4 +124,23 @@ public static PrivateSiteDataMap getPublicKeysetKeys( return result; } + + public static List getPublicSaltSites(Collection operators) { + final PrivateSiteDataMap result = getPublicSitesMap(operators); + return result.keySet().stream().toList(); + } + + public static PrivateSiteDataMap getPublicClientKeypairs( + Collection globalClientSideKeypair, + Collection operators) { + final PrivateSiteDataMap result = getPublicSitesMap(operators); + + globalClientSideKeypair.forEach(clientSideKeypair -> { + result.forEach((publicSiteId, publicSiteData) -> { + publicSiteData.add(clientSideKeypair); + }); + }); + + return result; + } } diff --git a/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java b/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java index 8f6adec9..4910c300 100644 --- a/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java +++ b/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java @@ -109,6 +109,10 @@ public void overwritesExistingDataWhenChanged() throws Exception { reader.loadContent(); Long oldVersion = reader.getMetadata().getLong("version"); + // This test relies on our version generator returning a new timestamp, but the code can execute so fast we don't get a new version + // This small sleep makes this test much more stable + Thread.sleep(100); + Site updatedSite = new Site(scopedSiteId, "site 1 updated", true); MultiScopeStoreWriter> multiStore = new MultiScopeStoreWriter<>(fileManager, siteStoreFactory, MultiScopeStoreWriter::areCollectionsEqual); @@ -204,6 +208,7 @@ public void uploadPrivateWithEncryption() throws Exception { Map allKeys = new HashMap<>(); allKeys.put(1, encryptionKey); when(cloudEncryptionKeyProvider.getAll()).thenReturn(allKeys); + when(cloudEncryptionKeyProvider.getKey(1)).thenReturn(encryptionKey); SiteStoreFactory siteStoreFactory = new SiteStoreFactory( cloudStorage, @@ -241,6 +246,7 @@ public void uploadPublicWithEncryption() throws Exception { Map allKeys = new HashMap<>(); allKeys.put(1, encryptionKey); when(cloudEncryptionKeyProvider.getAll()).thenReturn(allKeys); + when(cloudEncryptionKeyProvider.getKey(1)).thenReturn(encryptionKey); SiteStoreFactory siteStoreFactory = new SiteStoreFactory( cloudStorage, diff --git a/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java new file mode 100644 index 00000000..f7555106 --- /dev/null +++ b/src/test/java/com/uid2/admin/store/writer/EncryptedSaltStoreWriterTest.java @@ -0,0 +1,127 @@ +package com.uid2.admin.store.writer; + +import com.uid2.admin.store.FileManager; +import com.uid2.admin.store.version.VersionGenerator; +import com.uid2.shared.cloud.CloudStorageException; +import com.uid2.shared.cloud.TaggableCloudStorage; +import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.RotatingSaltProvider; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.scope.StoreScope; +import io.vertx.core.json.JsonObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; +import static com.uid2.shared.util.CloudEncryptionHelpers.decryptInputStream; + +public class EncryptedSaltStoreWriterTest { + private AutoCloseable mocks; + + @Mock + private FileManager fileManager; + + @Mock + TaggableCloudStorage taggableCloudStorage; + + @Mock + RotatingSaltProvider rotatingSaltProvider; + + @Mock + RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider; + + @Mock + VersionGenerator versionGenerator; + + @Mock + StoreScope storeScope; + + CloudEncryptionKey encryptionKey; + + JsonObject config; + + private final Integer siteId = 1; + + @Captor + private ArgumentCaptor pathCaptor; + @Captor + private ArgumentCaptor cloudPathCaptor; + + @BeforeEach + public void setUp() throws Exception { + mocks = MockitoAnnotations.openMocks(this); + config = new JsonObject(); + config.put("salt_snapshot_location_prefix", "test"); + + when(versionGenerator.getVersion()).thenReturn(1L); + when(rotatingSaltProvider.getMetadataPath()).thenReturn("test/path/"); + when(storeScope.resolve(any())).thenReturn(new CloudPath("test/path/")); + + // Setup Cloud Encryption Keys + byte[] keyBytes = new byte[32]; + new Random().nextBytes(keyBytes); + String base64Key = Base64.getEncoder().encodeToString(keyBytes); + encryptionKey = new CloudEncryptionKey(1, 1, 0, 0, base64Key); + + Map mockKeyMap = new HashMap<>(); + mockKeyMap.put(encryptionKey.getId(), encryptionKey); + when(rotatingCloudEncryptionKeyProvider.getAll()).thenReturn(mockKeyMap); + when(rotatingCloudEncryptionKeyProvider.getKey(1)).thenReturn(mockKeyMap.get(1)); + when(rotatingCloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId)).thenReturn(encryptionKey); + } + + private RotatingSaltProvider.SaltSnapshot makeSnapshot(Instant effective, Instant expires, int nsalts) { + SaltEntry[] entries = new SaltEntry[nsalts]; + for (int i = 0; i < entries.length; ++i) { + entries[i] = new SaltEntry(i, "hashed_id", effective.toEpochMilli(), "salt"); + } + return new RotatingSaltProvider.SaltSnapshot(effective, expires, entries, "test_first_level_salt"); + } + + private void verifyFile(String filelocation, RotatingSaltProvider.SaltSnapshot snapshot) throws IOException { + InputStream encoded = Files.newInputStream(Paths.get(filelocation)); + String contents = decryptInputStream(encoded, rotatingCloudEncryptionKeyProvider); + SaltEntry[] entries = snapshot.getAllRotatingSalts(); + Integer idx = 0; + for (String line : contents.split("\n")) { + String[] entrySplit = line.split(","); + assertEquals(entries[idx].getId(), Long.parseLong(entrySplit[0])); + assertEquals(entries[idx].getSalt(), entrySplit[2]); + idx++; + } + } + + @Test + public void testUploadNew() throws Exception { + RotatingSaltProvider.SaltSnapshot snapshot = makeSnapshot(Instant.now(), Instant.ofEpochMilli(Instant.now().toEpochMilli() + 10000), 1000000); + + when(rotatingSaltProvider.getMetadata()).thenThrow(new CloudStorageException("The specified key does not exist: AmazonS3Exception: test-core-bucket")); + when(rotatingSaltProvider.getSnapshots()).thenReturn(null); + + when(taggableCloudStorage.list(anyString())).thenReturn(new ArrayList<>()); + + EncryptedSaltStoreWriter encryptedSaltStoreWriter = new EncryptedSaltStoreWriter(config, rotatingSaltProvider, + fileManager, taggableCloudStorage, versionGenerator, storeScope, rotatingCloudEncryptionKeyProvider, siteId); + + encryptedSaltStoreWriter.upload(snapshot); + + verify(taggableCloudStorage).upload(pathCaptor.capture(), cloudPathCaptor.capture(), any()); + assertEquals(cloudPathCaptor.getValue(), "test/path"); + + verifyFile(pathCaptor.getValue(), snapshot); + } +}