diff --git a/pom.xml b/pom.xml index d8e2d5f4..5d212dfd 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ 1.12.2 5.11.2 - 8.0.10-alpha-172-SNAPSHOT + 8.0.11-alpha-173-SNAPSHOT 0.5.10 ${project.version} diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java new file mode 100644 index 00000000..98d43dd0 --- /dev/null +++ b/src/main/java/com/uid2/admin/job/EncryptionJob/ClientSideKeypairEncryptionJob.java @@ -0,0 +1,36 @@ +package com.uid2.admin.job.EncryptionJob; + +import com.uid2.admin.job.model.Job; +import com.uid2.admin.model.PrivateSiteDataMap; +import com.uid2.admin.store.MultiScopeStoreWriter; +import com.uid2.admin.util.PublicSiteUtil; +import com.uid2.shared.auth.OperatorKey; +import com.uid2.shared.model.ClientSideKeypair; + +import java.util.Collection; + +public class ClientSideKeypairEncryptionJob extends Job { + private final Collection globalOperators; + private final Collection globalClientSideKeypairs; + + private final MultiScopeStoreWriter> multiScopeStoreWriter; + + public ClientSideKeypairEncryptionJob(Collection globalOperators, Collection globalClientSideKeypairs, + MultiScopeStoreWriter> multiScopeStoreWriter) { + this.globalOperators = globalOperators; + this.globalClientSideKeypairs = globalClientSideKeypairs; + this.multiScopeStoreWriter = multiScopeStoreWriter; + } + + @Override + public String getId() { + return "cloud-encryption-sync-clientside-keypair"; + } + + @Override + public void execute() throws Exception { + // Only public operators support clientside keypair + PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators); + multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null); + } +} diff --git a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java index ce8c5da3..186809d4 100644 --- a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java +++ b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java @@ -17,12 +17,13 @@ import com.uid2.shared.cloud.CloudUtils; import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.cloud.TaggableCloudStorage; +import com.uid2.shared.model.ClientSideKeypair; import com.uid2.shared.model.EncryptionKey; import com.uid2.shared.model.KeysetKey; import com.uid2.shared.model.Site; import com.uid2.shared.store.CloudPath; import com.uid2.admin.legacy.LegacyClientKey; -import com.uid2.shared.store.RotatingEncryptedSaltProvider; +import com.uid2.shared.store.EncryptedRotatingSaltProvider; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.scope.GlobalScope; @@ -122,6 +123,15 @@ public void execute() throws Exception { rotatingCloudEncryptionKeyProvider ); + ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory( + cloudStorage, + new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)), + versionGenerator, + clock, + rotatingCloudEncryptionKeyProvider, + fileManager + ); + CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp)); GlobalScope operatorScope = new GlobalScope(operatorMetadataPath); RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope); @@ -138,6 +148,7 @@ public void execute() throws Exception { keysetKeyStoreFactory.getGlobalReader().loadContent(); } saltProvider.loadContent(); + clientSideKeypairStoreFactory.getGlobalReader().loadContent(); } Collection globalOperators = operatorKeyProvider.getAll(); @@ -146,6 +157,8 @@ public void execute() throws Exception { Collection globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet(); Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id"); Map globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls(); + Collection globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll(); + MultiScopeStoreWriter> siteWriter = new MultiScopeStoreWriter<>( fileManager, siteStoreFactory, @@ -166,6 +179,10 @@ public void execute() throws Exception { fileManager, saltStoreFactory, MultiScopeStoreWriter::areCollectionsEqual); + MultiScopeStoreWriter> clientSideKeypairWriter = new MultiScopeStoreWriter<>( + fileManager, + clientSideKeypairStoreFactory, + MultiScopeStoreWriter::areCollectionsEqual); SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators); ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators); @@ -179,12 +196,14 @@ public void execute() throws Exception { ); KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls); SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter); + ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter); siteEncryptionSyncJob.execute(); clientEncryptionSyncJob.execute(); encryptionKeyEncryptionSyncJob.execute(); keyAclEncryptionSyncJob.execute(); saltEncryptionJob.execute(); + clientSideKeypairEncryptionJob.execute(); if(config.getBoolean(enableKeysetConfigProp)) { Map globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets(); diff --git a/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java new file mode 100644 index 00000000..a305d966 --- /dev/null +++ b/src/main/java/com/uid2/admin/store/factory/ClientSideKeypairStoreFactory.java @@ -0,0 +1,79 @@ +package com.uid2.admin.store.factory; + +import com.uid2.admin.store.writer.ClientSideKeypairStoreWriter; +import com.uid2.admin.store.writer.StoreWriter; +import com.uid2.shared.model.ClientSideKeypair; +import com.uid2.shared.store.reader.RotatingClientSideKeypairStore; +import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; +import com.uid2.shared.store.reader.StoreReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.uid2.admin.store.Clock; +import com.uid2.admin.store.FileManager; +import com.uid2.admin.store.version.VersionGenerator; +import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.store.CloudPath; +import com.uid2.shared.store.scope.EncryptedScope; +import com.uid2.shared.store.scope.GlobalScope; +import com.uid2.shared.store.scope.SiteScope; + +import java.util.Collection; + +public class ClientSideKeypairStoreFactory implements EncryptedStoreFactory> { + private final ICloudStorage fileStreamProvider; + private final CloudPath rootMetadataPath; + private final VersionGenerator versionGenerator; + private final Clock clock; + private final FileManager fileManager; + private final RotatingClientSideKeypairStore globalReader; + private final RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; + + public ClientSideKeypairStoreFactory( + ICloudStorage fileStreamProvider, + CloudPath rootMetadataPath, + VersionGenerator versionGenerator, + Clock clock, + RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider, + FileManager fileManager) { + this.fileStreamProvider = fileStreamProvider; + this.rootMetadataPath = rootMetadataPath; + this.versionGenerator = versionGenerator; + this.clock = clock; + this.cloudEncryptionKeyProvider = cloudEncryptionKeyProvider; + this.fileManager = fileManager; + GlobalScope globalScope = new GlobalScope(rootMetadataPath); + globalReader = new RotatingClientSideKeypairStore(fileStreamProvider, globalScope); + } + + public RotatingClientSideKeypairStore getGlobalReader() { + return globalReader; + } + + @Override + public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) { + return new ClientSideKeypairStoreWriter(getEncryptedReader(siteId, isPublic), + fileManager, + versionGenerator, + clock, + new EncryptedScope(rootMetadataPath, siteId, isPublic)); + } + + @Override + public StoreReader> getEncryptedReader(Integer siteId, boolean isPublic) { + return new RotatingClientSideKeypairStore(fileStreamProvider, new EncryptedScope(rootMetadataPath, siteId, isPublic)); + } + + @Override + public RotatingCloudEncryptionKeyProvider getCloudEncryptionProvider() { + return cloudEncryptionKeyProvider; + } + + @Override + public StoreReader> getReader(Integer siteId) { + return new RotatingClientSideKeypairStore(fileStreamProvider, new SiteScope(rootMetadataPath, siteId)); + } + + @Override + public StoreWriter> getWriter(Integer siteId) { + return new ClientSideKeypairStoreWriter(getReader(siteId), fileManager, versionGenerator, clock, new SiteScope(rootMetadataPath, siteId)); + } +} diff --git a/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java index b2acb5f0..afce4442 100644 --- a/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java +++ b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java @@ -7,7 +7,7 @@ import com.uid2.shared.Const; import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.store.CloudPath; -import com.uid2.shared.store.RotatingEncryptedSaltProvider; +import com.uid2.shared.store.EncryptedRotatingSaltProvider; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.reader.StoreReader; @@ -38,7 +38,7 @@ public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, FileManag @Override public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) { EncryptedScope scope = new EncryptedScope(rootMetadatapath, siteId, isPublic); - RotatingEncryptedSaltProvider saltProvider = new RotatingEncryptedSaltProvider(taggableCloudStorage, + EncryptedRotatingSaltProvider saltProvider = new EncryptedRotatingSaltProvider(taggableCloudStorage, scope.resolve(new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp))).toString(), cloudEncryptionKeyProvider ); return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, scope, cloudEncryptionKeyProvider, siteId); } diff --git a/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java index fe48e533..24a96d06 100644 --- a/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/ClientSideKeypairStoreWriter.java @@ -6,6 +6,7 @@ import com.uid2.admin.store.version.VersionGenerator; import com.uid2.shared.model.ClientSideKeypair; import com.uid2.shared.store.reader.RotatingClientSideKeypairStore; +import com.uid2.shared.store.reader.StoreReader; import com.uid2.shared.store.scope.StoreScope; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; @@ -17,7 +18,7 @@ public class ClientSideKeypairStoreWriter implements StoreWriter> store, FileManager fileManager, VersionGenerator versionGenerator, Clock clock, StoreScope scope) { FileName dataFile = new FileName("client_side_keypairs", ".json"); String dataType = "client_side_keypairs"; writer = new ScopedStoreWriter(store, fileManager, versionGenerator, clock, scope, dataFile, dataType); diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java index 2ea6899b..46a4b2a5 100644 --- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java @@ -141,7 +141,6 @@ protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, S } } - //cloudStorage.upload(newSaltsFile.toString(), location, this.currentTags); this.upload(newSaltsFile.toString(), location); } diff --git a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java index 5b657fc1..f3ec8fd9 100644 --- a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java +++ b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java @@ -6,10 +6,7 @@ import com.uid2.shared.auth.Keyset; import com.uid2.shared.auth.OperatorKey; import com.uid2.shared.auth.OperatorType; -import com.uid2.shared.model.EncryptionKey; -import com.uid2.shared.model.KeysetKey; -import com.uid2.shared.model.SaltEntry; -import com.uid2.shared.model.Site; +import com.uid2.shared.model.*; import com.uid2.shared.store.RotatingSaltProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -144,4 +141,18 @@ public static PrivateSiteDataMap getPublicSal return result; } + + public static PrivateSiteDataMap getPublicClientKeypairs( + Collection globalClientSideKeypair, + Collection operators) { + final PrivateSiteDataMap result = getPublicSitesMap(operators); + + globalClientSideKeypair.forEach(clientSideKeypair -> { + result.forEach((publicSiteId, publicSiteData) -> { + publicSiteData.add(clientSideKeypair); + }); + }); + + return result; + } } diff --git a/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java b/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java index 8f6adec9..4910c300 100644 --- a/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java +++ b/src/test/java/com/uid2/admin/store/MultiScopeStoreWriterTest.java @@ -109,6 +109,10 @@ public void overwritesExistingDataWhenChanged() throws Exception { reader.loadContent(); Long oldVersion = reader.getMetadata().getLong("version"); + // This test relies on our version generator returning a new timestamp, but the code can execute so fast we don't get a new version + // This small sleep makes this test much more stable + Thread.sleep(100); + Site updatedSite = new Site(scopedSiteId, "site 1 updated", true); MultiScopeStoreWriter> multiStore = new MultiScopeStoreWriter<>(fileManager, siteStoreFactory, MultiScopeStoreWriter::areCollectionsEqual); @@ -204,6 +208,7 @@ public void uploadPrivateWithEncryption() throws Exception { Map allKeys = new HashMap<>(); allKeys.put(1, encryptionKey); when(cloudEncryptionKeyProvider.getAll()).thenReturn(allKeys); + when(cloudEncryptionKeyProvider.getKey(1)).thenReturn(encryptionKey); SiteStoreFactory siteStoreFactory = new SiteStoreFactory( cloudStorage, @@ -241,6 +246,7 @@ public void uploadPublicWithEncryption() throws Exception { Map allKeys = new HashMap<>(); allKeys.put(1, encryptionKey); when(cloudEncryptionKeyProvider.getAll()).thenReturn(allKeys); + when(cloudEncryptionKeyProvider.getKey(1)).thenReturn(encryptionKey); SiteStoreFactory siteStoreFactory = new SiteStoreFactory( cloudStorage, diff --git a/src/test/java/com/uid2/admin/store/writer/SaltStoreWriterTest.java b/src/test/java/com/uid2/admin/store/writer/SaltStoreWriterTest.java new file mode 100644 index 00000000..3355a313 --- /dev/null +++ b/src/test/java/com/uid2/admin/store/writer/SaltStoreWriterTest.java @@ -0,0 +1,5 @@ +package com.uid2.admin.store.writer; + +public class SaltStoreWriterTest { + +}