diff --git a/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java index 988b17da..6ab992f8 100644 --- a/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java +++ b/src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java @@ -7,17 +7,18 @@ import com.uid2.admin.util.PublicSiteUtil; import com.uid2.shared.auth.OperatorKey; import com.uid2.shared.model.SaltEntry; +import com.uid2.shared.store.RotatingSaltProvider; import java.util.Collection; public class SaltEncryptionJob extends Job { private final Collection globalOperators; - private final Collection saltEntries; - private final MultiScopeStoreWriter> multiScopeStoreWriter; + private final Collection saltEntries; + private final MultiScopeStoreWriter> multiScopeStoreWriter; public SaltEncryptionJob(Collection globalOperators, - Collection saltEntries, - MultiScopeStoreWriter> multiScopeStoreWriter) { + Collection saltEntries, + MultiScopeStoreWriter> multiScopeStoreWriter) { this.globalOperators = globalOperators; this.saltEntries = saltEntries; this.multiScopeStoreWriter = multiScopeStoreWriter; @@ -31,9 +32,9 @@ public String getId() { @Override public void execute() throws Exception { - PrivateSiteDataMap desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators); + PrivateSiteDataMap desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators); multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null); - PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators); + PrivateSiteDataMap desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators); multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null); } } diff --git a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java index 31c87b3e..ce8c5da3 100644 --- a/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java +++ b/src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java @@ -16,11 +16,14 @@ import com.uid2.shared.auth.RotatingOperatorKeyProvider; import com.uid2.shared.cloud.CloudUtils; import com.uid2.shared.cloud.ICloudStorage; +import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.model.EncryptionKey; import com.uid2.shared.model.KeysetKey; import com.uid2.shared.model.Site; import com.uid2.shared.store.CloudPath; import com.uid2.admin.legacy.LegacyClientKey; +import com.uid2.shared.store.RotatingEncryptedSaltProvider; +import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; import com.uid2.shared.store.scope.GlobalScope; import io.vertx.core.json.JsonObject; @@ -32,12 +35,12 @@ public class EncryptedFilesSyncJob extends Job { private final JsonObject config; private final WriteLock writeLock; - private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider; + private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider; public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) { this.config = config; this.writeLock = writeLock; - this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider; + this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider; } @Override @@ -47,20 +50,22 @@ public String getId() { @Override public void execute() throws Exception { - ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config); + TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config); FileStorage fileStorage = new TmpFileStorage(); ObjectWriter jsonWriter = JsonUtil.createJsonWriter(); Clock clock = new InstantClock(); VersionGenerator versionGenerator = new EpochVersionGenerator(clock); FileManager fileManager = new FileManager(cloudStorage, fileStorage); + RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp)); + SiteStoreFactory siteStoreFactory = new SiteStoreFactory( cloudStorage, new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)), jsonWriter, versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory( @@ -69,7 +74,7 @@ public void execute() throws Exception { jsonWriter, versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory( @@ -77,7 +82,7 @@ public void execute() throws Exception { new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)), versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory( @@ -86,7 +91,7 @@ public void execute() throws Exception { jsonWriter, versionGenerator, clock, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, fileManager); KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory( @@ -96,7 +101,7 @@ public void execute() throws Exception { versionGenerator, clock, fileManager, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, config.getBoolean(enableKeysetConfigProp)); KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory( @@ -105,15 +110,24 @@ public void execute() throws Exception { versionGenerator, clock, fileManager, - RotatingCloudEncryptionKeyProvider, + rotatingCloudEncryptionKeyProvider, config.getBoolean(enableKeysetConfigProp)); + SaltStoreFactory saltStoreFactory = new SaltStoreFactory( + config, + new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)), + fileManager, + cloudStorage, + versionGenerator, + rotatingCloudEncryptionKeyProvider + ); + CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp)); GlobalScope operatorScope = new GlobalScope(operatorMetadataPath); RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope); synchronized (writeLock) { - RotatingCloudEncryptionKeyProvider.loadContent(); + rotatingCloudEncryptionKeyProvider.loadContent(); operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata()); siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata()); clientKeyStoreFactory.getGlobalReader().loadContent(); @@ -123,7 +137,9 @@ public void execute() throws Exception { keysetStoreFactory.getGlobalReader().loadContent(); keysetKeyStoreFactory.getGlobalReader().loadContent(); } + saltProvider.loadContent(); } + Collection globalOperators = operatorKeyProvider.getAll(); Collection globalSites = siteStoreFactory.getGlobalReader().getAllSites(); Collection globalClients = clientKeyStoreFactory.getGlobalReader().getAll(); @@ -146,6 +162,10 @@ public void execute() throws Exception { fileManager, keyAclStoreFactory, MultiScopeStoreWriter::areMapsEqual); + MultiScopeStoreWriter> saltWriter = new MultiScopeStoreWriter<>( + fileManager, + saltStoreFactory, + MultiScopeStoreWriter::areCollectionsEqual); SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators); ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators); @@ -158,10 +178,13 @@ public void execute() throws Exception { encryptionKeyWriter ); KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls); + SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter); + siteEncryptionSyncJob.execute(); clientEncryptionSyncJob.execute(); encryptionKeyEncryptionSyncJob.execute(); keyAclEncryptionSyncJob.execute(); + saltEncryptionJob.execute(); if(config.getBoolean(enableKeysetConfigProp)) { Map globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets(); diff --git a/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java index 21f31a1e..b2acb5f0 100644 --- a/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java +++ b/src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java @@ -4,6 +4,7 @@ import com.uid2.admin.store.version.VersionGenerator; import com.uid2.admin.store.writer.EncyptedSaltStoreWriter; import com.uid2.admin.store.writer.StoreWriter; +import com.uid2.shared.Const; import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.store.CloudPath; import com.uid2.shared.store.RotatingEncryptedSaltProvider; @@ -18,18 +19,16 @@ public class SaltStoreFactory implements EncryptedStoreFactory> { JsonObject config; CloudPath rootMetadatapath; - RotatingSaltProvider saltProvider; FileManager fileManager; TaggableCloudStorage taggableCloudStorage; VersionGenerator versionGenerator; RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider; - public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, RotatingSaltProvider saltProvider, FileManager fileManager, + public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, FileManager fileManager, TaggableCloudStorage taggableCloudStorage, VersionGenerator versionGenerator, RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) { this.config = config; this.rootMetadatapath = rootMetadataPath; - this.saltProvider = saltProvider; this.fileManager = fileManager; this.taggableCloudStorage = taggableCloudStorage; this.versionGenerator = versionGenerator; @@ -38,15 +37,15 @@ public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, RotatingS @Override public StoreWriter> getEncryptedWriter(Integer siteId, boolean isPublic) { - return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, - new EncryptedScope(rootMetadatapath, siteId, isPublic), cloudEncryptionKeyProvider, siteId); + EncryptedScope scope = new EncryptedScope(rootMetadatapath, siteId, isPublic); + RotatingEncryptedSaltProvider saltProvider = new RotatingEncryptedSaltProvider(taggableCloudStorage, + scope.resolve(new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp))).toString(), cloudEncryptionKeyProvider ); + return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, scope, cloudEncryptionKeyProvider, siteId); } @Override public StoreReader> getEncryptedReader(Integer siteId, boolean isPublic) { - return new RotatingEncryptedSaltProvider(taggableCloudStorage, - new EncryptedScope(rootMetadatapath, siteId, isPublic).getMetadataPath().toString(), - cloudEncryptionKeyProvider); + return null; } @Override diff --git a/src/main/java/com/uid2/admin/store/writer/EncyptedSaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/EncyptedSaltStoreWriter.java index d29f8a29..7bf85fc3 100644 --- a/src/main/java/com/uid2/admin/store/writer/EncyptedSaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/EncyptedSaltStoreWriter.java @@ -5,6 +5,7 @@ import com.uid2.shared.cloud.TaggableCloudStorage; import com.uid2.shared.encryption.AesGcm; import com.uid2.shared.model.CloudEncryptionKey; +import com.uid2.shared.model.SaltEntry; import com.uid2.shared.store.CloudPath; import com.uid2.shared.store.RotatingSaltProvider; import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider; @@ -13,8 +14,12 @@ import org.slf4j.LoggerFactory; import io.vertx.core.json.JsonObject; +import java.io.BufferedWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Base64; +import java.util.Collection; public class EncyptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter { private StoreScope scope; @@ -37,11 +42,19 @@ protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnap } @Override - protected void upload(String data, String location) throws Exception { + protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception { if (siteId == null) { throw new IllegalStateException("Site ID is not set."); } + StringBuilder stringBuilder = new StringBuilder(); + + for (SaltEntry entry: snapshot.getAllRotatingSalts()) { + stringBuilder.append(entry.getId()).append(",").append(entry.getLastUpdated()).append(",").append(entry.getSalt()).append("\n"); + } + + String data = stringBuilder.toString(); + CloudEncryptionKey encryptionKey = null; try { encryptionKey = cloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId); @@ -59,13 +72,19 @@ protected void upload(String data, String location) throws Exception { throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId); } + final Path newSaltsFile = Files.createTempFile("salts", ".txt"); + try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) { + w.write(encryptedJson.encodePrettily()); + } - super.upload(encryptedJson.encodePrettily(), location); + this.upload(newSaltsFile.toString(), location); } @Override public void upload(Object data, JsonObject extraMeta) throws Exception { - super.upload((RotatingSaltProvider.SaltSnapshot) data); + for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection) data) { + super.upload(saltSnapshot); + } } @Override diff --git a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java index 6d350527..2ea6899b 100644 --- a/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java +++ b/src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java @@ -16,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -46,7 +47,12 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { final Instant now = Instant.now(); final long generated = now.getEpochSecond(); - final JsonObject metadata = provider.getMetadata(); + JsonObject metadata = null; + try { + metadata = provider.getMetadata(); + } catch (Exception e) { + metadata = new JsonObject(); + } // bump up metadata version metadata.put("version", versionGenerator.getVersion()); metadata.put("generated", generated); @@ -54,9 +60,14 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception { final JsonArray snapshotsMetadata = new JsonArray(); metadata.put("salts", snapshotsMetadata); - final List snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data)) - .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) - .collect(Collectors.toList()); + List snapshots = null; + try { + snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data)) + .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective)) + .collect(Collectors.toList()); + } catch (Exception e) { + snapshots = List.of(data); + } // of the currently effective snapshots keep only the most recent one RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = snapshots.stream() .filter(snapshot -> snapshot.isEffective(now)) @@ -115,7 +126,7 @@ protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snaps return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli(); } - private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception { + protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception { // do not overwrite existing files if (!cloudStorage.list(location).isEmpty()) { // update the tags on the file to ensure it is still marked as current diff --git a/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java b/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java index dd81e397..f065ac79 100644 --- a/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java +++ b/src/main/java/com/uid2/admin/util/PrivateSiteUtil.java @@ -8,6 +8,7 @@ import com.uid2.shared.model.KeysetKey; import com.uid2.shared.model.SaltEntry; import com.uid2.shared.model.Site; +import com.uid2.shared.store.RotatingSaltProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -312,10 +313,10 @@ public static PrivateSiteDataMap getKeysetKeys(Collection getPrivateSaltEntries( - Collection globalSaltEntries, + public static PrivateSiteDataMap getPrivateSaltEntries( + Collection globalSaltEntries, Collection operators) { - final PrivateSiteDataMap result = getPrivateSites(operators); + final PrivateSiteDataMap result = getPrivateSites(operators); globalSaltEntries.forEach(saltEntry -> { result.forEach((publicSiteId, publicSiteData) -> { diff --git a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java index 7470c5c2..5b657fc1 100644 --- a/src/main/java/com/uid2/admin/util/PublicSiteUtil.java +++ b/src/main/java/com/uid2/admin/util/PublicSiteUtil.java @@ -10,6 +10,7 @@ import com.uid2.shared.model.KeysetKey; import com.uid2.shared.model.SaltEntry; import com.uid2.shared.model.Site; +import com.uid2.shared.store.RotatingSaltProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,10 +131,10 @@ public static PrivateSiteDataMap getPublicKeysetKeys( return result; } - public static PrivateSiteDataMap getPublicSaltEntries( - Collection globalSaltEntries, + public static PrivateSiteDataMap getPublicSaltEntries( + Collection globalSaltEntries, Collection operators) { - final PrivateSiteDataMap result = getPublicSitesMap(operators); + final PrivateSiteDataMap result = getPublicSitesMap(operators); globalSaltEntries.forEach(saltEntry -> { result.forEach((publicSiteId, publicSiteData) -> {