Got the encryption job working on salts

src/main/java/com/uid2/admin/job/EncryptionJob/SaltEncryptionJob.java

@@ -7,17 +7,18 @@
 import com.uid2.admin.util.PublicSiteUtil;
 import com.uid2.shared.auth.OperatorKey;
 import com.uid2.shared.model.SaltEntry;
+import com.uid2.shared.store.RotatingSaltProvider;
 
 import java.util.Collection;
 
 public class SaltEncryptionJob extends Job {
     private final Collection<OperatorKey> globalOperators;
-    private final Collection<SaltEntry> saltEntries;
-    private final MultiScopeStoreWriter<Collection<SaltEntry>> multiScopeStoreWriter;
+    private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
+    private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;
 
     public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
-                             Collection<SaltEntry> saltEntries,
-                             MultiScopeStoreWriter<Collection<SaltEntry>> multiScopeStoreWriter) {
+                             Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
+                             MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
         this.globalOperators = globalOperators;
         this.saltEntries = saltEntries;
         this.multiScopeStoreWriter = multiScopeStoreWriter;
@@ -31,9 +32,9 @@ public String getId() {
 
     @Override
     public void execute() throws Exception {
-        PrivateSiteDataMap<SaltEntry> desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators);
+        PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> desiredPrivateState = PrivateSiteUtil.getPrivateSaltEntries(saltEntries, globalOperators);
         multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
-        PrivateSiteDataMap<SaltEntry> desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators);
+        PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> desiredPublicState = PublicSiteUtil.getPublicSaltEntries(saltEntries, globalOperators);
         multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
     }
 }

src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java

@@ -16,11 +16,14 @@
 import com.uid2.shared.auth.RotatingOperatorKeyProvider;
 import com.uid2.shared.cloud.CloudUtils;
 import com.uid2.shared.cloud.ICloudStorage;
+import com.uid2.shared.cloud.TaggableCloudStorage;
 import com.uid2.shared.model.EncryptionKey;
 import com.uid2.shared.model.KeysetKey;
 import com.uid2.shared.model.Site;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.admin.legacy.LegacyClientKey;
+import com.uid2.shared.store.RotatingEncryptedSaltProvider;
+import com.uid2.shared.store.RotatingSaltProvider;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
 import com.uid2.shared.store.scope.GlobalScope;
 import io.vertx.core.json.JsonObject;
@@ -32,12 +35,12 @@
 public class EncryptedFilesSyncJob extends Job {
     private final JsonObject config;
     private final WriteLock writeLock;
-    private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
+    private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;
 
     public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
         this.config = config;
         this.writeLock = writeLock;
-        this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
+        this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
     }
 
     @Override
@@ -47,20 +50,22 @@ public String getId() {
 
     @Override
     public void execute() throws Exception {
-        ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
+        TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
         FileStorage fileStorage = new TmpFileStorage();
         ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
         Clock clock = new InstantClock();
         VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
         FileManager fileManager = new FileManager(cloudStorage, fileStorage);
 
+        RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));
+
         SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
                 cloudStorage,
                 new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
@@ -69,15 +74,15 @@ public void execute() throws Exception {
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
                 cloudStorage,
                 new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
@@ -86,7 +91,7 @@ public void execute() throws Exception {
                 jsonWriter,
                 versionGenerator,
                 clock,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 fileManager);
 
         KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
@@ -96,7 +101,7 @@ public void execute() throws Exception {
                 versionGenerator,
                 clock,
                 fileManager,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 config.getBoolean(enableKeysetConfigProp));
 
         KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
@@ -105,15 +110,24 @@ public void execute() throws Exception {
                 versionGenerator,
                 clock,
                 fileManager,
-                RotatingCloudEncryptionKeyProvider,
+                rotatingCloudEncryptionKeyProvider,
                 config.getBoolean(enableKeysetConfigProp));
 
+        SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
+                config,
+                new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
+                fileManager,
+                cloudStorage,
+                versionGenerator,
+                rotatingCloudEncryptionKeyProvider
+        );
+
         CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
         GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
         RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);
 
         synchronized (writeLock) {
-            RotatingCloudEncryptionKeyProvider.loadContent();
+            rotatingCloudEncryptionKeyProvider.loadContent();
             operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
             siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
             clientKeyStoreFactory.getGlobalReader().loadContent();
@@ -123,7 +137,9 @@ public void execute() throws Exception {
                 keysetStoreFactory.getGlobalReader().loadContent();
                 keysetKeyStoreFactory.getGlobalReader().loadContent();
             }
+            saltProvider.loadContent();
         }
+
         Collection<OperatorKey> globalOperators = operatorKeyProvider.getAll();
         Collection<Site> globalSites = siteStoreFactory.getGlobalReader().getAllSites();
         Collection<LegacyClientKey> globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
@@ -146,6 +162,10 @@ public void execute() throws Exception {
                 fileManager,
                 keyAclStoreFactory,
                 MultiScopeStoreWriter::areMapsEqual);
+        MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> saltWriter = new MultiScopeStoreWriter<>(
+                fileManager,
+                saltStoreFactory,
+                MultiScopeStoreWriter::areCollectionsEqual);
 
         SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
         ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
@@ -158,10 +178,13 @@ public void execute() throws Exception {
                 encryptionKeyWriter
         );
         KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
+        SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
+
         siteEncryptionSyncJob.execute();
         clientEncryptionSyncJob.execute();
         encryptionKeyEncryptionSyncJob.execute();
         keyAclEncryptionSyncJob.execute();
+        saltEncryptionJob.execute();
 
         if(config.getBoolean(enableKeysetConfigProp)) {
             Map<Integer, Keyset> globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();

src/main/java/com/uid2/admin/store/factory/SaltStoreFactory.java

@@ -4,6 +4,7 @@
 import com.uid2.admin.store.version.VersionGenerator;
 import com.uid2.admin.store.writer.EncyptedSaltStoreWriter;
 import com.uid2.admin.store.writer.StoreWriter;
+import com.uid2.shared.Const;
 import com.uid2.shared.cloud.TaggableCloudStorage;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.RotatingEncryptedSaltProvider;
@@ -18,18 +19,16 @@
 public class SaltStoreFactory implements EncryptedStoreFactory<Collection<RotatingSaltProvider.SaltSnapshot>> {
     JsonObject config;
     CloudPath rootMetadatapath;
-    RotatingSaltProvider saltProvider;
     FileManager fileManager;
     TaggableCloudStorage taggableCloudStorage;
     VersionGenerator versionGenerator;
     RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider;
 
-    public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, RotatingSaltProvider saltProvider, FileManager fileManager,
+    public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, FileManager fileManager,
                             TaggableCloudStorage taggableCloudStorage, VersionGenerator versionGenerator,
                             RotatingCloudEncryptionKeyProvider cloudEncryptionKeyProvider) {
         this.config = config;
         this.rootMetadatapath = rootMetadataPath;
-        this.saltProvider = saltProvider;
         this.fileManager = fileManager;
         this.taggableCloudStorage = taggableCloudStorage;
         this.versionGenerator = versionGenerator;
@@ -38,15 +37,15 @@ public SaltStoreFactory(JsonObject config, CloudPath rootMetadataPath, RotatingS
 
     @Override
     public StoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> getEncryptedWriter(Integer siteId, boolean isPublic) {
-        return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator,
-                new EncryptedScope(rootMetadatapath, siteId, isPublic), cloudEncryptionKeyProvider, siteId);
+        EncryptedScope scope = new EncryptedScope(rootMetadatapath, siteId, isPublic);
+        RotatingEncryptedSaltProvider saltProvider = new RotatingEncryptedSaltProvider(taggableCloudStorage,
+                scope.resolve(new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp))).toString(), cloudEncryptionKeyProvider );
+        return new EncyptedSaltStoreWriter(config, saltProvider, fileManager, taggableCloudStorage, versionGenerator, scope, cloudEncryptionKeyProvider, siteId);
     }
 
     @Override
     public StoreReader<Collection<RotatingSaltProvider.SaltSnapshot>> getEncryptedReader(Integer siteId, boolean isPublic) {
-        return new RotatingEncryptedSaltProvider(taggableCloudStorage,
-                new EncryptedScope(rootMetadatapath, siteId, isPublic).getMetadataPath().toString(),
-                cloudEncryptionKeyProvider);
+        return null;
     }
 
     @Override

src/main/java/com/uid2/admin/store/writer/EncyptedSaltStoreWriter.java

@@ -5,6 +5,7 @@
 import com.uid2.shared.cloud.TaggableCloudStorage;
 import com.uid2.shared.encryption.AesGcm;
 import com.uid2.shared.model.CloudEncryptionKey;
+import com.uid2.shared.model.SaltEntry;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.RotatingSaltProvider;
 import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
@@ -13,8 +14,12 @@
 import org.slf4j.LoggerFactory;
 import io.vertx.core.json.JsonObject;
 
+import java.io.BufferedWriter;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Base64;
+import java.util.Collection;
 
 public class EncyptedSaltStoreWriter extends SaltStoreWriter implements StoreWriter {
     private StoreScope scope;
@@ -37,11 +42,19 @@ protected java.lang.String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnap
     }
 
     @Override
-    protected void upload(String data, String location) throws Exception {
+    protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
         if (siteId == null) {
             throw new IllegalStateException("Site ID is not set.");
         }
 
+        StringBuilder stringBuilder = new StringBuilder();
+
+        for (SaltEntry entry: snapshot.getAllRotatingSalts()) {
+            stringBuilder.append(entry.getId()).append(",").append(entry.getLastUpdated()).append(",").append(entry.getSalt()).append("\n");
+        }
+
+        String data = stringBuilder.toString();
+
         CloudEncryptionKey encryptionKey = null;
         try {
             encryptionKey = cloudEncryptionKeyProvider.getEncryptionKeyForSite(siteId);
@@ -59,13 +72,19 @@ protected void upload(String data, String location) throws Exception {
             throw new IllegalStateException("No Cloud Encryption keys available for encryption for site ID: " + siteId);
         }
 
+        final Path newSaltsFile = Files.createTempFile("salts", ".txt");
+        try (BufferedWriter w = Files.newBufferedWriter(newSaltsFile)) {
+            w.write(encryptedJson.encodePrettily());
+        }
 
-        super.upload(encryptedJson.encodePrettily(), location);
+        this.upload(newSaltsFile.toString(), location);
     }
 
     @Override
     public void upload(Object data, JsonObject extraMeta) throws Exception {
-        super.upload((RotatingSaltProvider.SaltSnapshot) data);
+        for(RotatingSaltProvider.SaltSnapshot saltSnapshot: (Collection<RotatingSaltProvider.SaltSnapshot>) data) {
+            super.upload(saltSnapshot);
+        }
     }
 
     @Override

src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java

@@ -16,6 +16,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
@@ -46,17 +47,27 @@ public void upload(RotatingSaltProvider.SaltSnapshot data) throws Exception {
         final Instant now = Instant.now();
         final long generated = now.getEpochSecond();
 
-        final JsonObject metadata = provider.getMetadata();
+        JsonObject metadata = null;
+        try {
+            metadata = provider.getMetadata();
+        } catch (Exception e) {
+            metadata = new JsonObject();
+        }
         // bump up metadata version
         metadata.put("version", versionGenerator.getVersion());
         metadata.put("generated", generated);
 
         final JsonArray snapshotsMetadata = new JsonArray();
         metadata.put("salts", snapshotsMetadata);
 
-        final List<RotatingSaltProvider.SaltSnapshot> snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data))
-                .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective))
-                .collect(Collectors.toList());
+        List<RotatingSaltProvider.SaltSnapshot> snapshots = null;
+        try {
+            snapshots = Stream.concat(provider.getSnapshots().stream(), Stream.of(data))
+                    .sorted(Comparator.comparing(RotatingSaltProvider.SaltSnapshot::getEffective))
+                    .collect(Collectors.toList());
+        } catch (Exception e) {
+            snapshots = List.of(data);
+        }
         // of the currently effective snapshots keep only the most recent one
         RotatingSaltProvider.SaltSnapshot newestEffectiveSnapshot = snapshots.stream()
                 .filter(snapshot -> snapshot.isEffective(now))
@@ -115,7 +126,7 @@ protected String getSaltSnapshotLocation(RotatingSaltProvider.SaltSnapshot snaps
         return saltSnapshotLocationPrefix + snapshot.getEffective().toEpochMilli();
     }
 
-    private void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
+    protected void uploadSaltsSnapshot(RotatingSaltProvider.SaltSnapshot snapshot, String location) throws Exception {
         // do not overwrite existing files
         if (!cloudStorage.list(location).isEmpty()) {
             // update the tags on the file to ensure it is still marked as current

src/main/java/com/uid2/admin/util/PrivateSiteUtil.java

@@ -8,6 +8,7 @@
 import com.uid2.shared.model.KeysetKey;
 import com.uid2.shared.model.SaltEntry;
 import com.uid2.shared.model.Site;
+import com.uid2.shared.store.RotatingSaltProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -312,10 +313,10 @@ public static PrivateSiteDataMap<KeysetKey> getKeysetKeys(Collection<OperatorKey
         return result;
     }
 
-    public static PrivateSiteDataMap<SaltEntry> getPrivateSaltEntries(
-            Collection<SaltEntry> globalSaltEntries,
+    public static PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> getPrivateSaltEntries(
+            Collection<RotatingSaltProvider.SaltSnapshot> globalSaltEntries,
             Collection<OperatorKey> operators) {
-        final PrivateSiteDataMap<SaltEntry> result = getPrivateSites(operators);
+        final PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> result = getPrivateSites(operators);
 
         globalSaltEntries.forEach(saltEntry -> {
             result.forEach((publicSiteId, publicSiteData) -> {

src/main/java/com/uid2/admin/util/PublicSiteUtil.java

@@ -10,6 +10,7 @@
 import com.uid2.shared.model.KeysetKey;
 import com.uid2.shared.model.SaltEntry;
 import com.uid2.shared.model.Site;
+import com.uid2.shared.store.RotatingSaltProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -130,10 +131,10 @@ public static PrivateSiteDataMap<KeysetKey> getPublicKeysetKeys(
         return result;
     }
 
-    public static PrivateSiteDataMap<SaltEntry> getPublicSaltEntries(
-            Collection<SaltEntry> globalSaltEntries,
+    public static PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> getPublicSaltEntries(
+            Collection<RotatingSaltProvider.SaltSnapshot> globalSaltEntries,
             Collection<OperatorKey> operators) {
-        final PrivateSiteDataMap<SaltEntry> result = getPublicSitesMap(operators);
+        final PrivateSiteDataMap<RotatingSaltProvider.SaltSnapshot> result = getPublicSitesMap(operators);
 
         globalSaltEntries.forEach(saltEntry -> {
             result.forEach((publicSiteId, publicSiteData) -> {