Skip to content

Commit

Permalink
Merge pull request #353 from IABTechLab/cbc-UID2-4571-cloud-encryptio…
Browse files Browse the repository at this point in the history
…n-cleanup

Adding more work on salt encryption
  • Loading branch information
cody-constine-ttd authored Dec 17, 2024
2 parents 6885a54 + 6c1bbbe commit 08b94bb
Show file tree
Hide file tree
Showing 17 changed files with 612 additions and 33 deletions.
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ CMD java \
-Djava.security.egd=file:/dev/./urandom \
-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory \
-Dlogback.configurationFile=${LOGBACK_CONF} \
-Xmx4g \
-jar ${JAR_NAME}-${JAR_VERSION}.jar
4 changes: 2 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.uid2</groupId>
<artifactId>uid2-admin</artifactId>
<version>5.16.0</version>
<version>5.16.8-alpha-103-SNAPSHOT</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand All @@ -16,7 +16,7 @@
<!-- check micrometer.version vertx-micrometer-metrics consumes before bumping up -->
<micrometer.version>1.12.2</micrometer.version>
<junit-jupiter.version>5.11.2</junit-jupiter.version>
<uid2-shared.version>8.0.6</uid2-shared.version>
<uid2-shared.version>8.0.25</uid2-shared.version>
<okta-jwt.version>0.5.10</okta-jwt.version>
<image.version>${project.version}</image.version>
</properties>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public String getId() {
public void execute() throws Exception {
PrivateSiteDataMap<LegacyClientKey> desiredPrivateState = PrivateSiteUtil.getClientKeys(globalOperators, globalClientKeys);
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, null);
PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys,globalOperators);
PrivateSiteDataMap<LegacyClientKey> desiredPublicState = PublicSiteUtil.getPublicClients(globalClientKeys, globalOperators);
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package com.uid2.admin.job.EncryptionJob;

import com.uid2.admin.job.model.Job;
import com.uid2.admin.model.PrivateSiteDataMap;
import com.uid2.admin.store.MultiScopeStoreWriter;
import com.uid2.admin.util.PublicSiteUtil;
import com.uid2.shared.auth.OperatorKey;
import com.uid2.shared.model.ClientSideKeypair;

import java.util.Collection;

public class ClientSideKeypairEncryptionJob extends Job {
private final Collection<OperatorKey> globalOperators;
private final Collection<ClientSideKeypair> globalClientSideKeypairs;

private final MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter;

public ClientSideKeypairEncryptionJob(Collection<OperatorKey> globalOperators, Collection<ClientSideKeypair> globalClientSideKeypairs,
MultiScopeStoreWriter<Collection<ClientSideKeypair>> multiScopeStoreWriter) {
this.globalOperators = globalOperators;
this.globalClientSideKeypairs = globalClientSideKeypairs;
this.multiScopeStoreWriter = multiScopeStoreWriter;
}

@Override
public String getId() {
return "cloud-encryption-sync-clientside-keypair";
}

@Override
public void execute() throws Exception {
// Only public operators support clientside keypair
PrivateSiteDataMap<ClientSideKeypair> desiredPublicState = PublicSiteUtil.getPublicClientKeypairs(globalClientSideKeypairs, globalOperators);
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, null);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package com.uid2.admin.job.EncryptionJob;

import com.uid2.admin.job.model.Job;
import com.uid2.admin.model.PrivateSiteDataMap;
import com.uid2.admin.store.MultiScopeStoreWriter;
import com.uid2.admin.util.PrivateSiteUtil;
import com.uid2.admin.util.PublicSiteUtil;
import com.uid2.shared.auth.OperatorKey;
import com.uid2.shared.model.SaltEntry;
import com.uid2.shared.store.RotatingSaltProvider;

import java.util.Collection;
import java.util.List;

public class SaltEncryptionJob extends Job {
private final Collection<OperatorKey> globalOperators;
private final Collection<RotatingSaltProvider.SaltSnapshot> saltEntries;
private final MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter;

public SaltEncryptionJob(Collection<OperatorKey> globalOperators,
Collection<RotatingSaltProvider.SaltSnapshot> saltEntries,
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> multiScopeStoreWriter) {
this.globalOperators = globalOperators;
this.saltEntries = saltEntries;
this.multiScopeStoreWriter = multiScopeStoreWriter;
}


@Override
public String getId() {
return "cloud-encryption-sync-salts";
}

@Override
public void execute() throws Exception {
List<Integer> desiredPrivateState = PrivateSiteUtil.getPrivateSaltSites(globalOperators);
multiScopeStoreWriter.uploadPrivateWithEncryption(desiredPrivateState, saltEntries, null);
List<Integer> desiredPublicState = PublicSiteUtil.getPublicSaltSites(globalOperators);
multiScopeStoreWriter.uploadPublicWithEncryption(desiredPublicState, saltEntries, null);
}
}
64 changes: 52 additions & 12 deletions src/main/java/com/uid2/admin/job/jobsync/EncryptedFilesSyncJob.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import com.fasterxml.jackson.databind.ObjectWriter;
import com.uid2.admin.job.EncryptionJob.*;
import com.uid2.admin.job.EncryptionJob.ClientKeyEncryptionJob;
import com.uid2.admin.job.model.Job;
import com.uid2.admin.store.*;
import com.uid2.admin.store.factory.*;
Expand All @@ -17,12 +16,15 @@
import com.uid2.shared.auth.RotatingOperatorKeyProvider;
import com.uid2.shared.cloud.CloudUtils;
import com.uid2.shared.cloud.ICloudStorage;
import com.uid2.shared.cloud.TaggableCloudStorage;
import com.uid2.shared.model.ClientSideKeypair;
import com.uid2.shared.model.EncryptionKey;
import com.uid2.shared.model.KeysetKey;
import com.uid2.shared.model.Site;
import com.uid2.shared.store.CloudPath;
import com.uid2.admin.legacy.LegacyClientKey;
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
import com.uid2.shared.store.EncryptedRotatingSaltProvider;
import com.uid2.shared.store.RotatingSaltProvider;
import com.uid2.shared.store.reader.RotatingCloudEncryptionKeyProvider;
import com.uid2.shared.store.scope.GlobalScope;
import io.vertx.core.json.JsonObject;
Expand All @@ -34,12 +36,12 @@
public class EncryptedFilesSyncJob extends Job {
private final JsonObject config;
private final WriteLock writeLock;
private final RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider;
private final RotatingCloudEncryptionKeyProvider rotatingCloudEncryptionKeyProvider;

public EncryptedFilesSyncJob(JsonObject config, WriteLock writeLock, RotatingCloudEncryptionKeyProvider RotatingCloudEncryptionKeyProvider) {
this.config = config;
this.writeLock = writeLock;
this.RotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
this.rotatingCloudEncryptionKeyProvider = RotatingCloudEncryptionKeyProvider;
}

@Override
Expand All @@ -49,20 +51,22 @@ public String getId() {

@Override
public void execute() throws Exception {
ICloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
TaggableCloudStorage cloudStorage = CloudUtils.createStorage(config.getString(Const.Config.CoreS3BucketProp), config);
FileStorage fileStorage = new TmpFileStorage();
ObjectWriter jsonWriter = JsonUtil.createJsonWriter();
Clock clock = new InstantClock();
VersionGenerator versionGenerator = new EpochVersionGenerator(clock);
FileManager fileManager = new FileManager(cloudStorage, fileStorage);

RotatingSaltProvider saltProvider = new RotatingSaltProvider(cloudStorage, config.getString(Const.Config.SaltsMetadataPathProp));

SiteStoreFactory siteStoreFactory = new SiteStoreFactory(
cloudStorage,
new CloudPath(config.getString(Const.Config.SitesMetadataPathProp)),
jsonWriter,
versionGenerator,
clock,
RotatingCloudEncryptionKeyProvider,
rotatingCloudEncryptionKeyProvider,
fileManager);

ClientKeyStoreFactory clientKeyStoreFactory = new ClientKeyStoreFactory(
Expand All @@ -71,15 +75,15 @@ public void execute() throws Exception {
jsonWriter,
versionGenerator,
clock,
RotatingCloudEncryptionKeyProvider,
rotatingCloudEncryptionKeyProvider,
fileManager);

EncryptionKeyStoreFactory encryptionKeyStoreFactory = new EncryptionKeyStoreFactory(
cloudStorage,
new CloudPath(config.getString(Const.Config.KeysMetadataPathProp)),
versionGenerator,
clock,
RotatingCloudEncryptionKeyProvider,
rotatingCloudEncryptionKeyProvider,
fileManager);

KeyAclStoreFactory keyAclStoreFactory = new KeyAclStoreFactory(
Expand All @@ -88,7 +92,7 @@ public void execute() throws Exception {
jsonWriter,
versionGenerator,
clock,
RotatingCloudEncryptionKeyProvider,
rotatingCloudEncryptionKeyProvider,
fileManager);

KeysetStoreFactory keysetStoreFactory = new KeysetStoreFactory(
Expand All @@ -98,7 +102,7 @@ public void execute() throws Exception {
versionGenerator,
clock,
fileManager,
RotatingCloudEncryptionKeyProvider,
rotatingCloudEncryptionKeyProvider,
config.getBoolean(enableKeysetConfigProp));

KeysetKeyStoreFactory keysetKeyStoreFactory = new KeysetKeyStoreFactory(
Expand All @@ -107,15 +111,33 @@ public void execute() throws Exception {
versionGenerator,
clock,
fileManager,
RotatingCloudEncryptionKeyProvider,
rotatingCloudEncryptionKeyProvider,
config.getBoolean(enableKeysetConfigProp));

SaltStoreFactory saltStoreFactory = new SaltStoreFactory(
config,
new CloudPath(config.getString(Const.Config.SaltsMetadataPathProp)),
fileManager,
cloudStorage,
versionGenerator,
rotatingCloudEncryptionKeyProvider
);

ClientSideKeypairStoreFactory clientSideKeypairStoreFactory = new ClientSideKeypairStoreFactory(
cloudStorage,
new CloudPath(config.getString(Const.Config.ClientSideKeypairsMetadataPathProp)),
versionGenerator,
clock,
rotatingCloudEncryptionKeyProvider,
fileManager
);

CloudPath operatorMetadataPath = new CloudPath(config.getString(Const.Config.OperatorsMetadataPathProp));
GlobalScope operatorScope = new GlobalScope(operatorMetadataPath);
RotatingOperatorKeyProvider operatorKeyProvider = new RotatingOperatorKeyProvider(cloudStorage, cloudStorage, operatorScope);

synchronized (writeLock) {
RotatingCloudEncryptionKeyProvider.loadContent();
rotatingCloudEncryptionKeyProvider.loadContent();
operatorKeyProvider.loadContent(operatorKeyProvider.getMetadata());
siteStoreFactory.getGlobalReader().loadContent(siteStoreFactory.getGlobalReader().getMetadata());
clientKeyStoreFactory.getGlobalReader().loadContent();
Expand All @@ -125,13 +147,18 @@ public void execute() throws Exception {
keysetStoreFactory.getGlobalReader().loadContent();
keysetKeyStoreFactory.getGlobalReader().loadContent();
}
saltProvider.loadContent();
clientSideKeypairStoreFactory.getGlobalReader().loadContent();
}

Collection<OperatorKey> globalOperators = operatorKeyProvider.getAll();
Collection<Site> globalSites = siteStoreFactory.getGlobalReader().getAllSites();
Collection<LegacyClientKey> globalClients = clientKeyStoreFactory.getGlobalReader().getAll();
Collection<EncryptionKey> globalEncryptionKeys = encryptionKeyStoreFactory.getGlobalReader().getSnapshot().getActiveKeySet();
Integer globalMaxKeyId = encryptionKeyStoreFactory.getGlobalReader().getMetadata().getInteger("max_key_id");
Map<Integer, EncryptionKeyAcl> globalKeyAcls = keyAclStoreFactory.getGlobalReader().getSnapshot().getAllAcls();
Collection<ClientSideKeypair> globalClientSideKeypair = clientSideKeypairStoreFactory.getGlobalReader().getAll();

MultiScopeStoreWriter<Collection<Site>> siteWriter = new MultiScopeStoreWriter<>(
fileManager,
siteStoreFactory,
Expand All @@ -148,6 +175,14 @@ public void execute() throws Exception {
fileManager,
keyAclStoreFactory,
MultiScopeStoreWriter::areMapsEqual);
MultiScopeStoreWriter<Collection<RotatingSaltProvider.SaltSnapshot>> saltWriter = new MultiScopeStoreWriter<>(
fileManager,
saltStoreFactory,
MultiScopeStoreWriter::areCollectionsEqual);
MultiScopeStoreWriter<Collection<ClientSideKeypair>> clientSideKeypairWriter = new MultiScopeStoreWriter<>(
fileManager,
clientSideKeypairStoreFactory,
MultiScopeStoreWriter::areCollectionsEqual);

SiteEncryptionJob siteEncryptionSyncJob = new SiteEncryptionJob(siteWriter, globalSites, globalOperators);
ClientKeyEncryptionJob clientEncryptionSyncJob = new ClientKeyEncryptionJob(clientWriter, globalClients, globalOperators);
Expand All @@ -160,10 +195,15 @@ public void execute() throws Exception {
encryptionKeyWriter
);
KeyAclEncryptionJob keyAclEncryptionSyncJob = new KeyAclEncryptionJob(keyAclWriter, globalOperators, globalKeyAcls);
SaltEncryptionJob saltEncryptionJob = new SaltEncryptionJob(globalOperators, saltProvider.getSnapshots(), saltWriter);
ClientSideKeypairEncryptionJob clientSideKeypairEncryptionJob = new ClientSideKeypairEncryptionJob(globalOperators, globalClientSideKeypair, clientSideKeypairWriter);

siteEncryptionSyncJob.execute();
clientEncryptionSyncJob.execute();
encryptionKeyEncryptionSyncJob.execute();
keyAclEncryptionSyncJob.execute();
saltEncryptionJob.execute();
clientSideKeypairEncryptionJob.execute();

if(config.getBoolean(enableKeysetConfigProp)) {
Map<Integer, Keyset> globalKeysets = keysetStoreFactory.getGlobalReader().getSnapshot().getAllKeysets();
Expand Down
14 changes: 14 additions & 0 deletions src/main/java/com/uid2/admin/store/MultiScopeStoreWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,13 @@ public void uploadPrivateWithEncryption(Map<Integer, T> desiredState, JsonObject
}
}

public void uploadPrivateWithEncryption(List<Integer> siteIds, T desiredState, JsonObject extraMeta) throws Exception {
EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
for (Integer siteId : siteIds) {
encryptedFactory.getEncryptedWriter(siteId,false).upload(desiredState, extraMeta);
}
}

public void uploadPublicWithEncryption(Map<Integer, T> desiredPublicState, JsonObject extraMeta) throws Exception {
EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
for (Map.Entry<Integer, T> entry : desiredPublicState.entrySet()) {
Expand All @@ -77,6 +84,13 @@ public void uploadPublicWithEncryption(Map<Integer, T> desiredPublicState, JsonO
}
}

public void uploadPublicWithEncryption(List<Integer> siteIds, T desiredState, JsonObject extraMeta) throws Exception {
EncryptedStoreFactory<T> encryptedFactory = (EncryptedStoreFactory<T>) factory;
for (Integer siteId : siteIds) {
encryptedFactory.getEncryptedWriter(siteId,true).upload(desiredState, extraMeta);
}
}

public static <K, V> boolean areMapsEqual(Map<K, V> a, Map<K, V> b) {
return a.size() == b.size() && a.entrySet().stream().allMatch(b.entrySet()::contains);
}
Expand Down
Loading

0 comments on commit 08b94bb

Please sign in to comment.