Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Attribute anonymizer #235

Draft
wants to merge 7 commits into
base: master
Choose a base branch
from
Draft

Attribute anonymizer #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9ae53df
simple attribute anonymizer
madox2 Nov 19, 2024
68ddf60
activation export
madox2 Nov 20, 2024
d6deed6
sequential anonymization support
madox2 Nov 20, 2024
5aef17c
disable attributes option
madox2 Nov 20, 2024
10295ff
anonymize archetype ref
madox2 Nov 20, 2024
a49fcdb
exclude attributes option
madox2 Nov 21, 2024
859fae5
Merge remote-tracking branch 'origin/master' into attribute-anonymizer
madox2 Nov 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 60 additions & 3 deletions infra/common/src/main/java/com/evolveum/midpoint/common/RoleMiningExportUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.Base64;
import java.util.List;
import java.util.UUID;
import java.util.*;
import javax.crypto.Cipher;
import javax.crypto.spec.SecretKeySpec;

Expand Down Expand Up @@ -62,6 +60,59 @@ public String getDisplayString() {
}
}

public static class SequentialAnonymizer {
private final Map<String, String> anonymizedValues = new HashMap<>();
private final String baseName;
private int index = 0;

public SequentialAnonymizer(String baseName) {
this.baseName = baseName;
}

public String anonymize(String value) {
if (!anonymizedValues.containsKey(value)) {
anonymizedValues.put(value, baseName + index++);
}
return anonymizedValues.get(value);
}
}

private static class ScopedSequentialAnonymizer {
private final Map<String, SequentialAnonymizer> scopedAnonymizers = new HashMap<>();
private final String baseName;

public ScopedSequentialAnonymizer(String baseName) {
this.baseName = baseName;
}

public String anonymize(String scope, String value) {
if (!scopedAnonymizers.containsKey(scope)) {
scopedAnonymizers.put(scope, new SequentialAnonymizer(baseName));
}
return scopedAnonymizers.get(scope).anonymize(value);
}
}

public static class AttributeValueAnonymizer {
private final ScopedSequentialAnonymizer sequentialAnonymizer = new ScopedSequentialAnonymizer("att");
private final NameMode nameMode;
private final String encryptKey;

public AttributeValueAnonymizer(NameMode nameMode, String encryptKey) {
this.nameMode = nameMode;
this.encryptKey = encryptKey;
}

public String anonymize(String attributeName, String attributeValue) {
var anonymized = switch(nameMode) {
case ENCRYPTED -> encrypt(attributeValue, encryptKey);
case SEQUENTIAL -> sequentialAnonymizer.anonymize(attributeName, attributeValue);
case ORIGINAL -> attributeValue;
};
return anonymized + EXPORT_SUFFIX;
}
}

private static PolyStringType encryptName(String name, int iterator, String prefix, @NotNull NameMode nameMode, String key) {
if (nameMode.equals(NameMode.ENCRYPTED)) {
return PolyStringType.fromOrig(encrypt(name, key) + EXPORT_SUFFIX);
Expand All @@ -85,6 +136,12 @@ public static PolyStringType encryptRoleName(String name, int iterator, NameMode
return encryptName(name, iterator, "Role", nameMode, key);
}

public static ObjectReferenceType encryptObjectReference(ObjectReferenceType targetRef, SecurityMode securityMode, String key) {
ObjectReferenceType encryptedTargetRef = targetRef.clone();
encryptedTargetRef.setOid(encryptedUUID(encryptedTargetRef.getOid(), securityMode, key));
return encryptedTargetRef;
}

public static AssignmentType encryptObjectReference(@NotNull AssignmentType assignmentObject,
SecurityMode securityMode, String key) {
ObjectReferenceType encryptedTargetRef = assignmentObject.getTargetRef();
Expand Down
154 changes: 151 additions & 3 deletions ...a/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningConsumerWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,18 @@

import java.io.IOException;
import java.io.Writer;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.stream.Stream;
import javax.xml.namespace.QName;

import com.evolveum.midpoint.common.mining.utils.RoleAnalysisAttributeDefUtils;
import com.evolveum.midpoint.prism.*;
import com.evolveum.midpoint.prism.path.ItemPath;
import com.evolveum.midpoint.util.DOMUtil;
import com.evolveum.midpoint.util.logging.Trace;

import com.evolveum.midpoint.util.logging.TraceManager;
Expand All @@ -28,9 +36,6 @@
import com.evolveum.midpoint.ninja.util.FileReference;
import com.evolveum.midpoint.ninja.util.NinjaUtils;
import com.evolveum.midpoint.ninja.util.OperationStatus;
import com.evolveum.midpoint.prism.PrismContainerValue;
import com.evolveum.midpoint.prism.PrismSerializer;
import com.evolveum.midpoint.prism.SerializationOptions;
import com.evolveum.midpoint.prism.query.ObjectFilter;
import com.evolveum.midpoint.prism.query.ObjectQuery;
import com.evolveum.midpoint.schema.result.OperationResult;
Expand All @@ -49,8 +54,12 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker<Exp
private int processedRoleIterator = 0;
private int processedUserIterator = 0;
private int processedOrgIterator = 0;
private final SequentialAnonymizer defaultAttributeNameAnonymizer = new SequentialAnonymizer("default_attr");
private final SequentialAnonymizer extensionAttributeNameAnonymizer = new SequentialAnonymizer("extension_attr");
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please verify if this work on multiple threads.

private AttributeValueAnonymizer attributeValuesAnonymizer;

private boolean orgAllowed;
private boolean attributesAllowed;
private boolean firstObject = true;
private boolean jsonFormat = false;

Expand All @@ -68,6 +77,16 @@ public class ExportMiningConsumerWorker extends AbstractWriterConsumerWorker<Exp
private List<String> businessRolePrefix;
private List<String> businessRoleSuffix;


private Set<ItemPath> attrPathsUser;
private Set<ItemPath> attrPathsRole;
private Set<ItemPath> attrPathsOrg;

private static final String ARCHETYPE_REF_ATTRIBUTE_NAME = "archetypeRef";
private static final List<String> DEFAULT_EXCLUDED_ATTRIBUTES = List.of("description", "documentation", "emailAddress",
"telephoneNumber", "fullName", "givenName", "familyName", "additionalName", "nickName", "personalNumber", "identifier",
"jpegPhoto");

public ExportMiningConsumerWorker(NinjaContext context, ExportMiningOptions options, BlockingQueue<FocusType> queue,
OperationStatus operation) {
super(context, options, queue, operation);
Expand All @@ -81,9 +100,16 @@ protected void init() {
securityMode = options.getSecurityLevel();
encryptKey = RoleMiningExportUtils.updateEncryptKey(securityMode);
orgAllowed = options.isIncludeOrg();
attributesAllowed = options.isIncludeAttributes();

nameMode = options.getNameMode();

attrPathsUser = extractDefaultAttributePaths(UserType.COMPLEX_TYPE, options.getExcludedAttributesUser());
attrPathsRole = extractDefaultAttributePaths(RoleType.COMPLEX_TYPE, options.getExcludedAttributesUser());
attrPathsOrg = extractDefaultAttributePaths(OrgType.COMPLEX_TYPE, options.getExcludedAttributesOrg());

attributeValuesAnonymizer = new AttributeValueAnonymizer(nameMode, encryptKey);

SerializationOptions serializationOptions = SerializationOptions.createSerializeForExport()
.serializeReferenceNames(true)
.serializeForExport(true)
Expand Down Expand Up @@ -142,6 +168,10 @@ private void write(Writer writer, PrismContainerValue<?> prismContainerValue) th
@NotNull
private OrgType getPreparedOrgObject(@NotNull FocusType object) {
OrgType org = new OrgType();

fillAttributes(object, org, attrPathsOrg, options.getExcludedAttributesOrg());
fillActivation(object, org);

org.setName(encryptOrgName(object.getName().toString(), processedOrgIterator++, nameMode, encryptKey));
org.setOid(encryptedUUID(object.getOid(), securityMode, encryptKey));

Expand Down Expand Up @@ -172,6 +202,8 @@ && filterAllowedOrg(oid)) {
@NotNull
private UserType getPreparedUserObject(@NotNull FocusType object) {
UserType user = new UserType();
fillAttributes(object, user, attrPathsUser, options.getExcludedAttributesUser());
fillActivation(object, user);

List<AssignmentType> assignment = object.getAssignment();
if (assignment == null || assignment.isEmpty()) {
Expand Down Expand Up @@ -212,6 +244,9 @@ && filterAllowedRole(oid)) {
@NotNull
private RoleType getPreparedRoleObject(@NotNull FocusType object) {
RoleType role = new RoleType();
fillAttributes(object, role, attrPathsRole, options.getExcludedAttributesRole());
fillActivation(object, role);

String roleName = object.getName().toString();
PolyStringType encryptedName = encryptRoleName(roleName, processedRoleIterator++, nameMode, encryptKey);
role.setName(encryptedName);
Expand Down Expand Up @@ -342,4 +377,117 @@ private void loadRoleCategoryIdentifiers() {

return null;
}

private Set<ItemPath> extractDefaultAttributePaths(QName type, List<String> excludedDefaultAttributes) {
var def = PrismContext.get().getSchemaRegistry().findContainerDefinitionByType(type);
var excludedAttributes = Stream.concat(excludedDefaultAttributes.stream(), DEFAULT_EXCLUDED_ATTRIBUTES.stream()).toList();
return extractAttributePaths(def, excludedAttributes);
}


private Set<ItemPath> extractAttributePaths(PrismContainerDefinition<?> containerDef, List<String> excludedAttributeNames) {
Set<ItemPath> paths = new HashSet<>();
for (ItemDefinition<?> def : containerDef.getDefinitions()) {
var attributeName = def.getItemName().toString();
if (excludedAttributeNames.contains(attributeName)) {
continue;
}
var isArchetypeRef = attributeName.equals(ARCHETYPE_REF_ATTRIBUTE_NAME);
if (!isArchetypeRef && (def.isOperational() || !def.isSingleValue())) {
// unsupported types
continue;
}
if (def instanceof PrismReferenceDefinition refDef) {
paths.add(refDef.getItemName());
}
if (def instanceof PrismPropertyDefinition<?> propertyDef && RoleAnalysisAttributeDefUtils.isSupportedPropertyType(propertyDef.getTypeClass())) {
paths.add(propertyDef.getItemName());
}
}
return paths;
}

private Object anonymizeAttributeValue(Item<?, ?> item) {
var def = item.getDefinition();
var type = def.getTypeClass();
var realValue = Objects.requireNonNull(item.getRealValue());

if (def instanceof PrismReferenceDefinition) {
var referenceValue = (ObjectReferenceType) realValue;
return encryptObjectReference(referenceValue, securityMode, encryptKey);
}

// NOTE: do not encrypt ordinal values and booleans
var shouldAnonymizeValue = !List.of(Integer.class, Double.class, Boolean.class).contains(type);

if (shouldAnonymizeValue) {
var attributeName = item.getDefinition().getItemName().toString();
return attributeValuesAnonymizer.anonymize(attributeName, realValue.toString());
}
return realValue;
}

public String anonymizeAttributeName(Item<?, ?> item, SequentialAnonymizer attributeNameAnonymizer) {
String originalAtributeName = item.getDefinition().getItemName().toString();
if (nameMode.equals(NameMode.ORIGINAL)) {
return originalAtributeName;
}
return attributeNameAnonymizer.anonymize(originalAtributeName);
}

private void anonymizeAttribute(FocusType newObject, PrismContainer<?> itemContainer, ItemPath path, SequentialAnonymizer attributeNameAnonymizer) {
Item<?, ?> item = itemContainer.findItem(path);
if (item == null || item.getRealValue() == null) {
return;
}
try {
String anonymizedAttributeName = anonymizeAttributeName(item, attributeNameAnonymizer);
Object anonymizedAttributeValue = anonymizeAttributeValue(item);

QName propertyName = new QName(item.getDefinition().getItemName().getNamespaceURI(), anonymizedAttributeName);
PrismPropertyDefinition<Object> propertyDefinition = context
.getPrismContext()
.definitionFactory()
.newPropertyDefinition(propertyName, DOMUtil.XSD_STRING);
PrismProperty<Object> anonymizedProperty = propertyDefinition.instantiate();
anonymizedProperty.setRealValue(anonymizedAttributeValue);
newObject.asPrismObject().addExtensionItem(anonymizedProperty);
} catch (Exception e) {
LOGGER.error("Failed to clone item for {} object {}. ", itemContainer.getClass(), item, e);
}
}

private void fillAttributes(
@NotNull FocusType origObject,
@NotNull FocusType newObject,
@NotNull Set<ItemPath> defaultAttributePaths,
@NotNull List<String> excludedAttributes
) {
if (!attributesAllowed) {
return;
}
var origContainer = origObject.asPrismObject();
newObject.extension(new ExtensionType());

for (var path: defaultAttributePaths) {
anonymizeAttribute(newObject, origContainer, path, defaultAttributeNameAnonymizer);
}

if (origContainer.getExtension() != null) {
PrismContainerDefinition<?> definition = origContainer.getExtensionContainerValue().getDefinition();
var extensionAttributePaths = extractAttributePaths(definition, excludedAttributes);
for (var path: extensionAttributePaths) {
anonymizeAttribute(newObject, origContainer.getExtension(), path, extensionAttributeNameAnonymizer);
}
}
}

private void fillActivation(@NotNull FocusType origObject, @NotNull FocusType newObject) {
if (origObject.getActivation() == null) {
return;
}
var activation = new ActivationType().effectiveStatus(origObject.getActivation().getEffectiveStatus());
newObject.setActivation(activation);
}

}
42 changes: 42 additions & 0 deletions tools/ninja/src/main/java/com/evolveum/midpoint/ninja/action/mining/ExportMiningOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

import com.evolveum.midpoint.common.RoleMiningExportUtils;
import com.evolveum.midpoint.ninja.action.BasicExportOptions;
import com.evolveum.midpoint.ninja.util.ItemPathConverter;
import com.evolveum.midpoint.prism.path.ItemPath;

@Parameters(resourceBundle = "messages", commandDescriptionKey = "exportMining")
public class ExportMiningOptions extends BaseMiningOptions implements BasicExportOptions {
Expand All @@ -35,6 +37,11 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor
public static final String P_SUFFIX_BUSINESS_LONG = "--business-role-suffix";
public static final String P_ORG = "-do";
public static final String P_ORG_LONG = "--disable-org";
public static final String P_ATTRIBUTE = "-da";
public static final String P_ATTRIBUTE_LONG = "--disable-attribute";
public static final String P_EXCLUDE_ATTRIBUTES_USER_LONG = "--exclude-user-attribute";
public static final String P_EXCLUDE_ATTRIBUTES_ROLE_LONG = "--exclude-role-attribute";
public static final String P_EXCLUDE_ATTRIBUTES_ORG_LONG = "--exclude-org-attribute";
public static final String P_NAME_OPTIONS = "-nm";
public static final String P_NAME_OPTIONS_LONG = "--name-mode";
public static final String P_ARCHETYPE_OID_APPLICATION_LONG = "--application-role-archetype-oid";
Expand Down Expand Up @@ -66,6 +73,9 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor
@Parameter(names = { P_ORG, P_ORG_LONG }, descriptionKey = "export.prevent.org")
private boolean disableOrg = false;

@Parameter(names = { P_ATTRIBUTE, P_ATTRIBUTE_LONG }, descriptionKey = "export.prevent.attribute")
private boolean disableAttribute = false;

@Parameter(names = { P_NAME_OPTIONS, P_NAME_OPTIONS_LONG }, descriptionKey = "export.name.options")
private RoleMiningExportUtils.NameMode nameMode = RoleMiningExportUtils.NameMode.SEQUENTIAL;

Expand All @@ -77,6 +87,18 @@ public class ExportMiningOptions extends BaseMiningOptions implements BasicExpor
descriptionKey = "export.business.role.archetype.oid")
private String businessRoleArchetypeOid = "00000000-0000-0000-0000-000000000321";

@Parameter(names = { P_EXCLUDE_ATTRIBUTES_USER_LONG }, descriptionKey = "export.exclude.attributes.user",
validateWith = ItemPathConverter.class, converter = ItemPathConverter.class)
private List<ItemPath> excludedAttributesUser = new ArrayList<>();

@Parameter(names = { P_EXCLUDE_ATTRIBUTES_ROLE_LONG }, descriptionKey = "export.exclude.attributes.role",
validateWith = ItemPathConverter.class, converter = ItemPathConverter.class)
private List<ItemPath> excludedAttributesRole = new ArrayList<>();

@Parameter(names = { P_EXCLUDE_ATTRIBUTES_ORG_LONG }, descriptionKey = "export.exclude.attributes.org",
validateWith = ItemPathConverter.class, converter = ItemPathConverter.class)
private List<ItemPath> excludedAttributesOrg = new ArrayList<>();

public RoleMiningExportUtils.SecurityMode getSecurityLevel() {
return securityMode;
}
Expand All @@ -85,6 +107,10 @@ public boolean isIncludeOrg() {
return !disableOrg;
}

public boolean isIncludeAttributes() {
return !disableAttribute;
}

public String getApplicationRoleArchetypeOid() {
return applicationRoleArchetypeOid;
}
Expand Down Expand Up @@ -136,4 +162,20 @@ public List<String> getBusinessRoleSuffix() {
String[] separateSuffixes = businessRoleSuffix.split(DELIMITER);
return new ArrayList<>(Arrays.asList(separateSuffixes));
}

private List<String> itemPathsToStrings(List<ItemPath> itemPaths) {
return itemPaths.stream().map(ItemPath::toString).toList();
}

public List<String> getExcludedAttributesUser() {
return itemPathsToStrings(excludedAttributesUser);
}

public List<String> getExcludedAttributesRole() {
return itemPathsToStrings(excludedAttributesRole);
}

public List<String> getExcludedAttributesOrg() {
return itemPathsToStrings(excludedAttributesOrg);
}
}
Loading