Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding various OCSF 1.1 fields to log type static mappings #1403

Merged
merged 5 commits into from
Nov 25, 2024
Merged

Adding various OCSF 1.1 fields to log type static mappings #1403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ccfca01
Adding various OCSF 1.1 fields to log type static mappings
toepkerd-zz Oct 30, 2024
b839147
fixing IT failures
toepkerd-zz Oct 30, 2024
eaffdc5
removed vestigial exception throw
toepkerd-zz Oct 30, 2024
b0d09f0
turning all ocsf 1.0 replacements with additions
toepkerd-zz Nov 11, 2024
c5c3798
fixed ITs
toepkerd-zz Nov 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,9 @@ private List<FieldMappingDoc> createFieldMappingDocs(List<LogType> logTypes) {
if (mapping.getOcsf() != null) {
schemaFields.put("ocsf", mapping.getOcsf());
}
if (mapping.getOcsf11() != null) {
schemaFields.put("ocsf11", mapping.getOcsf11());
}
fieldMappingMap.put(
key,
new FieldMappingDoc(
Expand All @@ -574,6 +577,7 @@ private List<FieldMappingDoc> createFieldMappingDocs(List<LogType> logTypes) {
} else {
// merge with existing doc
existingDoc.getSchemaFields().put("ocsf", mapping.getOcsf());
existingDoc.getSchemaFields().put("ocsf11", mapping.getOcsf11());
existingDoc.getLogTypes().add(logType.getName());
}
}));
Expand Down Expand Up @@ -702,7 +706,7 @@ public void getRuleFieldMappingsAllSchemas(String logType, ActionListener<List<L
(delegatedListener, fieldMappingDocs) -> {
List<LogType.Mapping> ruleFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( e -> {
ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf")));
ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf"), e.getSchemaFields().get("ocsf11")));
});
delegatedListener.onResponse(ruleFieldMappings);
}
Expand All @@ -725,7 +729,8 @@ public void getRequiredFields(String logType, ActionListener<List<LogType.Mappin
LogType.Mapping requiredField = new LogType.Mapping(
e.getRawField(),
e.getSchemaFields().get(defaultSchemaField),
e.getSchemaFields().get("ocsf")
e.getSchemaFields().get("ocsf"),
e.getSchemaFields().get("ocsf11")
);
requiredFields.add(requiredField);
});
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ public void onResponse(List<LogType.Mapping> mappings) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
} else if (indexFields.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (indexFields.contains(mapping.getOcsf11())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11()));
}
}
aliasMappingsObj.field("properties", aliasMappingFields);
Expand Down Expand Up @@ -483,6 +485,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
String alias = requiredField.getEcs();
String rawPath = requiredField.getRawField();
String ocsfPath = requiredField.getOcsf();
String ocsf11Path = requiredField.getOcsf11();
if (allFieldsFromIndex.contains(rawPath)) {
// if the alias was already added into applyable aliases, then skip to avoid duplicates
if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) {
Expand All @@ -497,6 +500,9 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
} else if (allFieldsFromIndex.contains(ocsf11Path)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsf11Path);
} else if ((alias == null && allFieldsFromIndex.contains(rawPath) == false) || allFieldsFromIndex.contains(alias) == false) {
if (alias != null) {
// we don't want to send back aliases which have same name as existing field in index
Expand All @@ -520,6 +526,8 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
for (LogType.Mapping mapping : requiredFields) {
if (allFieldsFromIndex.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (allFieldsFromIndex.contains(mapping.getOcsf11())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11()));
} else if (mapping.getEcs() != null) {
shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases);
} else if (mapping.getEcs() == null) {
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/model/LogType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public class LogType implements Writeable {
private static final String RAW_FIELD = "raw_field";
public static final String ECS = "ecs";
public static final String OCSF = "ocsf";
public static final String OCSF11 = "ocsf11";
public static final String IOC_FIELDS = "ioc_fields";
public static final String IOC = "ioc";
public static final String FIELDS = "fields";
Expand Down Expand Up @@ -67,7 +68,7 @@ public LogType(Map<String, Object> logTypeAsMap) {
if (mappings.size() > 0) {
this.mappings = new ArrayList<>(mappings.size());
this.mappings = mappings.stream().map(e ->
new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF))
new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF), e.get(OCSF11))
).collect(Collectors.toList());
}
if (logTypeAsMap.containsKey(IOC_FIELDS)) {
Expand Down Expand Up @@ -120,17 +121,20 @@ public static class Mapping implements Writeable {
private String rawField;
private String ecs;
private String ocsf;
private String ocsf11;

public Mapping(StreamInput sin) throws IOException {
this.rawField = sin.readString();
this.ecs = sin.readOptionalString();
this.ocsf = sin.readOptionalString();
this.ocsf11 = sin.readOptionalString();
}

public Mapping(String rawField, String ecs, String ocsf) {
public Mapping(String rawField, String ecs, String ocsf, String ocsf11) {
this.rawField = rawField;
this.ecs = ecs;
this.ocsf = ocsf;
this.ocsf11 = ocsf11;
}

public String getRawField() {
Expand All @@ -145,11 +149,14 @@ public String getOcsf() {
return ocsf;
}

public String getOcsf11() { return ocsf11; }

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(rawField);
out.writeOptionalString(ecs);
out.writeOptionalString(ocsf);
out.writeOptionalString(ocsf11);
}

public static Mapping readFrom(StreamInput sin) throws IOException {
Expand Down
24 changes: 16 additions & 8 deletions src/main/resources/OSMapping/cloudtrail_logtype.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
{
"raw_field":"eventType",
"ecs":"aws.cloudtrail.event_type",
"ocsf": "unmapped.eventType"
"ocsf" : "unmapped.eventType",
"ocsf11": "metadata.event_code"
},
{
"raw_field":"eventCategory",
Expand Down Expand Up @@ -69,7 +70,8 @@
{
"raw_field":"additionalEventData.MFAUsed",
"ecs":"aws.cloudtrail.additional_event_data.mfaUsed",
"ocsf": "mfa"
"ocsf": "mfa",
"ocsf11": "is_mfa"
},
{
"raw_field":"responseElements",
Expand Down Expand Up @@ -124,12 +126,14 @@
{
"raw_field":"requestParameters.userName",
"ecs":"aws.cloudtrail.request_parameters.username",
"ocsf": "unmapped.requestParameters.userName"
"ocsf": "unmapped.requestParameters.userName",
"ocsf11": "user.name"
},
{
"raw_field":"requestParameters.roleArn",
"ecs":"aws.cloudtrail.request_parameters.roleArn",
"ocsf": "user.uuid"
"ocsf": "user.uuid",
"ocsf11": "user.uid"
},
{
"raw_field":"requestParameters.roleSessionName",
Expand All @@ -149,17 +153,20 @@
{
"raw_field":"userIdentity.principalId",
"ecs":"aws.cloudtrail.user_identity.principalId",
"ocsf": "actor.user.uid"
"ocsf": "actor.user.uid",
"ocsf11":"actor.user.uid_alt"
},
{
"raw_field":"userIdentity.arn",
"ecs":"aws.cloudtrail.user_identity.arn",
"ocsf": "actor.user.uuid"
"ocsf": "actor.user.uuid",
"ocsf11": "actor.user.uid"
},
{
"raw_field":"userIdentity.accountId",
"ecs":"aws.cloudtrail.user_identity.accountId",
"ocsf": "actor.user.account_uid"
"ocsf": "actor.user.account_uid",
"ocsf11": "actor.user.account.uid"
},
{
"raw_field":"userIdentity.accessKeyId",
Expand Down Expand Up @@ -199,7 +206,8 @@
{
"raw_field":"userIdentity.sessionContext.attributes.mfaAuthenticated",
"ecs":"aws.cloudtrail.user_identity.session_context.attributes.mfaAuthenticated",
"ocsf": "actor.session.mfa"
"ocsf": "actor.session.mfa",
"ocsf11": "actor.session.is_mfa"
},
{
"raw_field":"userIdentity.webIdFederationData.federatedProvider",
Expand Down
16 changes: 14 additions & 2 deletions src/main/resources/OSMapping/dns_logtype.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
{
"raw_field":"account_id",
"ecs":"aws.route53.account_id",
"ocsf": "cloud.account_uid"
"ocsf": "cloud.account_uid",
"ocsf11": "cloud.account.uid"
},
{
"raw_field":"region",
Expand Down Expand Up @@ -114,12 +115,23 @@
{
"raw_field":"firewall_rule_action",
"ecs":"aws.route53.srcids.firewall_rule_action",
"ocsf": "disposition_id"
"ocsf": "disposition_id",
"ocsf11": "disposition"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this match with the ocsf mapping?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you're referring to replacing the "ocsf" field to have value "disposition" instead of "disposition_id" (and not introducing the "ocsf11" field at all), this runs the risk of creating mapping conflicts with existing customer detectors who already used the old static mappings to create that detector's mappings alias indices. Adding instead of replacing fields (despite the fact that both disposition and disposition_id exist in both OCSF1.0 and OCSF1.1) works around this.

},
{
"raw_field":"creationTime",
"ecs":"timestamp",
"ocsf": "unmapped.creationTime"
},
{
"raw_field":"rcode",
"ecs":"aws.route53.rcode",
"ocsf":"rcode"
},
{
"raw_field":"firewall_rule_group_id",
"ecs":"aws.route53.srcids.firewall_rule_group_id",
"ocsf":"firewall_rule.uid"
}
]
}
Loading
Loading