Skip to content

Commit

Permalink
[BUG] Resolve aliases in monitor input to concrete indices before com…
Browse files Browse the repository at this point in the history 
…puting ioc-containing fields from concrete index docs (#1173)

* resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs

Signed-off-by: Surya Sashank Nistala <[email protected]>

* clear indices after test

Signed-off-by: Surya Sashank Nistala <[email protected]>

---------

Signed-off-by: Surya Sashank Nistala <[email protected]>
  • Loading branch information
@eirsep
eirsep authored Jul 18, 2024
1 parent c8ad00f commit 8af8542
Show file tree
Hide file tree
Showing 6 changed files with 271 additions and 42 deletions.
11 changes: 9 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import org.opensearch.commons.alerting.model.Monitor;
import org.opensearch.commons.alerting.model.MonitorMetadata;
import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput;
import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger;

import java.util.List;
import java.util.Map;
Expand All @@ -16,14 +15,18 @@ public class IocScanContext<Data> {
private final ThreatIntelInput threatIntelInput; // deserialize threat intel input
private final List<String> indices; // user's log data indices
private final Map<String, List<String>> iocTypeToIndices;
public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List<Data> data, ThreatIntelInput threatIntelInput, List<String> indices, Map<String, List<String>> iocTypeToIndices) {
private final Map<String, List<String>> concreteIndexToMonitorInputIndicesMap;

public IocScanContext(Monitor monitor, MonitorMetadata monitorMetadata, boolean dryRun, List<Data> data, ThreatIntelInput threatIntelInput, List<String> indices, Map<String, List<String>> iocTypeToIndices, Map<String,
List<String>> concreteIndexToMonitorInputIndicesMap) {
this.monitor = monitor;
this.monitorMetadata = monitorMetadata;
this.dryRun = dryRun;
this.data = data;
this.threatIntelInput = threatIntelInput;
this.indices = indices;
this.iocTypeToIndices = iocTypeToIndices;
this.concreteIndexToMonitorInputIndicesMap = concreteIndexToMonitorInputIndicesMap;
}

public Monitor getMonitor() {
Expand All @@ -50,6 +53,10 @@ public List<String> getIndices() {
return indices;
}

public Map<String, List<String>> getConcreteIndexToMonitorInputIndicesMap() {
return concreteIndexToMonitorInputIndicesMap;
}

public Map<String, List<String>> getIocTypeToIndices() {
return iocTypeToIndices;
}
Expand Down
46 changes: 26 additions & 20 deletions ...ain/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void scanIoCs(IocScanContext<Data> iocScanContext,
Monitor monitor = iocScanContext.getMonitor();

long startTime = System.currentTimeMillis();
IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext.getThreatIntelInput().getPerIocTypeScanInputList());
IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext);
BiConsumer<List<STIX2IOC>, Exception> iocScanResultConsumer = (List<STIX2IOC> maliciousIocs, Exception e) -> {
long scanEndTime = System.currentTimeMillis();
long timeTaken = scanEndTime - startTime;
Expand Down Expand Up @@ -113,31 +113,37 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs(
* 4. doc id to iocs map (reverse mapping of 2)
*/
private IocLookupDtos extractIocsPerType
(List<Data> data, List<PerIocTypeScanInput> iocTypeToIndexFieldMappings) {
(List<Data> data, IocScanContext<Data> context) {
Map<String, Set<String>> iocsPerIocTypeMap = new HashMap<>();
Map<String, Set<String>> iocValueToDocIdMap = new HashMap<>();
Map<String, Set<String>> docIdToIocsMap = new HashMap<>();
for (Data datum : data) {
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) {
for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) {
String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase();
String index = getIndexName(datum);
List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
for (String field : fields) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
iocs.addAll(vals);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
String concreteIndex = getIndexName(datum);
if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)
&& false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty()
) {
// if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc.
String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0);
List<String> fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index);
for (String field : fields) {
List<String> vals = getValuesAsStringList(datum, field);
String id = getId(datum);
String docId = id + ":" + index;
Set<String> iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
docIdToIocsMap.put(docId, iocs);
for (String ioc : vals) {
Set<String> docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>());
docIds.add(docId);
iocValueToDocIdMap.put(ioc, docIds);
}
if (false == vals.isEmpty()) {
iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>());
iocs.addAll(vals);
iocsPerIocTypeMap.put(iocType, iocs);
}
}
}
}
Expand Down
15 changes: 13 additions & 2 deletions .../securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.opensearch.action.support.GroupedActionListener;
import org.opensearch.action.support.HandledTransportAction;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.inject.Inject;
import org.opensearch.common.settings.Settings;
Expand All @@ -34,6 +35,7 @@
import org.opensearch.securityanalytics.threatIntel.iocscan.service.SaIoCScanService;
import org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner;
import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService;
import org.opensearch.securityanalytics.util.IndexUtils;
import org.opensearch.tasks.Task;
import org.opensearch.transport.TransportService;

Expand All @@ -48,6 +50,7 @@
import java.util.function.BiConsumer;

import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelMonitorUtils.getThreatIntelInputFromBytesReference;
import static org.opensearch.securityanalytics.util.IndexUtils.getConcreteindexToMonitorInputIndicesMap;

public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAction<DocLevelMonitorFanOutRequest, DocLevelMonitorFanOutResponse> {
private static final Logger log = LogManager.getLogger(TransportThreatIntelMonitorFanOutAction.class);
Expand All @@ -60,6 +63,7 @@ public class TransportThreatIntelMonitorFanOutAction extends HandledTransportAct

private final NamedXContentRegistry xContentRegistry;
private final SaIoCScanService saIoCScanService;
private final IndexNameExpressionResolver indexNameExpressionResolver;

@Inject
public TransportThreatIntelMonitorFanOutAction(
Expand All @@ -70,7 +74,8 @@ public TransportThreatIntelMonitorFanOutAction(
Settings settings,
ActionFilters actionFilters,
SATIFSourceConfigService saTifSourceConfigService,
SaIoCScanService saIoCScanService
SaIoCScanService saIoCScanService,
IndexNameExpressionResolver indexNameExpressionResolver
) {
super(ThreatIntelMonitorRunner.FAN_OUT_ACTION_NAME, transportService, actionFilters, DocLevelMonitorFanOutRequest::new);
this.clusterService = clusterService;
Expand All @@ -79,6 +84,7 @@ public TransportThreatIntelMonitorFanOutAction(
this.settings = settings;
this.saTifSourceConfigService = saTifSourceConfigService;
this.saIoCScanService = saIoCScanService;
this.indexNameExpressionResolver = indexNameExpressionResolver;
}

@Override
Expand Down Expand Up @@ -173,14 +179,19 @@ private void onGetIocTypeToIndices(Map<String, List<String>> iocTypeToIndicesMap
actionListener.onFailure(e);
}
};
Map<String, List<String>> concreteindexToMonitorInputIndicesMap = getConcreteindexToMonitorInputIndicesMap(
remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(),
clusterService,
indexNameExpressionResolver);
saIoCScanService.scanIoCs(new IocScanContext<>(
request.getMonitor(),
request.getMonitorMetadata(),
false,
hits,
threatIntelInput,
indices,
iocTypeToIndicesMap
iocTypeToIndicesMap,
concreteindexToMonitorInputIndicesMap
), resultConsumer);
},
e -> {
Expand Down
33 changes: 27 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,6 @@
*/
package org.opensearch.securityanalytics.util;

import java.util.Optional;
import java.util.SortedMap;

import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.core.action.ActionListener;
import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.opensearch.action.support.IndicesOptions;
import org.opensearch.action.support.master.AcknowledgedResponse;
Expand All @@ -17,17 +12,22 @@
import org.opensearch.cluster.metadata.IndexAbstraction;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;
import java.util.Optional;
import java.util.SortedMap;

public class IndexUtils {

Expand Down Expand Up @@ -210,4 +210,25 @@ public static String getIndexNameWithAlias(ClusterState clusterState, String ali
return entry.map(Map.Entry::getKey).orElse(null);
}

public static Map<String, List<String>> getConcreteindexToMonitorInputIndicesMap(List<String> indices, ClusterService clusterService, IndexNameExpressionResolver resolver) {
Map<String, List<String>> result = new HashMap<>();

for (String index : indices) {
String[] concreteIndices = resolver.concreteIndexNames(
clusterService.state(),
IndicesOptions.lenientExpand(),
true,
index
);
for (String concreteIndex : concreteIndices) {
if (!result.containsKey(concreteIndex)) {
result.put(concreteIndex, new ArrayList<>());
}
result.get(concreteIndex).add(index);
}
}

return result;
}

}
69 changes: 61 additions & 8 deletions src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.common.xcontent.json.JsonXContent;
import org.opensearch.commons.ConfigConstants;
import org.opensearch.commons.alerting.model.action.Action;
import org.opensearch.commons.alerting.model.ScheduledJob;
import org.opensearch.commons.alerting.model.action.Action;
import org.opensearch.commons.alerting.util.IndexUtilsKt;
import org.opensearch.commons.rest.SecureRestClientBuilder;
import org.opensearch.core.common.Strings;
Expand Down Expand Up @@ -64,8 +64,8 @@
import org.opensearch.securityanalytics.model.CustomLogType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorInput;
import org.opensearch.securityanalytics.model.DetectorTrigger;
import org.opensearch.securityanalytics.model.DetectorRule;
import org.opensearch.securityanalytics.model.DetectorTrigger;
import org.opensearch.securityanalytics.model.Rule;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.model.threatintel.IocFinding;
Expand All @@ -75,6 +75,7 @@
import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.test.rest.OpenSearchRestTestCase;

import javax.management.MBeanServerInvocationHandler;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
Expand All @@ -96,17 +97,18 @@
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;

import static org.opensearch.action.admin.indices.create.CreateIndexRequest.MAPPINGS;
import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.MAPPER_BASE_URI;
import static org.opensearch.securityanalytics.TestHelpers.adLdapLogMappings;
import static org.opensearch.securityanalytics.TestHelpers.appLogMappings;
import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule;
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers;
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggersAndType;
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
import static org.opensearch.securityanalytics.TestHelpers.s3AccessLogMappings;
import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE;
Expand Down Expand Up @@ -753,7 +755,7 @@ private String toJsonString(IocFinding iocFinding) throws IOException {
return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS)));
}

public String toJsonString(ThreatIntelAlert alert) throws IOException {
public String toJsonString(ThreatIntelAlert alert) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder();
return IndexUtilsKt.string(shuffleXContent(alert.toXContent(builder, ToXContent.EMPTY_PARAMS)));
}
Expand Down Expand Up @@ -2006,7 +2008,7 @@ protected String createVpcFlowDetector(String indexName) throws IOException {
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -2066,7 +2068,7 @@ protected String createAdLdapDetector(String indexName) throws IOException {
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);

return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
return ((List<String>) ((Map<String, Object>) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0);
}

@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -2225,6 +2227,57 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti
}
}

protected Map<String, Map<String, Boolean>> createTestAlias(
String alias, int numOfAliasIndices, boolean includeWriteIndex
) throws IOException {
return createTestAlias(
alias,
randomAliasIndices(alias, numOfAliasIndices, includeWriteIndex),
true
);
}

protected Map<String, Map<String, Boolean>> createTestAlias(
String alias, Map<String, Boolean> indices, boolean createIndices) throws IOException {
Map<String, Boolean> indicesMap = new java.util.HashMap<>(indices);
Map<String, Map<String, Boolean>> result = new java.util.HashMap<>();
XContentBuilder indicesJson = XContentFactory.jsonBuilder()
.startObject()
.startArray("actions");
for (Map.Entry<String, Boolean> entry : indicesMap.entrySet()) {
if (createIndices)
createTestIndex(entry.getKey(), windowsIndexMapping());
boolean isWriteIndex = entry.getValue();
indicesJson.startObject()
.startObject("add")
.field("index", entry.getKey())
.field("alias", alias)
.field("is_write_index", isWriteIndex)
.endObject()
.endObject();
}
indicesJson.endArray().endObject();
makeRequest(client(), "POST", "/_aliases", Collections.emptyMap(), new StringEntity(indicesJson.toString(), ContentType.APPLICATION_JSON));
result.put(alias, indicesMap);
return result;
}


protected static Map<String, Boolean> randomAliasIndices(
String alias, int num, boolean includeWriteIndex) {
Map<String, Boolean> indices = new HashMap<>();
int writeIndex = randomIntBetween(0, num - 1);
for (int i = 0; i < num; i++) {
String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
while (indexName.equals(alias) || indices.containsKey(indexName)) {
indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
}
boolean isWriteIndex = includeWriteIndex && i == writeIndex;
indices.put(indexName, isWriteIndex);
}
return indices;
}

public static class LogIndices {
public String vpcFlowsIndex;
public String adLdapLogsIndex;
Expand Down
Loading

0 comments on commit 8af8542

Please sign in to comment.