Skip to content

Commit

Permalink
make threat intel run with standard detectors (opensearch-project#1234)
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90 @jowg-amazon
sbcd90 authored and jowg-amazon committed Aug 9, 2024
1 parent d957c80 commit c8db6dd
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 4 deletions.
4 changes: 2 additions & 2 deletions .../securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ protected void doExecute(Task task, GetThreatIntelAlertsRequest request, ActionL
SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest();
threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config");
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder));
transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap(
searchResponse -> {
Expand Down
4 changes: 2 additions & 2 deletions ...curityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, Acti
SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest();
threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config");
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD)));
boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE)));
threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder));
transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap(
searchResponse -> {
Expand Down
155 changes: 155 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,24 @@

import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.HttpStatus;
import org.junit.Assert;
import org.opensearch.client.Request;
import org.opensearch.client.Response;
import org.opensearch.client.ResponseException;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.commons.alerting.model.IntervalSchedule;
import org.opensearch.commons.alerting.model.Monitor;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.search.SearchHit;
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase;
import org.opensearch.securityanalytics.action.ListIOCsActionRequest;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.model.Detector;
import org.opensearch.securityanalytics.model.DetectorTrigger;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.threatIntel.common.RefreshType;
import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType;
Expand All @@ -37,6 +42,8 @@
import java.util.Map;

import static java.util.Collections.emptyList;
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType;
import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers;
import static org.opensearch.securityanalytics.TestHelpers.randomIndex;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH;
Expand Down Expand Up @@ -422,6 +429,154 @@ public void testCreateThreatIntelMonitor() throws IOException {
assertEquals(totalHitsVal.intValue(), 0);
}

public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException {
String index = createTestIndex(randomIndex(), windowsIndexMapping());

// Execute CreateMappingsAction to add alias mapping for index
Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
// both req params and req body are supported
createMappingRequest.setJsonEntity(
"{ \"index_name\":\"" + index + "\"," +
" \"rule_topic\":\"" + randomDetectorType() + "\", " +
" \"partial\":true" +
"}"
);

Response response = client().performRequest(createMappingRequest);
assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());

Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())));

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));

Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
Map<String, Object> responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(0, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());
List<String> vals = List.of("ip1", "ip2");
indexSourceConfigsAndIocs(1, vals);
String monitorName = "test_monitor_name";


/**create monitor */
ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index);
response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
Assert.assertEquals(201, response.getStatusLine().getStatusCode());
Map<String, Object> responseBody = asMap(response);

try {
makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor));
fail();
} catch (Exception e) {
/** creating a second threat intel monitor should fail*/
assertTrue(e.getMessage().contains("already exists"));
}

final String monitorId = responseBody.get("id").toString();
Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId);

Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId);
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
int i = 1;
for (String val : vals) {
String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(index, "" + i++, doc);
} catch (IOException e) {
fail();
}
}

Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap());
Map<String, Object> executeResults = entityAsMap(executeResponse);
assertEquals(1, 1);

String matchAllRequest = getMatchAllRequest();
Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
HashMap<String, Object> hits = (HashMap<String, Object>) asMap(searchMonitorResponse).get("hits");
HashMap<String, Object> totalHits = (HashMap<String, Object>) hits.get("total");
Integer totalHitsVal = (Integer) totalHits.get("value");
assertEquals(totalHitsVal.intValue(), 1);
makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));


iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(2, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());

//alerts
List<SearchHit> searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest);
Assert.assertEquals(4, searchHits.size());

for (String val : vals) {
String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val);
try {
indexDoc(index, "" + i++, doc);
} catch (IOException e) {
fail();
}
}
executeAlertingMonitor(monitorId, Collections.emptyMap());
iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search",
Map.of(), null);
responseAsMap = responseAsMap(iocFindingsResponse);
Assert.assertEquals(4, ((List<Map<String, Object>>) responseAsMap.get("ioc_findings")).size());

// Use ListIOCs API to confirm expected number of findings are returned
String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0");
Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null);
Map<String, Object> listIocsResponseMap = responseAsMap(listIocsResponse);
List<Map<String, Object>> iocsMap = (List<Map<String, Object>>) listIocsResponseMap.get("iocs");
assertEquals(2, iocsMap.size());
iocsMap.forEach((iocDetails) -> {
String iocId = (String) iocDetails.get("id");
int numFindings = (Integer) iocDetails.get("num_findings");
assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId())));
assertEquals(2, numFindings);
});

//alerts via system index search
searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest);
Assert.assertEquals(4, searchHits.size());

// alerts via API
Map<String, String> params = new HashMap<>();
Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null);
Map<String, Object> getAlertsBody = asMap(getAlertsResponse);
Assert.assertEquals(4, getAlertsBody.get("total_alerts"));


ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto(
monitorId,
iocScanMonitor.getName() + "update",
iocScanMonitor.getPerIocTypeScanInputList(),
new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()),
false,
null,
List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1))
);
//update monitor
response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto));
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
responseBody = asMap(response);
assertEquals(responseBody.get("id").toString(), monitorId);
assertEquals(((HashMap<String, Object>) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update");

//delete
Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null);
Assert.assertEquals(200, delete.getStatusLine().getStatusCode());

searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON));
Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode());
hits = (HashMap<String, Object>) asMap(searchMonitorResponse).get("hits");
totalHits = (HashMap<String, Object>) hits.get("total");
totalHitsVal = (Integer) totalHits.get("value");
assertEquals(totalHitsVal.intValue(), 0);
}

public void testCreateThreatIntelMonitor_invalidMonitorJson() throws IOException {
ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto("test-index");

Expand Down

0 comments on commit c8db6dd

Please sign in to comment.