Skip to content

Commit

Permalink
test detector updation when feed updation job runs
Browse files Browse the repository at this point in the history 
Signed-off-by: Surya Sashank Nistala <[email protected]>
  • Loading branch information
@eirsep
eirsep committed Oct 20, 2023
1 parent 7a24bd0 commit 74a7440
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 25 deletions.
64 changes: 42 additions & 22 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,15 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.search.join.ScoreMode;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.client.Client;
import org.opensearch.commons.alerting.model.DocLevelQuery;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.rest.RestRequest;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.action.IndexDetectorAction;
Expand Down Expand Up @@ -144,28 +147,45 @@ private static String constructId(Detector detector, String iocType) {
}

public void updateDetectorsWithLatestThreatIntelRules() {
//todo : fix query for fetching detectors with threat intel enabled = true
// String searchReq = "{ \"query\": { \"match\": { \"detector.threat_intel_enabled\": true } } }";
SearchRequest searchRequest = new SearchRequest(DETECTORS_INDEX);
SearchSourceBuilder ssb = searchRequest.source();
ssb.size(9999);
client.execute(SearchDetectorAction.INSTANCE, new SearchDetectorRequest(new SearchRequest().source(ssb)),
ActionListener.wrap(r -> {
List<Detector> detectors = getDetectors(r, xContentRegistry);
detectors.forEach(detector -> {
assert detector.getThreatIntelEnabled();
client.execute(IndexDetectorAction.INSTANCE, new IndexDetectorRequest(
detector.getId(), WriteRequest.RefreshPolicy.IMMEDIATE,
RestRequest.Method.PUT,
detector),
ActionListener.wrap(
res -> log.debug("updated {} with latest threat intel info", res.getDetector().getId()),
e -> log.error(() -> new ParameterizedMessage("Failed to update detector {} with latest threat intel info", detector.getId()), e)));
}
);
}, e -> {
log.error("Failed to fetch detectors to update with threat intel queries.", e);
}));
try {
QueryBuilder queryBuilder =
QueryBuilders.nestedQuery("detector",
QueryBuilders.boolQuery().must(
QueryBuilders.matchQuery("detector.threat_intel_enabled", true)
), ScoreMode.Avg);
SearchRequest searchRequest = new SearchRequest(DETECTORS_INDEX);
SearchSourceBuilder ssb = searchRequest.source();
ssb.query(queryBuilder);
ssb.size(9999);
CountDownLatch countDownLatch = new CountDownLatch(1);
client.execute(SearchDetectorAction.INSTANCE, new SearchDetectorRequest(searchRequest),
ActionListener.wrap(r -> {
List<Detector> detectors = getDetectors(r, xContentRegistry);
detectors.forEach(detector -> {
assert detector.getThreatIntelEnabled();
client.execute(IndexDetectorAction.INSTANCE, new IndexDetectorRequest(
detector.getId(), WriteRequest.RefreshPolicy.IMMEDIATE,
RestRequest.Method.PUT,
detector),
ActionListener.wrap(
res -> {
log.debug("updated {} with latest threat intel info", res.getDetector().getId());
countDownLatch.countDown();
},
e -> {
log.error(() -> new ParameterizedMessage("Failed to update detector {} with latest threat intel info", detector.getId()), e);
countDownLatch.countDown();
}));
}
);
}, e -> {
log.error("Failed to fetch detectors to update with threat intel queries.", e);
countDownLatch.countDown();
}));
countDownLatch.await();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}


}
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.transport.TransportService;

import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -449,7 +450,7 @@ public void onResponse(Map<String, Map<String, String>> ruleFieldMappings) {
Collectors.toList());

// Process doc level monitors
if (!docLevelRules.isEmpty()) {
if (!docLevelRules.isEmpty() || detector.getThreatIntelEnabled()) {
if (detector.getDocLevelMonitorId() == null) {
monitorsToBeAdded.add(createDocLevelMonitorRequest(docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST));
} else {
Expand Down Expand Up @@ -1452,6 +1453,7 @@ public void indexDetector() throws Exception {
.source(request.getDetector().toXContentWithUser(XContentFactory.jsonBuilder(), new ToXContent.MapParams(Map.of("with_type", "true"))))
.timeout(indexTimeout);
} else {
request.getDetector().setLastUpdateTime(Instant.now());
indexRequest = new IndexRequest(Detector.DETECTORS_INDEX)
.setRefreshPolicy(request.getRefreshPolicy())
.source(request.getDetector().toXContentWithUser(XContentFactory.jsonBuilder(), new ToXContent.MapParams(Map.of("with_type", "true"))))
Expand Down
10 changes: 8 additions & 2 deletions .../java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ public void testCreateDetector_threatIntelEnabled_testJobRunner() throws IOExcep

Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));


String request = "{\n" +
" \"query\" : {\n" +
" \"match_all\":{\n" +
Expand All @@ -103,6 +104,7 @@ public void testCreateDetector_threatIntelEnabled_testJobRunner() throws IOExcep
List<SearchHit> hits = executeSearch(Detector.DETECTORS_INDEX, request);
SearchHit hit = hits.get(0);
Map<String, Object> detectorMap = (HashMap<String, Object>) (hit.getSourceAsMap().get("detector"));
String detectoraLstUpdateTime1 = detectorMap.get("last_update_time").toString();

List<String> monitorIds = ((List<String>) (detectorMap).get("monitor_id"));
assertEquals(1, monitorIds.size());
Expand Down Expand Up @@ -153,8 +155,12 @@ public void testCreateDetector_threatIntelEnabled_testJobRunner() throws IOExcep
assertNotEquals(newFeedTimestamp.get(i), originalFeedTimestamp.get(i));
}

// verify detector is updated by checking last updated time of detector
// TODO
// verify detectors updated with latest threat intel feed data
hits = executeSearch(Detector.DETECTORS_INDEX, request);
hit = hits.get(0);
detectorMap = (HashMap<String, Object>) (hit.getSourceAsMap().get("detector"));
String detectoraLstUpdateTime2 = detectorMap.get("last_update_time").toString();
assertFalse(detectoraLstUpdateTime2.equals(detectoraLstUpdateTime1));

}

Expand Down

0 comments on commit 74a7440

Please sign in to comment.