Sigma Aggregation rule fixes

Signed-off-by: Subhobrata Dey <[email protected]>

src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java

@@ -59,7 +59,6 @@ public void getFindingsByDetectorId(String detectorId, Table table, ActionListen
             public void onResponse(GetDetectorResponse getDetectorResponse) {
                 // Get all monitor ids from detector
                 Detector detector = getDetectorResponse.getDetector();
-                List<String> monitorIds = detector.getMonitorIds();
                 ActionListener<GetFindingsResponse> getFindingsResponseListener = new ActionListener<>() {
                     @Override
                     public void onResponse(GetFindingsResponse resp) {
@@ -87,12 +86,20 @@ public void onFailure(Exception e) {
                 // monitor --> detectorId mapping
                 Map<String, Detector> monitorToDetectorMapping = new HashMap<>();
                 detector.getMonitorIds().forEach(
-                        monitorId -> monitorToDetectorMapping.put(monitorId, detector)
+                        monitorId -> {
+                            if (detector.getRuleIdMonitorIdMap().containsKey("chained_findings_monitor")) {
+                                if (!detector.getRuleIdMonitorIdMap().get("chained_findings_monitor").equals(monitorId)) {
+                                    monitorToDetectorMapping.put(monitorId, detector);
+                                }
+                            } else {
+                                monitorToDetectorMapping.put(monitorId, detector);
+                            }
+                        }
                 );
                 // Get findings for all monitor ids
                 FindingsService.this.getFindingsByMonitorIds(
                         monitorToDetectorMapping,
-                        monitorIds,
+                        new ArrayList<>(monitorToDetectorMapping.keySet()),
                         DetectorMonitorConfig.getAllFindingsIndicesPattern(detector.getDetectorType()),
                         table,
                         getFindingsResponseListener

src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java

@@ -816,7 +816,7 @@ private IndexMonitorRequest createBucketLevelMonitorRequest(
                     ? new BoolQueryBuilder()
                     : QueryBuilders.boolQuery().must(searchSourceBuilder.query());
                 RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS)
-                    .gt("{{period_end}}||-" + aggItem.getTimeframe())
+                    .gt("{{period_end}}||-" + (aggItem.getTimeframe() != null? aggItem.getTimeframe(): "1h"))
                     .lte("{{period_end}}")
                     .format("epoch_millis");
                 boolQueryBuilder.must(timeRangeFilter);

src/test/java/org/opensearch/securityanalytics/TestHelpers.java

@@ -88,6 +88,11 @@ public static Detector randomDetectorWithTriggers(List<String> rules, List<Detec
                 rules.stream().map(DetectorRule::new).collect(Collectors.toList()));
         return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null);
     }
+    public static Detector randomDetectorWithTriggersAndScheduleAndEnabled(List<String> rules, List<DetectorTrigger> triggers, Schedule schedule, boolean enabled) {
+        DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(),
+                rules.stream().map(DetectorRule::new).collect(Collectors.toList()));
+        return randomDetector(null, null, null, List.of(input), triggers, schedule, enabled, null, null);
+    }
 
     public static Detector randomDetectorWithTriggers(List<String> rules, List<DetectorTrigger> triggers, String detectorType, DetectorInput input) {
         return randomDetector(null, detectorType, null, List.of(input), triggers, null, null, null, null);

src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java

@@ -1483,7 +1483,7 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor
         Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);
 
         assertNotNull(getFindingsBody);
-        assertEquals(10, getFindingsBody.get("total_findings"));
+        assertEquals(6, getFindingsBody.get("total_findings"));
 
         String findingDetectorId = ((Map<String, Object>)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString();
         assertEquals(detectorId, findingDetectorId);
@@ -1495,7 +1495,6 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor
         List<Map<String, Object>> findings = (List)getFindingsBody.get("findings");
 
         Set<String> docLevelRules = new HashSet<>(List.of(randomDocRuleId));
-        List<String> bucketLevelMonitorFindingDocs = new ArrayList<>();
         for(Map<String, Object> finding : findings) {
             List<Map<String, Object>> queries = (List<Map<String, Object>>) finding.get("queries");
             Set<String> findingRules = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet());
@@ -1504,16 +1503,10 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor
                 docLevelFinding.addAll((List<String>) finding.get("related_doc_ids"));
             } else {
                 List<String> findingDocs = (List<String>) finding.get("related_doc_ids");
-                if (((Map<String, Object>) ((List<Object>) finding.get("queries")).get(0)).get("query").equals("_id:*")) {
-                    Assert.assertEquals(1, findingDocs.size());
-                    bucketLevelMonitorFindingDocs.addAll(findingDocs);
-                } else {
-                    Assert.assertEquals(4, findingDocs.size());
-                    assertTrue(Arrays.asList("1", "2", "3", "4").containsAll(findingDocs));
-                }
+                Assert.assertEquals(4, findingDocs.size());
+                assertTrue(Arrays.asList("1", "2", "3", "4").containsAll(findingDocs));
             }
         }
-        assertTrue(bucketLevelMonitorFindingDocs.containsAll(Arrays.asList("1", "2", "3", "4")));
         // Verify doc level finding
         assertTrue(Arrays.asList("1", "2", "3", "4", "5").containsAll(docLevelFinding));
     }
@@ -1652,7 +1645,7 @@ public void testCreateDetector_verifyWorkflowExecutionMultipleBucketLevelDocLeve
 
         // Assert findings
         assertNotNull(getFindingsBody);
-        assertEquals(33, getFindingsBody.get("total_findings"));
+        assertEquals(19, getFindingsBody.get("total_findings"));
     }
 
 

src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java

@@ -4,6 +4,7 @@
  */
 package org.opensearch.securityanalytics.resthandler;
 
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -19,6 +20,7 @@
 import org.opensearch.client.Response;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.client.ResponseException;
+import org.opensearch.commons.alerting.model.IntervalSchedule;
 import org.opensearch.commons.alerting.model.Monitor.MonitorType;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
@@ -166,6 +168,82 @@ public void testCreatingADetector() throws IOException {
         Assert.assertEquals(5, noOfSigmaRuleMatches);
     }
 
+    public void testCreatingADetectorScheduledJobFinding() throws IOException, InterruptedException {
+        String index = createTestIndex(randomIndex(), windowsIndexMapping());
+
+        // Execute CreateMappingsAction to add alias mapping for index
+        Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI);
+        // both req params and req body are supported
+        createMappingRequest.setJsonEntity(
+                "{ \"index_name\":\"" + index + "\"," +
+                        "  \"rule_topic\":\"" + randomDetectorType() + "\", " +
+                        "  \"partial\":true" +
+                        "}"
+        );
+
+        Response response = client().performRequest(createMappingRequest);
+        assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode());
+
+        Detector detector = randomDetectorWithTriggersAndScheduleAndEnabled(getRandomPrePackagedRules(),
+                List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of())),
+                new IntervalSchedule(1, ChronoUnit.MINUTES, null), true);
+
+        Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));
+        Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse));
+
+        Map<String, Object> responseBody = asMap(createResponse);
+
+        String createdId = responseBody.get("_id").toString();
+        int createdVersion = Integer.parseInt(responseBody.get("_version").toString());
+        Assert.assertNotEquals("response is missing Id", Detector.NO_ID, createdId);
+        Assert.assertTrue("incorrect version", createdVersion > 0);
+        Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, createdId), createResponse.getHeader("Location"));
+        Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("rule_topic_index"));
+        Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("findings_index"));
+        Assert.assertFalse(((Map<String, Object>) responseBody.get("detector")).containsKey("alert_index"));
+
+        String detectorTypeInResponse = (String) ((Map<String, Object>)responseBody.get("detector")).get("detector_type");
+        Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse);
+
+        Thread.sleep(30000);
+        indexDoc(index, "1", randomDoc());
+        Thread.sleep(70000);
+
+        // Call GetFindings API
+        Map<String, String> params = new HashMap<>();
+        params.put("detector_id", createdId);
+        Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
+        Map<String, Object> getFindingsBody = entityAsMap(getFindingsResponse);
+        Assert.assertEquals(1, getFindingsBody.get("total_findings"));
+
+        // Call GetAlerts API
+        params = new HashMap<>();
+        params.put("detector_id", createdId);
+        Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null);
+        Map<String, Object> getAlertsBody = asMap(getAlertsResponse);
+        // TODO enable asserts here when able
+        Assert.assertEquals(1, getAlertsBody.get("total_alerts"));
+
+        Thread.sleep(30000);
+        indexDoc(index, "2", randomDoc());
+        Thread.sleep(70000);
+
+        // Call GetFindings API
+        params = new HashMap<>();
+        params.put("detector_id", createdId);
+        getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null);
+        getFindingsBody = entityAsMap(getFindingsResponse);
+        Assert.assertEquals(2, getFindingsBody.get("total_findings"));
+
+        // Call GetAlerts API
+        params = new HashMap<>();
+        params.put("detector_id", createdId);
+        getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null);
+        getAlertsBody = asMap(getAlertsResponse);
+        // TODO enable asserts here when able
+        Assert.assertEquals(2, getAlertsBody.get("total_alerts"));
+    }
+
     @SuppressWarnings("unchecked")
     public void test_searchDetectors_detectorsIndexNotExists() throws IOException {
         try {