Backports for 2.5 (#927)

* query_field_names bugfix (#335) Signed-off-by: Petar Dzepina <[email protected]> * Reduce log level for informative message (#203) Signed-off-by: Enrico Tröger <[email protected]> * fix detector writeTo() method missing fields (#695) * fix detector writeTo() method missing fields Signed-off-by: Surya Sashank Nistala <[email protected]> * fix test Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * fix null query filter conversion from sigma to query string query (#722) * fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala <[email protected]> * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala <[email protected]> * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala <[email protected]> --------- Signed-off-by: Surya Sashank Nistala <[email protected]> * Bump version 2.5.1, fix build Signed-off-by: Chase Engelbrecht <[email protected]> * Manual cherry-pick of #873 Signed-off-by: Chase Engelbrecht <[email protected]> * Fix miss from manual cherry-pick Signed-off-by: Chase Engelbrecht <[email protected]> * Undo exceptional case not originally present Signed-off-by: Chase Engelbrecht <[email protected]> * Revert test from previous commit reversion Signed-off-by: Chase Engelbrecht <[email protected]> * Fix another miss from manual cherry-pick Signed-off-by: Chase Engelbrecht <[email protected]> * Revert behavior for no rules in detector Signed-off-by: Chase Engelbrecht <[email protected]> --------- Signed-off-by: Petar Dzepina <[email protected]> Signed-off-by: Enrico Tröger <[email protected]> Signed-off-by: Surya Sashank Nistala <[email protected]> Signed-off-by: Chase Engelbrecht <[email protected]> Co-authored-by: Petar Dzepina <[email protected]> Co-authored-by: Enrico Tröger <[email protected]> Co-authored-by: Surya Sashank Nistala <[email protected]>
opensearch-project · Mar 15, 2024 · 2d10915 · 2d10915
1 parent 0025b7a
commit 2d10915
Show file tree

Hide file tree

Showing 12 changed files with 535 additions and 207 deletions.
diff --git a/build.gradle b/build.gradle
@@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.5.1-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -125,6 +125,9 @@ repositories {
 }
 
 sourceSets.main.java.srcDirs = ['src/main/generated','src/main/java']
+configurations {
+    zipArchive
+}
 
 dependencies {
     javaRestTestImplementation project.sourceSets.main.runtimeClasspath
@@ -134,6 +137,11 @@ dependencies {
     api "org.opensearch:common-utils:${common_utils_version}"
     api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
+
+    // Needed for integ tests
+    zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
+    zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"
 }
 
 // RPM & Debian build
@@ -209,15 +217,6 @@ integTest.getClusters().forEach{c -> {
     c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
 }}
 
-String alertingFilePath = "src/test/resources/alerting"
-String alertingPlugin = "opensearch-alerting-" + plugin_no_snapshot + ".zip"
-String alertingRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + alertingPlugin
-String notificationsFilePath = "src/test/resources/notifications"
-String notificationsCoreFilePath = "src/test/resources/notifications-core"
-String notificationsPlugin = "opensearch-notifications-" + plugin_no_snapshot + ".zip"
-String notificationsCorePlugin = "opensearch-notifications-core-" + plugin_no_snapshot + ".zip"
-String notificationsRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsPlugin
-String notificationsCoreRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsCorePlugin
 testClusters.integTest {
     testDistribution = 'ARCHIVE'
 
@@ -233,58 +232,33 @@ testClusters.integTest {
             debugPort += 1
         }
     }
-    setting 'path.repo', repo.absolutePath
     plugin(provider({
         new RegularFile() {
             @Override
             File getAsFile() {
-                File dir = new File(rootDir.path + "/" + alertingFilePath)
-
-                if (!dir.exists()) {
-                    dir.mkdirs()
-                }
-
-                File f = new File(dir, alertingPlugin)
-                if (!f.exists()) {
-                    new URL(alertingRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
-                }
-                fileTree(alertingFilePath).getSingleFile()
+                return configurations.zipArchive.asFileTree.matching {
+                    include '**/opensearch-notifications-core*'
+                }.singleFile
             }
         }
     }))
     plugin(provider({
         new RegularFile() {
             @Override
             File getAsFile() {
-                File dir = new File(rootDir.path + "/" + notificationsCoreFilePath)
-
-                if (!dir.exists()) {
-                    dir.mkdirs()
-                }
-
-                File f = new File(dir, notificationsCorePlugin)
-                if (!f.exists()) {
-                    new URL(notificationsCoreRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
-                }
-                fileTree(notificationsCoreFilePath).getSingleFile()
+                return configurations.zipArchive.asFileTree.matching {
+                    include '**/notifications*'
+                }.singleFile
             }
         }
     }))
     plugin(provider({
         new RegularFile() {
             @Override
             File getAsFile() {
-                File dir = new File(rootDir.path + "/" + notificationsFilePath)
-
-                if (!dir.exists()) {
-                    dir.mkdirs()
-                }
-
-                File f = new File(dir, notificationsPlugin)
-                if (!f.exists()) {
-                    new URL(notificationsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
-                }
-                fileTree(notificationsFilePath).getSingleFile()
+                return configurations.zipArchive.asFileTree.matching {
+                    include '**/alerting*'
+                }.singleFile
             }
         }
     }))

diff --git a/.../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/.../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java
@@ -233,7 +233,7 @@ private String executorName() {
     }
 
     private void deleteOldIndices(String tag, String... indices) {
-        logger.error("info deleteOldIndices");
+        logger.info("info deleteOldIndices");
         ClusterStateRequest clusterStateRequest = new ClusterStateRequest()
                 .clear()
                 .indices(indices)

diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
@@ -71,9 +71,11 @@ public void createMappingAction(String indexName, String ruleTopic, String alias
         // since you can't update documents in non-write indices
         String index = indexName;
         boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false;
-        if (IndexUtils.isDataStream(indexName, this.clusterService.state())) {
+        if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) {
+            log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName);
             String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state());
             if (writeIndex != null) {
+                log.debug("Write index for {} is {}", indexName, writeIndex);
                 index = writeIndex;
             }
         }
@@ -85,6 +87,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
                 applyAliasMappings(getMappingsResponse.getMappings(), ruleTopic, aliasMappings, partial, new ActionListener<>() {
                     @Override
                     public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
+                        log.debug("Completed create mappings for {}", indexName);
                         // We will return ack==false if one of the requests returned that
                         // else return ack==true
                         Optional<AcknowledgedResponse> notAckd = createMappingResponse.stream()
@@ -103,6 +106,7 @@ public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
 
                     @Override
                     public void onFailure(Exception e) {
+                        log.debug("Failed to create mappings for {}", indexName );
                         actionListener.onFailure(e);
                     }
                 });

diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java
@@ -159,12 +159,12 @@ public Detector(StreamInput sin) throws IOException {
                 sin.readList(DetectorInput::readFrom),
                 sin.readList(DetectorTrigger::readFrom),
                 sin.readStringList(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
-                sin.readString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
+                sin.readOptionalString(),
                 sin.readMap(StreamInput::readString, StreamInput::readString)
             );
     }
@@ -197,8 +197,12 @@ public void writeTo(StreamOutput out) throws IOException {
             it.writeTo(out);
         }
         out.writeStringCollection(monitorIds);
-        out.writeString(ruleIndex);
-
+        out.writeOptionalString(ruleIndex);
+        out.writeOptionalString(alertsIndex);
+        out.writeOptionalString(alertsHistoryIndex);
+        out.writeOptionalString(alertsHistoryIndexPattern);
+        out.writeOptionalString(findingsIndex);
+        out.writeOptionalString(findingsIndexPattern);
         out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString);
     }
 

diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
@@ -131,7 +131,7 @@ public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enable
         this.reEscapeChar = "\\";
         this.reExpression = "%s: /%s/";
         this.cidrExpression = "%s: \"%s\"";
-        this.fieldNullExpression = "%s: null";
+        this.fieldNullExpression = "%s: (NOT [* TO *])";
         this.unboundValueStrExpression = "\"%s\"";
         this.unboundValueNumExpression = "\"%s\"";
         this.unboundWildcardExpression = "%s";
@@ -331,6 +331,8 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre
 
     @Override
     public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError {
+        String field = getFinalValueField();
+        ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
         SigmaString value = (SigmaString) condition.getValue();
         boolean containsWildcard = value.containsWildcard();
         return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));

diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java
@@ -180,6 +180,10 @@ public Map<String, Object> getQueryFields() {
         return queryFields;
     }
 
+    public void resetQueryFields() {
+        queryFields.clear();
+    }
+
     public abstract Object convertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition);
 
     public abstract Object convertConditionAnd(ConditionAND condition);