Merge branch 'opensearch-project:main' into main

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,15 +1,16 @@
 ### Description
 [Describe what this change achieves]
-
-### Issues Resolved
-[List any issues this PR will resolve]
-
+
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
+
 ### Check List
 - [ ] New functionality includes testing.
-  - [ ] All tests pass
 - [ ] New functionality has been documented.
-  - [ ] New functionality has javadoc added
-- [ ] Commits are signed per the DCO using --signoff 
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
+- [ ] Commits are signed per the DCO using `--signoff`.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
-For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
+For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/security-analytics/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/ci.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches:
       - "*"
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
   Get-CI-Image-Tag:
@@ -75,7 +77,7 @@ jobs:
         os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
-            os_build_args: -x jacocoTestReport
+            os_build_args: -x integTest
             working_directory: X:\
             os_java_options: -Xmx4096M
           - os: macos-latest

.github/workflows/multi-node-test-workflow.yml

@@ -7,7 +7,8 @@ on:
   push:
     branches:
       - "*"
-
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 jobs:
   Get-CI-Image-Tag:
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main

.github/workflows/security-test-workflow.yml

@@ -9,6 +9,7 @@ on:
       - "*"
 env:
   OPENSEARCH_INITIAL_ADMIN_PASSWORD: myStrongPassword123!
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
   build:

README.md

@@ -1,3 +1,5 @@
+[![Test Workflow](https://github.com/opensearch-project/security-analytics/workflows/Test%20Workflow/badge.svg)](https://github.com/opensearch-project/security-analytics/actions)
+[![codecov](https://codecov.io/gh/opensearch-project/security-analytics/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/security-analytics)
 ![Documentation](https://img.shields.io/badge/api-reference-blue.svg)
 ![Chat](https://img.shields.io/badge/chat-on%20forums-blue)
 ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success)

build-tools/opensearchplugin-coverage.gradle

@@ -3,9 +3,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+apply plugin: 'jacoco'
+
 /**
  * OpenSearch Plugin build tools don't work with the Gradle Jacoco Plugin to report coverage out of the box.
- * https://github.com/elastic/elasticsearch/issues/28867.
  *
  * This code sets up coverage reporting manually for OpenSearch plugin tests. This is complicated because:
  *  1. The OpenSearch integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM
@@ -14,13 +15,13 @@
  *
  *  To workaround these we start the cluster with jmx enabled and then use Jacoco's JMX MBean to get the execution data before the
  *  cluster is stopped and dump it to a file. Luckily our current security policy seems to allow this. This will also probably
- *  break if there are multiple nodes in the integTestCluster. But for now... it sorta works. 
+ *  break if there are multiple nodes in the integTestCluster. But for now... it sorta works.
  */
-apply plugin: 'jacoco'
 
 // Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's
 // testing tasks don't derive from Test so the jacoco plugin can't do this automatically.
 def jacocoDir = "${buildDir}/jacoco"
+
 task dummyTest(type: Test) {
     enabled = false
     workingDir = file("/") // Force absolute path to jacoco agent jar
@@ -31,19 +32,55 @@ task dummyTest(type: Test) {
     }
 }
 
+task dummyIntegTest(type: Test) {
+    enabled = false
+    workingDir = file("/") // Force absolute path to jacoco agent jar
+    jacoco {
+        destinationFile = file("${jacocoDir}/integTest.exec")
+        destinationFile.parentFile.mkdirs()
+        jmx = true
+    }
+}
+task dummyIntegTestRunner(type: Test) {
+    enabled = false
+    workingDir = file("/") // Force absolute path to jacoco agent jar
+    jacoco {
+        destinationFile = file("${jacocoDir}/integTestRunner.exec")
+        destinationFile.parentFile.mkdirs()
+        jmx = true
+    }
+}
+
+integTest {
+    systemProperty 'jacoco.dir', "${jacocoDir}"
+}
+
 jacocoTestReport {
-    dependsOn test
-    executionData dummyTest.jacoco.destinationFile
-    getSourceDirectories().from(sourceSets.main.allSource)
-    getClassDirectories().from(sourceSets.main.output)
+    dependsOn integTest, test
+    executionData.from dummyTest.jacoco.destinationFile, dummyIntegTest.jacoco.destinationFile, dummyIntegTestRunner.jacoco.destinationFile
+    sourceDirectories.from = "src/main/java"
+    classDirectories.from = sourceSets.main.output
     reports {
         html.required = true // human readable
+        csv.required = true
         xml.required = true // for coverlay
     }
 }
 
-project.gradle.projectsEvaluated {
-    jacocoTestReport.dependsOn test
+
+allprojects {
+    afterEvaluate {
+        jacocoTestReport.dependsOn integTest
+
+        testClusters.integTest {
+            jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:', 'javaagent:/')
+            systemProperty 'com.sun.management.jmxremote', "true"
+            systemProperty 'com.sun.management.jmxremote.authenticate', "false"
+            systemProperty 'com.sun.management.jmxremote.port', "7777"
+            systemProperty 'com.sun.management.jmxremote.ssl', "false"
+            systemProperty 'java.rmi.server.hostname', "127.0.0.1"
+        }
+    }
 }
 
-check.dependsOn jacocoTestReport
+check.dependsOn jacocoTestReport

build.gradle

@@ -14,14 +14,24 @@ buildscript {
         opensearch_build = version_tokens[0] + '.0'
         plugin_no_snapshot = opensearch_build
         opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
+        sa_commons_version = '1.0.0'
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
         }
+
+        alerting_spi_build = opensearch_build
+        alerting_spi_build += "-SNAPSHOT"
         if (isSnapshot) {
             opensearch_build += "-SNAPSHOT"
+
+            // TODO consider enabling snapshot options once SA commons is published to maven central
+//            sa_commons_version += "-SNAPSHOT"
         }
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-        kotlin_version = '1.6.10'
+        kotlin_version = '1.8.21'
+
+        sa_commons_file_name = "security-analytics-commons-${sa_commons_version}.jar"
+        sa_commons_file_path = "${project.rootDir}/${sa_commons_file_name}"
     }
 
     repositories {
@@ -46,7 +56,6 @@ apply plugin: 'opensearch.opensearchplugin'
 apply plugin: 'opensearch.testclusters'
 apply plugin: 'opensearch.java-rest-test'
 apply plugin: 'opensearch.pluginzip'
-apply from: 'build-tools/opensearchplugin-coverage.gradle'
 apply from: 'gradle/formatting.gradle'
 
 ext {
@@ -55,7 +64,7 @@ ext {
     noticeFile = rootProject.file('NOTICE')
 }
 
-licenseHeaders.enabled = true
+licenseHeaders.enabled = false
 testingConventions.enabled = false
 forbiddenApis.ignoreFailures = true
 
@@ -69,7 +78,7 @@ opensearchplugin {
     name 'opensearch-security-analytics'
     description 'OpenSearch Security Analytics plugin'
     classname 'org.opensearch.securityanalytics.SecurityAnalyticsPlugin'
-    extendedPlugins = ['opensearch-job-scheduler']
+    extendedPlugins = ['opensearch-job-scheduler', 'opensearch-alerting']
 }
 
 javaRestTest {
@@ -148,27 +157,36 @@ configurations {
         resolutionStrategy {
             // for spotless transitive dependency CVE
             force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0"
-            force "com.google.guava:guava:32.1.2-jre"
+            force "com.google.guava:guava:32.1.3-jre"
         }
     }
 }
 
 dependencies {
     javaRestTestImplementation project.sourceSets.main.runtimeClasspath
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: "${versions.commonslang}"
-    implementation "org.antlr:antlr4-runtime:4.10.1"
-    implementation "com.cronutils:cron-utils:9.1.6"
-    api "org.opensearch:common-utils:${common_utils_version}@jar"
-    api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
-    implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
+    compileOnly "org.antlr:antlr4-runtime:4.10.1"
+    compileOnly "com.cronutils:cron-utils:9.1.7"
+    compileOnly "org.opensearch:common-utils:${common_utils_version}@jar"
+    compileOnly "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
+    compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
+    compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}"
     implementation "org.apache.commons:commons-csv:1.10.0"
+    compileOnly "com.google.guava:guava:32.1.3-jre"
+
+    // TODO uncomment once SA commons is published to maven central
+//    api "org.opensearch:security-analytics-commons:${sa_commons_version}@jar"
+
+    // TODO remove once SA commons is published to maven central
+    api files(sa_commons_file_path)
 
     // Needed for integ tests
     zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1'
 }
 
 // RPM & Debian build
@@ -306,6 +324,11 @@ testClusters.integTest {
         plugins.add(firstPlugin)
     }
 }
+def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
+def usingMultiNode = project.properties.containsKey('numNodes')
+if (!usingRemoteCluster && !usingMultiNode) {
+    apply from: 'build-tools/opensearchplugin-coverage.gradle'
+}
 
 run {
     doFirst {
@@ -356,6 +379,12 @@ afterEvaluate {
             into opensearchplugin.name
         }
 
+        // TODO remove once SA commons is published to maven central
+        from(project.rootDir) {
+            include sa_commons_file_name
+            into opensearchplugin.name
+        }
+
         user 'root'
         permissionGroup 'root'
         fileMode 0644

release-notes/opensearch-security-analytics.release-notes-2.14.0.0.md

@@ -0,0 +1,22 @@
+## Version 2.14.0.0 2024-04-30
+
+Compatible with OpenSearch 2.14.0
+
+### Maintenance
+* Increment version to 2.14.0-SNAPSHOT. ([#1007](https://github.com/opensearch-project/security-analytics/pull/1007))
+* Updates sample cert and admin keystore ([#864](https://github.com/opensearch-project/security-analytics/pull/864))
+
+### Features
+* Add latest sigma rules ([#942](https://github.com/opensearch-project/security-analytics/pull/942))
+
+### Bug Fixes
+* Fix integ tests after add latest sigma rules ([#950](https://github.com/opensearch-project/security-analytics/pull/950))
+* Fix keywords bug and add comments ([#964](https://github.com/opensearch-project/security-analytics/pull/964))
+* Changes doc level query name field from id to rule name and adds validation ([#972](https://github.com/opensearch-project/security-analytics/pull/972))
+* Fix check for agg rules in detector trigger condition to create chained findings monitor ([#992](https://github.com/opensearch-project/security-analytics/pull/992))
+
+### Refactoring
+* Allow detectors to be stopped if underlying workflow is deleted. Don't allow them to then be started/edited ([#810](https://github.com/opensearch-project/security-analytics/pull/810))
+
+### Documentation
+* Added 2.14.0 release notes. ([#1009](https://github.com/opensearch-project/security-analytics/pull/1009))

release-notes/opensearch-security-analytics.release-notes-2.15.0.0.md

@@ -0,0 +1,19 @@
+## Version 2.15.0.0 2024-06-10
+
+Compatible with OpenSearch 2.15.0
+
+### Features
+* Alerts in correlations [Experminental] ([#1040](https://github.com/opensearch-project/security-analytics/pull/1040))
+* Alerts in Correlations Part 2 ([#1062](https://github.com/opensearch-project/security-analytics/pull/1062))
+
+### Maintenance
+* Increment version to 2.15.0-SNAPSHOT. ([#1055](https://github.com/opensearch-project/security-analytics/pull/1055))
+* Fix codecov calculation ([#1021](https://github.com/opensearch-project/security-analytics/pull/1021))
+* Stabilize integ tests ([#1014](https://github.com/opensearch-project/security-analytics/pull/1014))
+
+### Bug Fixes
+* Fix chained findings monitor logic in update detector flow ([#1019](https://github.com/opensearch-project/security-analytics/pull/1019))
+* Change default filter to time based fields ([#1030](https://github.com/opensearch-project/security-analytics/pull/1030))
+
+### Documentation
+* Added 2.15.0 release notes. ([#1061](https://github.com/opensearch-project/security-analytics/pull/1061))

release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md

@@ -0,0 +1,35 @@
+## Version 2.16.0.0 2024-07-23
+
+Compatible with OpenSearch 2.16.0
+
+### Features
+* Threat Intel Analytics ([#1098](https://github.com/opensearch-project/security-analytics/pull/1098))
+
+### Maintenance
+* Incremented version to 2.16.0. ([#1197](https://github.com/opensearch-project/security-analytics/pull/1197))
+* Fix build CI error due to action runner env upgrade node 20 ([#1143](https://github.com/opensearch-project/security-analytics/pull/1143))
+
+### Enhancement
+* added correlationAlert integ tests ([#1099](https://github.com/opensearch-project/security-analytics/pull/1099))
+* add filter to list ioc api to fetch only from available and refreshing apis. null check for alias of ioc indices ([#1131](https://github.com/opensearch-project/security-analytics/pull/1131))
+* Changes threat intel default store config model ([#1133](https://github.com/opensearch-project/security-analytics/pull/1133))
+* adds new tif source config type - url download ([#1142](https://github.com/opensearch-project/security-analytics/pull/1142))
+
+### Bug Fixes
+* pass integ tests ([#1082](https://github.com/opensearch-project/security-analytics/pull/1082))
+* set blank response when indexNotFound exception ([#1125](https://github.com/opensearch-project/security-analytics/pull/1125))
+* throw error when no iocs are stored due to incompatible ioc types from S3 downloaded iocs file ([#1129](https://github.com/opensearch-project/security-analytics/pull/1129))
+* fix findingIds filter on ioc findings search api ([#1130](https://github.com/opensearch-project/security-analytics/pull/1130))
+* Adjusted IOCTypes usage ([#1156](https://github.com/opensearch-project/security-analytics/pull/1156))
+* Fix the job scheduler parser, action listeners, and multi-node test ([#1157](https://github.com/opensearch-project/security-analytics/pull/1157))
+* ListIOCs API to return number of findings per IOC ([#1163](https://github.com/opensearch-project/security-analytics/pull/1163))
+* Ioc upload integ tests and fix update ([#1162](https://github.com/opensearch-project/security-analytics/pull/1162))
+* [BUG] Resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs ([#1173](https://github.com/opensearch-project/security-analytics/pull/1173))
+* Enum fix ([#1178](https://github.com/opensearch-project/security-analytics/pull/1178))
+* fix bug: threat intel monitor finding doesnt contain all doc_ids containing malicious IOC ([#1184](https://github.com/opensearch-project/security-analytics/pull/1184))
+* Fixed bulk indexing for IOCs ([#1187](https://github.com/opensearch-project/security-analytics/pull/1187))
+* Fix ioc upload update behavior and change error response ([#1192](https://github.com/opensearch-project/security-analytics/pull/1192))
+* Catch and wrap exceptions. ([#1198](https://github.com/opensearch-project/security-analytics/pull/1198))
+
+### Documentation
+* Added 2.16.0 release notes. ([#1196](https://github.com/opensearch-project/security-analytics/pull/1196))

src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseListener.java

@@ -1,8 +1,4 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-// Generated from Condition.g4 by ANTLR 4.10.1
+// Generated from java-escape by ANTLR 4.11.1
 package org.opensearch.securityanalytics.rules.condition;
 
 import org.antlr.v4.runtime.ParserRuleContext;
@@ -14,6 +10,7 @@
  * which can be extended to create a listener which only needs to handle a subset
  * of the available methods.
  */
+@SuppressWarnings("CheckReturnValue")
 public class ConditionBaseListener implements ConditionListener {
     /**
      * {@inheritDoc}

src/main/generated/org/opensearch/securityanalytics/rules/condition/ConditionBaseVisitor.java

@@ -1,8 +1,4 @@
-/*
- * Copyright OpenSearch Contributors
- * SPDX-License-Identifier: Apache-2.0
- */
-// Generated from Condition.g4 by ANTLR 4.10.1
+// Generated from java-escape by ANTLR 4.11.1
 package org.opensearch.securityanalytics.rules.condition;
 import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
 
@@ -14,6 +10,7 @@
  * @param <T> The return type of the visit operation. Use {@link Void} for
  * operations with no return type.
  */
+@SuppressWarnings("CheckReturnValue")
 public class ConditionBaseVisitor<T> extends AbstractParseTreeVisitor<T> implements ConditionVisitor<T> {
     /**
      * {@inheritDoc}