Merge branch 'opensearch-project:main' into main

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [21]
         os: [ ubuntu-latest ]
     name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -55,14 +55,14 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs-ubuntu
           path: build/testclusters/integTest-*/logs/*
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: security-analytics-plugin-${{ matrix.os }}
           path: security-analytics-artifacts
@@ -73,7 +73,7 @@ jobs:
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [21]
         os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
@@ -113,21 +113,21 @@ jobs:
           cp ./build/distributions/*.zip security-analytics-artifacts
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ failure() && matrix.os == 'macos-latest' }}
         with:
           name: logs-mac
           path: build/testclusters/integTest-*/logs/*
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ failure() && matrix.os == 'windows-latest' }}
         with:
           name: logs-windows
           path: build\testclusters\integTest-*\logs\*
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: security-analytics-plugin-${{ matrix.os }}
           path: security-analytics-artifacts

.github/workflows/multi-node-test-workflow.yml

@@ -19,7 +19,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     # Job name
     name: Build and test Security Analytics on linux
     # This job runs on Linux
@@ -45,7 +45,7 @@ jobs:
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c "./gradlew integTest -PnumNodes=3"
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs

.github/workflows/security-test-workflow.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     # Job name
     name: Build and test SecurityAnalytics
     # This job runs on Linux

release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md

@@ -0,0 +1,27 @@
+## Version 2.17.0.0 2024-09-05
+
+Compatible with OpenSearch 2.17.0
+
+### Maintenance
+* update build.gradle to use alerting-spi snapshot version ([#1217](https://github.com/opensearch-project/security-analytics/pull/1217))
+
+### Enhancement
+* added triggers in getDetectors API response ([#1226](https://github.com/opensearch-project/security-analytics/pull/1226))
+* secure rest tests for threat intel monitor apis ([#1212](https://github.com/opensearch-project/security-analytics/pull/1212))
+
+### Bug Fixes
+* Adds user validation for threat intel transport layer classes and stashes the thread context for all system index interactions ([#1207](https://github.com/opensearch-project/security-analytics/pull/1207))
+* fix mappings integ tests ([#1213](https://github.com/opensearch-project/security-analytics/pull/1213))
+* Bug fixes for threat intel ([#1223](https://github.com/opensearch-project/security-analytics/pull/1223))
+* make threat intel run with standard detectors ([#1234](https://github.com/opensearch-project/security-analytics/pull/1234))
+* Fixed searchString bug. Removed nested IOC mapping structure. ([#1239](https://github.com/opensearch-project/security-analytics/pull/1239))
+* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs ([#1240](https://github.com/opensearch-project/security-analytics/pull/1240))
+* Make threat intel source config release lock event driven ([#1254](https://github.com/opensearch-project/security-analytics/pull/1254))
+* Fix S3 validation errors not caught by action listener ([#1257](https://github.com/opensearch-project/security-analytics/pull/1257))
+* Clean up empty IOC indices created by failed source configs ([#1267](https://github.com/opensearch-project/security-analytics/pull/1267))
+* Fix threat intel multinode tests ([#1274](https://github.com/opensearch-project/security-analytics/pull/1274))
+* Update threat intel job mapping to new version ([#1272](https://github.com/opensearch-project/security-analytics/pull/1272))
+* Stash context for List IOCs Api ([#1278](https://github.com/opensearch-project/security-analytics/pull/1278))
+
+### Documentation
+* Added 2.17.0 release notes. ([#1290](https://github.com/opensearch-project/security-analytics/pull/1290))

release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md

@@ -0,0 +1,15 @@
+## Version 2.17.1.0 2024-09-27
+
+Compatible with OpenSearch 2.17.1
+
+### Maintenance
+* upgrade upload artifacts ([#1305](https://github.com/opensearch-project/security-analytics/pull/1305))
+* Incremented version to 2.17.1 ([#1304](https://github.com/opensearch-project/security-analytics/pull/1304))
+
+### Bug Fixes
+* [Alerts in Correlations] Stash context for system index ([#1297](https://github.com/opensearch-project/security-analytics/pull/1297))
+* threat intel monitor bug fixes ([#1317](https://github.com/opensearch-project/security-analytics/pull/1317))
+
+
+### Documentation
+* Added 2.17.1 release notes. ([#1331](https://github.com/opensearch-project/security-analytics/pull/1331))

release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md

@@ -0,0 +1,24 @@
+## Version 2.18.0.0 2024-10-28
+
+Compatible with OpenSearch 2.18.0
+
+### Maintenance
+* Incremented version to 2.18.0 ([#1314](https://github.com/opensearch-project/security-analytics/pull/1314))
+* update to lucene 9.12 ([#1349](https://github.com/opensearch-project/security-analytics/pull/1349))
+
+### Refactoring
+* separate doc-level monitor query indices created by detectors ([#1324](https://github.com/opensearch-project/security-analytics/pull/1324))
+* update number of replicas of system indices to 1-20 and number of primary shards for system indices to 1 ([#1358](https://github.com/opensearch-project/security-analytics/pull/1358))
+* update min number of replicas to 0 ([#1364](https://github.com/opensearch-project/security-analytics/pull/1364))
+* updated dedicated query index settings to true ([#1365](https://github.com/opensearch-project/security-analytics/pull/1365))
+* set the refresh policy to IMMEDIATE when updating correlation alerts ([#1382](https://github.com/opensearch-project/security-analytics/pull/1382))
+
+### Bug Fixes
+* remove redundant logic to fix OS launch exception and updates actions/upload-artifac2 to @V3 ([#1303](https://github.com/opensearch-project/security-analytics/pull/1303))
+* Add null check while adding fetched iocs into per-indicator-type map ([#1335](https://github.com/opensearch-project/security-analytics/pull/1335))
+* Fix notifications listener leak in threat intel monitor ([#1361](https://github.com/opensearch-project/security-analytics/pull/1361))
+* [Bug] Fixed ListIOCs number of findings cap. ([#1373](https://github.com/opensearch-project/security-analytics/pull/1373))
+* [Bug] Add exists check for IOCs index. ([#1392](https://github.com/opensearch-project/security-analytics/pull/1392))
+
+### Documentation
+* Added 2.18.0 release notes. ([#1399](https://github.com/opensearch-project/security-analytics/pull/1399))

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -70,7 +70,7 @@
 import org.opensearch.securityanalytics.action.IndexDetectorAction;
 import org.opensearch.securityanalytics.action.IndexRuleAction;
 import org.opensearch.securityanalytics.action.ListCorrelationsAction;
-import org.opensearch.securityanalytics.action.ListIOCsAction;
+import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction;
 import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction;
 import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction;
 import org.opensearch.securityanalytics.action.SearchDetectorAction;
@@ -113,7 +113,7 @@
 import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction;
 import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction;
 import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction;
-import org.opensearch.securityanalytics.resthandler.RestListIOCsAction;
+import org.opensearch.securityanalytics.threatIntel.resthandler.RestListIOCsAction;
 import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction;
 import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction;
 import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction;
@@ -197,7 +197,7 @@
 import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction;
 import org.opensearch.securityanalytics.transport.TransportIndexRuleAction;
 import org.opensearch.securityanalytics.transport.TransportListCorrelationAction;
-import org.opensearch.securityanalytics.transport.TransportListIOCsAction;
+import org.opensearch.securityanalytics.threatIntel.transport.TransportListIOCsAction;
 import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction;
 import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction;
 import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction;
@@ -226,6 +226,7 @@
 import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE;
 import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD;
 import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;
+import static org.opensearch.securityanalytics.util.CorrelationIndices.CORRELATION_ALERT_INDEX;
 
 public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension {
 
@@ -284,7 +285,11 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"));
+        List<SystemIndexDescriptor> descriptors = List.of(
+                new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"),
+                new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts")
+        );
+        return descriptors;
     }
 
 
@@ -327,7 +332,7 @@ public Collection<Object> createComponents(Client client,
         TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService);
         IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry);
         ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry);
-        SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
+        SaIoCScanService ioCScanService = new SaIoCScanService(client, clusterService, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
         DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService);
         return List.of(
                 detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService,
@@ -502,7 +507,9 @@ public List<Setting<?>> getSettings() {
                 SecurityAnalyticsSettings.BATCH_SIZE,
                 SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
                 SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
-                SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN
+                SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN,
+                SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT,
+                SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES
         );
     }
 

src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java

@@ -5,6 +5,8 @@
 package org.opensearch.securityanalytics.config.monitors;
 
 import java.util.List;
+import java.util.Random;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.securityanalytics.logtype.LogTypeService;
@@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) {
         return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType);
     }
 
+    public static String getRuleIndexOptimized(String logType) {
+        return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID());
+    }
+
     public static String getAlertsIndex(String logType) {
         return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType);
     }

src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java

@@ -13,6 +13,7 @@
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.client.Client;
 import org.opensearch.common.lucene.uid.Versions;
@@ -212,9 +213,10 @@ public void acknowledgeAlerts(List<String> alertIds, ActionListener<AckCorrelati
         client.search(searchRequest, new ActionListener<SearchResponse>() {
             @Override
             public void onResponse(SearchResponse searchResponse) {
+                // Set the refresh policy on the BulkRequest
+                bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
                 // Iterate through the search hits
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
-                    // Construct a script to update the document with the new state and acknowledgedTime
                     // Construct a script to update the document with the new state and acknowledgedTime
                     Script script = new Script(ScriptType.INLINE, "painless",
                             "ctx._source.state = params.state; ctx._source.acknowledged_time = params.time",

src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java

@@ -81,6 +81,7 @@ public void sendNotification(String configId, String severity, String subject, S
                 sendNotificationResponse -> {
                     if (sendNotificationResponse.getStatus() == RestStatus.OK) {
                         logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent());
+                        listener.onResponse(null);
                     } else {
                         listener.onFailure(new Exception("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()));
                     }

src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java

@@ -4,11 +4,13 @@
  */
 package org.opensearch.securityanalytics.correlation.index.codec;
 
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
+import org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120;
 import org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950;
 import org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990;
 import org.opensearch.securityanalytics.correlation.index.codec.correlation990.PerFieldCorrelationVectorsFormat990;
@@ -32,9 +34,16 @@ public enum CorrelationCodecVersion {
             new PerFieldCorrelationVectorsFormat990(Optional.empty()),
             (userCodec, mapperService) -> new CorrelationCodec990(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))),
             CorrelationCodec990::new
+    ),
+    V_9_12_0(
+            "CorrelationCodec9120",
+            new Lucene912Codec(),
+            new PerFieldCorrelationVectorsFormat990(Optional.empty()),
+            (userCodec, mapperService) -> new CorrelationCodec9120(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))),
+            CorrelationCodec9120::new
     );
 
-    private static final CorrelationCodecVersion CURRENT = V_9_9_0;
+    private static final CorrelationCodecVersion CURRENT = V_9_12_0;
     private final String codecName;
     private final Codec defaultCodecDelegate;
     private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat;

src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.securityanalytics.correlation.index.codec.correlation9120;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
+import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecVersion;
+
+public class CorrelationCodec9120 extends FilterCodec {
+    private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_12_0;
+    private final PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat;
+
+    public CorrelationCodec9120() {
+        this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldCorrelationVectorsFormat());
+    }
+
+    public CorrelationCodec9120(Codec delegate, PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat) {
+        super(VERSION.getCodecName(), delegate);
+        this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat;
+    }
+
+    @Override
+    public KnnVectorsFormat knnVectorsFormat() {
+        return perFieldCorrelationVectorsFormat;
+    }
+}

src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java

@@ -558,8 +558,17 @@ private void rolloverIndex(
         request.getCreateIndexRequest().index(pattern)
                 .mapping(map)
                 .settings(isCorrelation?
-                        Settings.builder().put("index.hidden", true).put("index.correlation", true).build():
-                        Settings.builder().put("index.hidden", true).build()
+                        Settings.builder()
+                                .put("index.hidden", true)
+                                .put("index.correlation", true)
+                                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                                .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                                .build():
+                        Settings.builder()
+                                .put("index.hidden", true)
+                                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                                .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas)
+                                .build()
                 );
         request.addMaxIndexDocsCondition(docsCondition);
         request.addMaxIndexAgeCondition(ageCondition);