Skip to content

Commit

Permalink
add search ioc findings api (opensearch-project#1093)
Browse files Browse the repository at this point in the history 
* add search ioc findings api

Signed-off-by: Subhobrata Dey <[email protected]>

add search ioc findings api

Signed-off-by: Subhobrata Dey <[email protected]>

add search ioc findings api

Signed-off-by: Subhobrata Dey <[email protected]>

add search ioc findings api

Signed-off-by: Subhobrata Dey <[email protected]>

* fix review comments for ioc findings api

Signed-off-by: Subhobrata Dey <[email protected]>

---------

Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90
sbcd90 authored Jun 26, 2024
1 parent ad80024 commit e47a6ac
Show file tree
Hide file tree
Showing 17 changed files with 1,222 additions and 132 deletions.
10 changes: 10 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction;
Expand All @@ -134,6 +135,7 @@
import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner;
import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportRemoteDocLevelMonitorFanOutAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction;
Expand Down Expand Up @@ -183,6 +185,7 @@
import org.opensearch.securityanalytics.transport.TransportTestS3ConnectionAction;
import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction;
import org.opensearch.securityanalytics.transport.TransportValidateRulesAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.securityanalytics.util.CorrelationRuleIndices;
import org.opensearch.securityanalytics.util.CustomLogTypeIndices;
Expand Down Expand Up @@ -352,6 +355,7 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestSearchThreatIntelMonitorAction(),
new RestRefreshTIFSourceConfigAction(),
new RestListIOCsAction(),
new RestGetIocFindingsAction(),
new RestTestS3ConnectionAction()
);
}
Expand Down Expand Up @@ -449,6 +453,11 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ENABLED,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_MAX_DOCS,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW,
SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS,
Expand Down Expand Up @@ -501,6 +510,7 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class),
new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class),
new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class),
new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class),
new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class)
);
}
Expand Down
122 changes: 117 additions & 5 deletions .../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.opensearch.common.unit.TimeValue;
import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;
import org.opensearch.securityanalytics.logtype.LogTypeService;
import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.threadpool.Scheduler;
import org.opensearch.threadpool.ThreadPool;
Expand All @@ -54,38 +55,52 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i
private volatile Boolean alertHistoryEnabled;
private volatile Boolean findingHistoryEnabled;

private volatile Boolean iocFindingHistoryEnabled;

private volatile Long alertHistoryMaxDocs;
private volatile Long findingHistoryMaxDocs;

private volatile Long iocFindingHistoryMaxDocs;

private volatile Long correlationHistoryMaxDocs;

private volatile TimeValue alertHistoryMaxAge;
private volatile TimeValue findingHistoryMaxAge;

private volatile TimeValue correlationHistoryMaxAge;

private volatile TimeValue iocFindingHistoryMaxAge;

private volatile TimeValue alertHistoryRolloverPeriod;
private volatile TimeValue findingHistoryRolloverPeriod;

private volatile TimeValue correlationHistoryRolloverPeriod;

private volatile TimeValue iocFindingHistoryRolloverPeriod;

private volatile TimeValue alertHistoryRetentionPeriod;
private volatile TimeValue findingHistoryRetentionPeriod;

private volatile TimeValue correlationHistoryRetentionPeriod;

private volatile TimeValue iocFindingHistoryRetentionPeriod;

private volatile boolean isClusterManager = false;

private Scheduler.Cancellable scheduledAlertsRollover = null;
private Scheduler.Cancellable scheduledFindingsRollover = null;

private Scheduler.Cancellable scheduledCorrelationHistoryRollover = null;

private Scheduler.Cancellable scheduledIocFindingHistoryRollover = null;

List<HistoryIndexInfo> alertHistoryIndices = new ArrayList<>();
List<HistoryIndexInfo> findingHistoryIndices = new ArrayList<>();

HistoryIndexInfo correlationHistoryIndex = null;

HistoryIndexInfo iocFindingHistoryIndex = null;

@Inject
public DetectorIndexManagementService(
Settings settings,
Expand Down Expand Up @@ -161,6 +176,27 @@ public DetectorIndexManagementService(

clusterService.getClusterSettings().addSettingsUpdateConsumer(CORRELATION_HISTORY_RETENTION_PERIOD, this::setCorrelationHistoryRetentionPeriod);

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_MAX_DOCS, maxDocs -> {
setIocFindingHistoryMaxDocs(maxDocs);
if (iocFindingHistoryIndex != null) {
iocFindingHistoryIndex.maxDocs = maxDocs;
}
});

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> {
setIocFindingHistoryMaxAge(maxAge);
if (iocFindingHistoryIndex != null) {
iocFindingHistoryIndex.maxAge = maxAge;
}
});

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> {
DetectorIndexManagementService.this.iocFindingHistoryRolloverPeriod = timeValue;
rescheduleIocFindingHistoryRollover();
});

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_RETENTION_PERIOD, this::setIocFindingHistoryRetentionPeriod);

initFromClusterSettings();
}

Expand Down Expand Up @@ -204,15 +240,19 @@ private void initFromClusterSettings() {
alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings);
findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings);
correlationHistoryMaxDocs = CORRELATION_HISTORY_MAX_DOCS.get(settings);
iocFindingHistoryMaxDocs = IOC_FINDING_HISTORY_MAX_DOCS.get(settings);
alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings);
findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings);
correlationHistoryMaxAge = CORRELATION_HISTORY_INDEX_MAX_AGE.get(settings);
iocFindingHistoryMaxAge = IOC_FINDING_HISTORY_INDEX_MAX_AGE.get(settings);
alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings);
findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings);
correlationHistoryRolloverPeriod = CORRELATION_HISTORY_ROLLOVER_PERIOD.get(settings);
iocFindingHistoryRolloverPeriod = IOC_FINDING_HISTORY_ROLLOVER_PERIOD.get(settings);
alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings);
findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings);
correlationHistoryRetentionPeriod = CORRELATION_HISTORY_RETENTION_PERIOD.get(settings);
iocFindingHistoryRetentionPeriod = IOC_FINDING_HISTORY_RETENTION_PERIOD.get(settings);
}

@Override
Expand All @@ -238,6 +278,9 @@ public void clusterChanged(ClusterChangedEvent event) {
if (correlationHistoryIndex != null && correlationHistoryIndex.indexAlias != null) {
correlationHistoryIndex.isInitialized = event.state().metadata().hasAlias(correlationHistoryIndex.indexAlias);
}
if (iocFindingHistoryIndex != null && iocFindingHistoryIndex.indexAlias != null) {
iocFindingHistoryIndex.isInitialized = event.state().metadata().hasAlias(iocFindingHistoryIndex.indexAlias);
}
}

private void onMaster() {
Expand All @@ -247,6 +290,7 @@ private void onMaster() {
rolloverAndDeleteAlertHistoryIndices();
rolloverAndDeleteFindingHistoryIndices();
rolloverAndDeleteCorrelationHistoryIndices();
rolloverAndDeleteIocFindingHistoryIndices();
}, TimeValue.timeValueSeconds(1), executorName());
// schedule the next rollover for approx MAX_AGE later
scheduledAlertsRollover = threadPool
Expand All @@ -255,11 +299,13 @@ private void onMaster() {
.scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName());
scheduledCorrelationHistoryRollover = threadPool
.scheduleWithFixedDelay(() -> rolloverAndDeleteCorrelationHistoryIndices(), correlationHistoryRolloverPeriod, executorName());
scheduledIocFindingHistoryRollover = threadPool
.scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName());
} catch (Exception e) {
// This should be run on cluster startup
logger.error(
"Error creating alert/finding/correlation indices. " +
"Alerts/Findings/Correlations can't be recorded until master node is restarted.",
"Error creating alert/finding/correlation/ioc finding indices. " +
"Alerts/Findings/Correlations/IOC Finding can't be recorded until master node is restarted.",
e
);
}
Expand All @@ -275,6 +321,9 @@ private void offMaster() {
if (scheduledCorrelationHistoryRollover != null) {
scheduledCorrelationHistoryRollover.cancel();
}
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
}

private String executorName() {
Expand Down Expand Up @@ -327,6 +376,10 @@ private List<String> getIndicesToDelete(ClusterStateResponse clusterStateRespons
if (indexToDelete != null) {
indicesToDelete.add(indexToDelete);
}
indexToDelete = getHistoryIndexToDelete(indexMetaData, iocFindingHistoryRetentionPeriod.millis(), iocFindingHistoryIndex != null? List.of(iocFindingHistoryIndex): List.of(), true);
if (indexToDelete != null) {
indicesToDelete.add(indexToDelete);
}
}
return indicesToDelete;
}
Expand Down Expand Up @@ -371,7 +424,7 @@ private void deleteAllOldHistoryIndices(List<String> indicesToDelete) {
public void onResponse(AcknowledgedResponse deleteIndicesResponse) {
if (!deleteIndicesResponse.isAcknowledged()) {
logger.error(
"Could not delete one or more Alerting/Finding/Correlation history indices: [" + indicesToDelete + "]. Retrying one by one."
"Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: [" + indicesToDelete + "]. Retrying one by one."
);
deleteOldHistoryIndex(indicesToDelete);
} else {
Expand All @@ -381,7 +434,7 @@ public void onResponse(AcknowledgedResponse deleteIndicesResponse) {

@Override
public void onFailure(Exception e) {
logger.error("Delete for Alerting/Finding/Correlation History Indices failed: [" + indicesToDelete + "]. Retrying one By one.");
logger.error("Delete for Alerting/Finding/Correlation/IOC Finding History Indices failed: [" + indicesToDelete + "]. Retrying one By one.");
deleteOldHistoryIndex(indicesToDelete);
}
}
Expand All @@ -399,7 +452,7 @@ private void deleteOldHistoryIndex(List<String> indicesToDelete) {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
if (!acknowledgedResponse.isAcknowledged()) {
logger.error("Could not delete one or more Alerting/Finding/Correlation history indices: " + index);
logger.error("Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: " + index);
}
}

Expand Down Expand Up @@ -455,6 +508,23 @@ private void rolloverAndDeleteCorrelationHistoryIndices() {
}
}

private void rolloverAndDeleteIocFindingHistoryIndices() {
try {
iocFindingHistoryIndex = new HistoryIndexInfo(
IocFindingService.IOC_FINDING_ALIAS_NAME,
IocFindingService.IOC_FINDING_INDEX_PATTERN,
IocFindingService.getIndexMapping(),
iocFindingHistoryMaxDocs,
iocFindingHistoryMaxAge,
clusterService.state().metadata().hasAlias(IocFindingService.IOC_FINDING_ALIAS_NAME)
);
rolloverIocFindingHistoryIndices();
deleteOldIndices("IOC Findings", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP);
} catch (Exception ex) {
logger.error("failed to construct ioc finding index info");
}
}

private List<String> getAllAlertsIndicesPatternForAllTypes(List<String> logTypes) {
return logTypes
.stream()
Expand Down Expand Up @@ -544,6 +614,20 @@ private void rolloverCorrelationHistoryIndices() {
}
}

private void rolloverIocFindingHistoryIndices() {
if (iocFindingHistoryIndex != null) {
rolloverIndex(
iocFindingHistoryIndex.isInitialized,
iocFindingHistoryIndex.indexAlias,
iocFindingHistoryIndex.indexPattern,
iocFindingHistoryIndex.indexMappings,
iocFindingHistoryIndex.maxDocs,
iocFindingHistoryIndex.maxAge,
true
);
}
}

private void rescheduleAlertRollover() {
if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) {
if (scheduledAlertsRollover != null) {
Expand Down Expand Up @@ -574,6 +658,16 @@ private void rescheduleCorrelationHistoryRollover() {
}
}

private void rescheduleIocFindingHistoryRollover() {
if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) {
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
scheduledIocFindingHistoryRollover = threadPool
.scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName());
}
}

private String alertMapping() {
String alertMapping = null;
try (
Expand Down Expand Up @@ -620,6 +714,10 @@ public void setCorrelationHistoryMaxDocs(Long correlationHistoryMaxDocs) {
this.correlationHistoryMaxDocs = correlationHistoryMaxDocs;
}

public void setIocFindingHistoryMaxDocs(Long iocFindingHistoryMaxDocs) {
this.iocFindingHistoryMaxDocs = iocFindingHistoryMaxDocs;
}

public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) {
this.alertHistoryMaxAge = alertHistoryMaxAge;
}
Expand All @@ -632,6 +730,10 @@ public void setCorrelationHistoryMaxAge(TimeValue correlationHistoryMaxAge) {
this.correlationHistoryMaxAge = correlationHistoryMaxAge;
}

public void setIocFindingHistoryMaxAge(TimeValue iocFindingHistoryMaxAge) {
this.iocFindingHistoryMaxAge = iocFindingHistoryMaxAge;
}

public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) {
this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod;
}
Expand All @@ -656,6 +758,10 @@ public void setCorrelationHistoryRetentionPeriod(TimeValue correlationHistoryRet
this.correlationHistoryRetentionPeriod = correlationHistoryRetentionPeriod;
}

public void setIocFindingHistoryRetentionPeriod(TimeValue iocFindingHistoryRetentionPeriod) {
this.iocFindingHistoryRetentionPeriod = iocFindingHistoryRetentionPeriod;
}

public void setClusterManager(boolean clusterManager) {
isClusterManager = clusterManager;
}
Expand All @@ -676,6 +782,9 @@ protected void doStop() {
if (scheduledCorrelationHistoryRollover != null) {
scheduledCorrelationHistoryRollover.cancel();
}
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
}

@Override
Expand All @@ -689,6 +798,9 @@ protected void doClose() {
if (scheduledCorrelationHistoryRollover != null) {
scheduledCorrelationHistoryRollover.cancel();
}
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
}

private static class HistoryIndexInfo {
Expand Down
Loading

0 comments on commit e47a6ac

Please sign in to comment.