Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/feature/threat_intel' into tim
Browse files Browse the repository at this point in the history
  • Loading branch information
@eirsep
eirsep committed Jun 26, 2024
2 parents f8bab1e + e47a6ac commit 34bc5a2
Show file tree
Hide file tree
Showing 23 changed files with 1,155 additions and 224 deletions.
11 changes: 9 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ public Collection<Object> createComponents(Client client,
TIFLockService threatIntelLockService = new TIFLockService(clusterService, client);
saTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService);
STIX2IOCFetchService stix2IOCFetchService = new STIX2IOCFetchService(client, clusterService);
SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry);
SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry, clusterService);
SecurityAnalyticsRunner.getJobRunnerInstance();
TIFSourceConfigRunner.getJobRunnerInstance().initialize(clusterService, threatIntelLockService, threadPool, saTifSourceConfigManagementService, saTifSourceConfigService);
CorrelationAlertService correlationAlertService = new CorrelationAlertService(client, xContentRegistry);
Expand Down Expand Up @@ -474,14 +474,21 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ENABLED,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_MAX_DOCS,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.IOC_FINDING_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW,
SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS,
SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA,
SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE,
SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL,
SecurityAnalyticsSettings.BATCH_SIZE,
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT
SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS
);
}

Expand Down
122 changes: 117 additions & 5 deletions .../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.opensearch.common.unit.TimeValue;
import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig;
import org.opensearch.securityanalytics.logtype.LogTypeService;
import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService;
import org.opensearch.securityanalytics.util.CorrelationIndices;
import org.opensearch.threadpool.Scheduler;
import org.opensearch.threadpool.ThreadPool;
Expand All @@ -54,38 +55,52 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i
private volatile Boolean alertHistoryEnabled;
private volatile Boolean findingHistoryEnabled;

private volatile Boolean iocFindingHistoryEnabled;

private volatile Long alertHistoryMaxDocs;
private volatile Long findingHistoryMaxDocs;

private volatile Long iocFindingHistoryMaxDocs;

private volatile Long correlationHistoryMaxDocs;

private volatile TimeValue alertHistoryMaxAge;
private volatile TimeValue findingHistoryMaxAge;

private volatile TimeValue correlationHistoryMaxAge;

private volatile TimeValue iocFindingHistoryMaxAge;

private volatile TimeValue alertHistoryRolloverPeriod;
private volatile TimeValue findingHistoryRolloverPeriod;

private volatile TimeValue correlationHistoryRolloverPeriod;

private volatile TimeValue iocFindingHistoryRolloverPeriod;

private volatile TimeValue alertHistoryRetentionPeriod;
private volatile TimeValue findingHistoryRetentionPeriod;

private volatile TimeValue correlationHistoryRetentionPeriod;

private volatile TimeValue iocFindingHistoryRetentionPeriod;

private volatile boolean isClusterManager = false;

private Scheduler.Cancellable scheduledAlertsRollover = null;
private Scheduler.Cancellable scheduledFindingsRollover = null;

private Scheduler.Cancellable scheduledCorrelationHistoryRollover = null;

private Scheduler.Cancellable scheduledIocFindingHistoryRollover = null;

List<HistoryIndexInfo> alertHistoryIndices = new ArrayList<>();
List<HistoryIndexInfo> findingHistoryIndices = new ArrayList<>();

HistoryIndexInfo correlationHistoryIndex = null;

HistoryIndexInfo iocFindingHistoryIndex = null;

@Inject
public DetectorIndexManagementService(
Settings settings,
Expand Down Expand Up @@ -161,6 +176,27 @@ public DetectorIndexManagementService(

clusterService.getClusterSettings().addSettingsUpdateConsumer(CORRELATION_HISTORY_RETENTION_PERIOD, this::setCorrelationHistoryRetentionPeriod);

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_MAX_DOCS, maxDocs -> {
setIocFindingHistoryMaxDocs(maxDocs);
if (iocFindingHistoryIndex != null) {
iocFindingHistoryIndex.maxDocs = maxDocs;
}
});

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> {
setIocFindingHistoryMaxAge(maxAge);
if (iocFindingHistoryIndex != null) {
iocFindingHistoryIndex.maxAge = maxAge;
}
});

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> {
DetectorIndexManagementService.this.iocFindingHistoryRolloverPeriod = timeValue;
rescheduleIocFindingHistoryRollover();
});

clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_RETENTION_PERIOD, this::setIocFindingHistoryRetentionPeriod);

initFromClusterSettings();
}

Expand Down Expand Up @@ -204,15 +240,19 @@ private void initFromClusterSettings() {
alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings);
findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings);
correlationHistoryMaxDocs = CORRELATION_HISTORY_MAX_DOCS.get(settings);
iocFindingHistoryMaxDocs = IOC_FINDING_HISTORY_MAX_DOCS.get(settings);
alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings);
findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings);
correlationHistoryMaxAge = CORRELATION_HISTORY_INDEX_MAX_AGE.get(settings);
iocFindingHistoryMaxAge = IOC_FINDING_HISTORY_INDEX_MAX_AGE.get(settings);
alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings);
findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings);
correlationHistoryRolloverPeriod = CORRELATION_HISTORY_ROLLOVER_PERIOD.get(settings);
iocFindingHistoryRolloverPeriod = IOC_FINDING_HISTORY_ROLLOVER_PERIOD.get(settings);
alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings);
findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings);
correlationHistoryRetentionPeriod = CORRELATION_HISTORY_RETENTION_PERIOD.get(settings);
iocFindingHistoryRetentionPeriod = IOC_FINDING_HISTORY_RETENTION_PERIOD.get(settings);
}

@Override
Expand All @@ -238,6 +278,9 @@ public void clusterChanged(ClusterChangedEvent event) {
if (correlationHistoryIndex != null && correlationHistoryIndex.indexAlias != null) {
correlationHistoryIndex.isInitialized = event.state().metadata().hasAlias(correlationHistoryIndex.indexAlias);
}
if (iocFindingHistoryIndex != null && iocFindingHistoryIndex.indexAlias != null) {
iocFindingHistoryIndex.isInitialized = event.state().metadata().hasAlias(iocFindingHistoryIndex.indexAlias);
}
}

private void onMaster() {
Expand All @@ -247,6 +290,7 @@ private void onMaster() {
rolloverAndDeleteAlertHistoryIndices();
rolloverAndDeleteFindingHistoryIndices();
rolloverAndDeleteCorrelationHistoryIndices();
rolloverAndDeleteIocFindingHistoryIndices();
}, TimeValue.timeValueSeconds(1), executorName());
// schedule the next rollover for approx MAX_AGE later
scheduledAlertsRollover = threadPool
Expand All @@ -255,11 +299,13 @@ private void onMaster() {
.scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName());
scheduledCorrelationHistoryRollover = threadPool
.scheduleWithFixedDelay(() -> rolloverAndDeleteCorrelationHistoryIndices(), correlationHistoryRolloverPeriod, executorName());
scheduledIocFindingHistoryRollover = threadPool
.scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName());
} catch (Exception e) {
// This should be run on cluster startup
logger.error(
"Error creating alert/finding/correlation indices. " +
"Alerts/Findings/Correlations can't be recorded until master node is restarted.",
"Error creating alert/finding/correlation/ioc finding indices. " +
"Alerts/Findings/Correlations/IOC Finding can't be recorded until master node is restarted.",
e
);
}
Expand All @@ -275,6 +321,9 @@ private void offMaster() {
if (scheduledCorrelationHistoryRollover != null) {
scheduledCorrelationHistoryRollover.cancel();
}
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
}

private String executorName() {
Expand Down Expand Up @@ -327,6 +376,10 @@ private List<String> getIndicesToDelete(ClusterStateResponse clusterStateRespons
if (indexToDelete != null) {
indicesToDelete.add(indexToDelete);
}
indexToDelete = getHistoryIndexToDelete(indexMetaData, iocFindingHistoryRetentionPeriod.millis(), iocFindingHistoryIndex != null? List.of(iocFindingHistoryIndex): List.of(), true);
if (indexToDelete != null) {
indicesToDelete.add(indexToDelete);
}
}
return indicesToDelete;
}
Expand Down Expand Up @@ -371,7 +424,7 @@ private void deleteAllOldHistoryIndices(List<String> indicesToDelete) {
public void onResponse(AcknowledgedResponse deleteIndicesResponse) {
if (!deleteIndicesResponse.isAcknowledged()) {
logger.error(
"Could not delete one or more Alerting/Finding/Correlation history indices: [" + indicesToDelete + "]. Retrying one by one."
"Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: [" + indicesToDelete + "]. Retrying one by one."
);
deleteOldHistoryIndex(indicesToDelete);
} else {
Expand All @@ -381,7 +434,7 @@ public void onResponse(AcknowledgedResponse deleteIndicesResponse) {

@Override
public void onFailure(Exception e) {
logger.error("Delete for Alerting/Finding/Correlation History Indices failed: [" + indicesToDelete + "]. Retrying one By one.");
logger.error("Delete for Alerting/Finding/Correlation/IOC Finding History Indices failed: [" + indicesToDelete + "]. Retrying one By one.");
deleteOldHistoryIndex(indicesToDelete);
}
}
Expand All @@ -399,7 +452,7 @@ private void deleteOldHistoryIndex(List<String> indicesToDelete) {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
if (!acknowledgedResponse.isAcknowledged()) {
logger.error("Could not delete one or more Alerting/Finding/Correlation history indices: " + index);
logger.error("Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: " + index);
}
}

Expand Down Expand Up @@ -455,6 +508,23 @@ private void rolloverAndDeleteCorrelationHistoryIndices() {
}
}

private void rolloverAndDeleteIocFindingHistoryIndices() {
try {
iocFindingHistoryIndex = new HistoryIndexInfo(
IocFindingService.IOC_FINDING_ALIAS_NAME,
IocFindingService.IOC_FINDING_INDEX_PATTERN,
IocFindingService.getIndexMapping(),
iocFindingHistoryMaxDocs,
iocFindingHistoryMaxAge,
clusterService.state().metadata().hasAlias(IocFindingService.IOC_FINDING_ALIAS_NAME)
);
rolloverIocFindingHistoryIndices();
deleteOldIndices("IOC Findings", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP);
} catch (Exception ex) {
logger.error("failed to construct ioc finding index info");
}
}

private List<String> getAllAlertsIndicesPatternForAllTypes(List<String> logTypes) {
return logTypes
.stream()
Expand Down Expand Up @@ -544,6 +614,20 @@ private void rolloverCorrelationHistoryIndices() {
}
}

private void rolloverIocFindingHistoryIndices() {
if (iocFindingHistoryIndex != null) {
rolloverIndex(
iocFindingHistoryIndex.isInitialized,
iocFindingHistoryIndex.indexAlias,
iocFindingHistoryIndex.indexPattern,
iocFindingHistoryIndex.indexMappings,
iocFindingHistoryIndex.maxDocs,
iocFindingHistoryIndex.maxAge,
true
);
}
}

private void rescheduleAlertRollover() {
if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) {
if (scheduledAlertsRollover != null) {
Expand Down Expand Up @@ -574,6 +658,16 @@ private void rescheduleCorrelationHistoryRollover() {
}
}

private void rescheduleIocFindingHistoryRollover() {
if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) {
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
scheduledIocFindingHistoryRollover = threadPool
.scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName());
}
}

private String alertMapping() {
String alertMapping = null;
try (
Expand Down Expand Up @@ -620,6 +714,10 @@ public void setCorrelationHistoryMaxDocs(Long correlationHistoryMaxDocs) {
this.correlationHistoryMaxDocs = correlationHistoryMaxDocs;
}

public void setIocFindingHistoryMaxDocs(Long iocFindingHistoryMaxDocs) {
this.iocFindingHistoryMaxDocs = iocFindingHistoryMaxDocs;
}

public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) {
this.alertHistoryMaxAge = alertHistoryMaxAge;
}
Expand All @@ -632,6 +730,10 @@ public void setCorrelationHistoryMaxAge(TimeValue correlationHistoryMaxAge) {
this.correlationHistoryMaxAge = correlationHistoryMaxAge;
}

public void setIocFindingHistoryMaxAge(TimeValue iocFindingHistoryMaxAge) {
this.iocFindingHistoryMaxAge = iocFindingHistoryMaxAge;
}

public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) {
this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod;
}
Expand All @@ -656,6 +758,10 @@ public void setCorrelationHistoryRetentionPeriod(TimeValue correlationHistoryRet
this.correlationHistoryRetentionPeriod = correlationHistoryRetentionPeriod;
}

public void setIocFindingHistoryRetentionPeriod(TimeValue iocFindingHistoryRetentionPeriod) {
this.iocFindingHistoryRetentionPeriod = iocFindingHistoryRetentionPeriod;
}

public void setClusterManager(boolean clusterManager) {
isClusterManager = clusterManager;
}
Expand All @@ -676,6 +782,9 @@ protected void doStop() {
if (scheduledCorrelationHistoryRollover != null) {
scheduledCorrelationHistoryRollover.cancel();
}
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
}

@Override
Expand All @@ -689,6 +798,9 @@ protected void doClose() {
if (scheduledCorrelationHistoryRollover != null) {
scheduledCorrelationHistoryRollover.cancel();
}
if (scheduledIocFindingHistoryRollover != null) {
scheduledIocFindingHistoryRollover.cancel();
}
}

private static class HistoryIndexInfo {
Expand Down
Loading

0 comments on commit 34bc5a2

Please sign in to comment.