diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index c8168d428..e458afce4 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -119,6 +119,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; @@ -134,6 +135,7 @@ import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner; import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportRemoteDocLevelMonitorFanOutAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction; @@ -183,6 +185,7 @@ import org.opensearch.securityanalytics.transport.TransportTestS3ConnectionAction; import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction; import org.opensearch.securityanalytics.transport.TransportValidateRulesAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -352,6 +355,7 @@ public List getRestHandlers(Settings settings, new RestSearchThreatIntelMonitorAction(), new RestRefreshTIFSourceConfigAction(), new RestListIOCsAction(), + new RestGetIocFindingsAction(), new RestTestS3ConnectionAction() ); } @@ -449,6 +453,11 @@ public List> getSettings() { SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE, SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD, SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ENABLED, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_RETENTION_PERIOD, SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, @@ -501,6 +510,7 @@ public List> getSettings() { new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class), new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class), new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class), + new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class), new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index f6630499f..d6cae5304 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -35,6 +35,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -54,9 +55,13 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile Boolean alertHistoryEnabled; private volatile Boolean findingHistoryEnabled; + private volatile Boolean iocFindingHistoryEnabled; + private volatile Long alertHistoryMaxDocs; private volatile Long findingHistoryMaxDocs; + private volatile Long iocFindingHistoryMaxDocs; + private volatile Long correlationHistoryMaxDocs; private volatile TimeValue alertHistoryMaxAge; @@ -64,16 +69,22 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile TimeValue correlationHistoryMaxAge; + private volatile TimeValue iocFindingHistoryMaxAge; + private volatile TimeValue alertHistoryRolloverPeriod; private volatile TimeValue findingHistoryRolloverPeriod; private volatile TimeValue correlationHistoryRolloverPeriod; + private volatile TimeValue iocFindingHistoryRolloverPeriod; + private volatile TimeValue alertHistoryRetentionPeriod; private volatile TimeValue findingHistoryRetentionPeriod; private volatile TimeValue correlationHistoryRetentionPeriod; + private volatile TimeValue iocFindingHistoryRetentionPeriod; + private volatile boolean isClusterManager = false; private Scheduler.Cancellable scheduledAlertsRollover = null; @@ -81,11 +92,15 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private Scheduler.Cancellable scheduledCorrelationHistoryRollover = null; + private Scheduler.Cancellable scheduledIocFindingHistoryRollover = null; + List alertHistoryIndices = new ArrayList<>(); List findingHistoryIndices = new ArrayList<>(); HistoryIndexInfo correlationHistoryIndex = null; + HistoryIndexInfo iocFindingHistoryIndex = null; + @Inject public DetectorIndexManagementService( Settings settings, @@ -161,6 +176,27 @@ public DetectorIndexManagementService( clusterService.getClusterSettings().addSettingsUpdateConsumer(CORRELATION_HISTORY_RETENTION_PERIOD, this::setCorrelationHistoryRetentionPeriod); + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_MAX_DOCS, maxDocs -> { + setIocFindingHistoryMaxDocs(maxDocs); + if (iocFindingHistoryIndex != null) { + iocFindingHistoryIndex.maxDocs = maxDocs; + } + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> { + setIocFindingHistoryMaxAge(maxAge); + if (iocFindingHistoryIndex != null) { + iocFindingHistoryIndex.maxAge = maxAge; + } + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.iocFindingHistoryRolloverPeriod = timeValue; + rescheduleIocFindingHistoryRollover(); + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_RETENTION_PERIOD, this::setIocFindingHistoryRetentionPeriod); + initFromClusterSettings(); } @@ -204,15 +240,19 @@ private void initFromClusterSettings() { alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings); findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings); correlationHistoryMaxDocs = CORRELATION_HISTORY_MAX_DOCS.get(settings); + iocFindingHistoryMaxDocs = IOC_FINDING_HISTORY_MAX_DOCS.get(settings); alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings); findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings); correlationHistoryMaxAge = CORRELATION_HISTORY_INDEX_MAX_AGE.get(settings); + iocFindingHistoryMaxAge = IOC_FINDING_HISTORY_INDEX_MAX_AGE.get(settings); alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings); findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); correlationHistoryRolloverPeriod = CORRELATION_HISTORY_ROLLOVER_PERIOD.get(settings); + iocFindingHistoryRolloverPeriod = IOC_FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings); findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings); correlationHistoryRetentionPeriod = CORRELATION_HISTORY_RETENTION_PERIOD.get(settings); + iocFindingHistoryRetentionPeriod = IOC_FINDING_HISTORY_RETENTION_PERIOD.get(settings); } @Override @@ -238,6 +278,9 @@ public void clusterChanged(ClusterChangedEvent event) { if (correlationHistoryIndex != null && correlationHistoryIndex.indexAlias != null) { correlationHistoryIndex.isInitialized = event.state().metadata().hasAlias(correlationHistoryIndex.indexAlias); } + if (iocFindingHistoryIndex != null && iocFindingHistoryIndex.indexAlias != null) { + iocFindingHistoryIndex.isInitialized = event.state().metadata().hasAlias(iocFindingHistoryIndex.indexAlias); + } } private void onMaster() { @@ -247,6 +290,7 @@ private void onMaster() { rolloverAndDeleteAlertHistoryIndices(); rolloverAndDeleteFindingHistoryIndices(); rolloverAndDeleteCorrelationHistoryIndices(); + rolloverAndDeleteIocFindingHistoryIndices(); }, TimeValue.timeValueSeconds(1), executorName()); // schedule the next rollover for approx MAX_AGE later scheduledAlertsRollover = threadPool @@ -255,11 +299,13 @@ private void onMaster() { .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); scheduledCorrelationHistoryRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteCorrelationHistoryIndices(), correlationHistoryRolloverPeriod, executorName()); + scheduledIocFindingHistoryRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName()); } catch (Exception e) { // This should be run on cluster startup logger.error( - "Error creating alert/finding/correlation indices. " + - "Alerts/Findings/Correlations can't be recorded until master node is restarted.", + "Error creating alert/finding/correlation/ioc finding indices. " + + "Alerts/Findings/Correlations/IOC Finding can't be recorded until master node is restarted.", e ); } @@ -275,6 +321,9 @@ private void offMaster() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } private String executorName() { @@ -327,6 +376,10 @@ private List getIndicesToDelete(ClusterStateResponse clusterStateRespons if (indexToDelete != null) { indicesToDelete.add(indexToDelete); } + indexToDelete = getHistoryIndexToDelete(indexMetaData, iocFindingHistoryRetentionPeriod.millis(), iocFindingHistoryIndex != null? List.of(iocFindingHistoryIndex): List.of(), true); + if (indexToDelete != null) { + indicesToDelete.add(indexToDelete); + } } return indicesToDelete; } @@ -371,7 +424,7 @@ private void deleteAllOldHistoryIndices(List indicesToDelete) { public void onResponse(AcknowledgedResponse deleteIndicesResponse) { if (!deleteIndicesResponse.isAcknowledged()) { logger.error( - "Could not delete one or more Alerting/Finding/Correlation history indices: [" + indicesToDelete + "]. Retrying one by one." + "Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: [" + indicesToDelete + "]. Retrying one by one." ); deleteOldHistoryIndex(indicesToDelete); } else { @@ -381,7 +434,7 @@ public void onResponse(AcknowledgedResponse deleteIndicesResponse) { @Override public void onFailure(Exception e) { - logger.error("Delete for Alerting/Finding/Correlation History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); + logger.error("Delete for Alerting/Finding/Correlation/IOC Finding History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); deleteOldHistoryIndex(indicesToDelete); } } @@ -399,7 +452,7 @@ private void deleteOldHistoryIndex(List indicesToDelete) { @Override public void onResponse(AcknowledgedResponse acknowledgedResponse) { if (!acknowledgedResponse.isAcknowledged()) { - logger.error("Could not delete one or more Alerting/Finding/Correlation history indices: " + index); + logger.error("Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: " + index); } } @@ -455,6 +508,23 @@ private void rolloverAndDeleteCorrelationHistoryIndices() { } } + private void rolloverAndDeleteIocFindingHistoryIndices() { + try { + iocFindingHistoryIndex = new HistoryIndexInfo( + IocFindingService.IOC_FINDING_ALIAS_NAME, + IocFindingService.IOC_FINDING_INDEX_PATTERN, + IocFindingService.getIndexMapping(), + iocFindingHistoryMaxDocs, + iocFindingHistoryMaxAge, + clusterService.state().metadata().hasAlias(IocFindingService.IOC_FINDING_ALIAS_NAME) + ); + rolloverIocFindingHistoryIndices(); + deleteOldIndices("IOC Findings", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP); + } catch (Exception ex) { + logger.error("failed to construct ioc finding index info"); + } + } + private List getAllAlertsIndicesPatternForAllTypes(List logTypes) { return logTypes .stream() @@ -544,6 +614,20 @@ private void rolloverCorrelationHistoryIndices() { } } + private void rolloverIocFindingHistoryIndices() { + if (iocFindingHistoryIndex != null) { + rolloverIndex( + iocFindingHistoryIndex.isInitialized, + iocFindingHistoryIndex.indexAlias, + iocFindingHistoryIndex.indexPattern, + iocFindingHistoryIndex.indexMappings, + iocFindingHistoryIndex.maxDocs, + iocFindingHistoryIndex.maxAge, + true + ); + } + } + private void rescheduleAlertRollover() { if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) { if (scheduledAlertsRollover != null) { @@ -574,6 +658,16 @@ private void rescheduleCorrelationHistoryRollover() { } } + private void rescheduleIocFindingHistoryRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) { + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } + scheduledIocFindingHistoryRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName()); + } + } + private String alertMapping() { String alertMapping = null; try ( @@ -620,6 +714,10 @@ public void setCorrelationHistoryMaxDocs(Long correlationHistoryMaxDocs) { this.correlationHistoryMaxDocs = correlationHistoryMaxDocs; } + public void setIocFindingHistoryMaxDocs(Long iocFindingHistoryMaxDocs) { + this.iocFindingHistoryMaxDocs = iocFindingHistoryMaxDocs; + } + public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) { this.alertHistoryMaxAge = alertHistoryMaxAge; } @@ -632,6 +730,10 @@ public void setCorrelationHistoryMaxAge(TimeValue correlationHistoryMaxAge) { this.correlationHistoryMaxAge = correlationHistoryMaxAge; } + public void setIocFindingHistoryMaxAge(TimeValue iocFindingHistoryMaxAge) { + this.iocFindingHistoryMaxAge = iocFindingHistoryMaxAge; + } + public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) { this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod; } @@ -656,6 +758,10 @@ public void setCorrelationHistoryRetentionPeriod(TimeValue correlationHistoryRet this.correlationHistoryRetentionPeriod = correlationHistoryRetentionPeriod; } + public void setIocFindingHistoryRetentionPeriod(TimeValue iocFindingHistoryRetentionPeriod) { + this.iocFindingHistoryRetentionPeriod = iocFindingHistoryRetentionPeriod; + } + public void setClusterManager(boolean clusterManager) { isClusterManager = clusterManager; } @@ -676,6 +782,9 @@ protected void doStop() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } @Override @@ -689,6 +798,9 @@ protected void doClose() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } private static class HistoryIndexInfo { diff --git a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java b/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java similarity index 71% rename from src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java rename to src/main/java/org/opensearch/securityanalytics/model/IocFinding.java index 04f54699f..6c34b2cb3 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java +++ b/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java @@ -13,6 +13,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -20,13 +21,13 @@ * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job * It's the inverse of an IoC finding which maps a document to list of IoC's */ -public class IoCMatch implements Writeable, ToXContent { +public class IocFinding implements Writeable, ToXContent { //TODO implement IoC_Match interface from security-analytics-commons public static final String ID_FIELD = "id"; public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; - public static final String FEED_IDS_FIELD = "feed_ids"; - public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id"; - public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name"; + public static final String IOC_WITH_FEED_IDS_FIELD = "ioc_feed_ids"; + public static final String MONITOR_ID_FIELD = "monitor_id"; + public static final String MONITOR_NAME_FIELD = "monitor_name"; public static final String IOC_VALUE_FIELD = "ioc_value"; public static final String IOC_TYPE_FIELD = "ioc_type"; public static final String TIMESTAMP_FIELD = "timestamp"; @@ -34,34 +35,34 @@ public class IoCMatch implements Writeable, ToXContent { private final String id; private final List relatedDocIds; - private final List feedIds; - private final String iocScanJobId; - private final String iocScanJobName; + private final List iocWithFeeds; + private final String monitorId; + private final String monitorName; private final String iocValue; private final String iocType; private final Instant timestamp; private final String executionId; - public IoCMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, - String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { - validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); + public IocFinding(String id, List relatedDocIds, List iocWithFeeds, String monitorId, + String monitorName, String iocValue, String iocType, Instant timestamp, String executionId) { + validateIoCMatch(id, monitorId, monitorName, iocValue, timestamp, executionId, relatedDocIds); this.id = id; this.relatedDocIds = relatedDocIds; - this.feedIds = feedIds; - this.iocScanJobId = iocScanJobId; - this.iocScanJobName = iocScanJobName; + this.iocWithFeeds = iocWithFeeds; + this.monitorId = monitorId; + this.monitorName = monitorName; this.iocValue = iocValue; this.iocType = iocType; this.timestamp = timestamp; this.executionId = executionId; } - public IoCMatch(StreamInput in) throws IOException { + public IocFinding(StreamInput in) throws IOException { id = in.readString(); relatedDocIds = in.readStringList(); - feedIds = in.readStringList(); - iocScanJobId = in.readString(); - iocScanJobName = in.readString(); + iocWithFeeds = in.readList(IocWithFeeds::readFrom); + monitorId = in.readString(); + monitorName = in.readString(); iocValue = in.readString(); iocType = in.readString(); timestamp = in.readInstant(); @@ -72,23 +73,37 @@ public IoCMatch(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeStringCollection(relatedDocIds); - out.writeStringCollection(feedIds); - out.writeString(iocScanJobId); - out.writeString(iocScanJobName); + out.writeCollection(iocWithFeeds); + out.writeString(monitorId); + out.writeString(monitorName); out.writeString(iocValue); out.writeString(iocType); out.writeInstant(timestamp); out.writeOptionalString(executionId); } + public Map asTemplateArg() { + return Map.of( + ID_FIELD,id, + RELATED_DOC_IDS_FIELD, relatedDocIds, + IOC_WITH_FEED_IDS_FIELD, iocWithFeeds, + MONITOR_ID_FIELD, monitorId, + MONITOR_NAME_FIELD, monitorName, + IOC_VALUE_FIELD, iocValue, + IOC_TYPE_FIELD, iocType, + TIMESTAMP_FIELD, timestamp, + EXECUTION_ID_FIELD, executionId + ); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject() .field(ID_FIELD, id) .field(RELATED_DOC_IDS_FIELD, relatedDocIds) - .field(FEED_IDS_FIELD, feedIds) - .field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId) - .field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName) + .field(IOC_WITH_FEED_IDS_FIELD, iocWithFeeds) + .field(MONITOR_ID_FIELD, monitorId) + .field(MONITOR_NAME_FIELD, monitorName) .field(IOC_VALUE_FIELD, iocValue) .field(IOC_TYPE_FIELD, iocType) .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) @@ -105,16 +120,16 @@ public List getRelatedDocIds() { return relatedDocIds; } - public List getFeedIds() { - return feedIds; + public List getFeedIds() { + return iocWithFeeds; } - public String getIocScanJobId() { - return iocScanJobId; + public String getMonitorId() { + return monitorId; } - public String getIocScanJobName() { - return iocScanJobName; + public String getMonitorName() { + return monitorName; } public String getIocValue() { @@ -133,12 +148,12 @@ public String getExecutionId() { return executionId; } - public static IoCMatch parse(XContentParser xcp) throws IOException { + public static IocFinding parse(XContentParser xcp) throws IOException { String id = null; List relatedDocIds = new ArrayList<>(); - List feedIds = new ArrayList<>(); - String iocScanJobId = null; - String iocScanName = null; + List feedIds = new ArrayList<>(); + String monitorId = null; + String monitorName = null; String iocValue = null; String iocType = null; Instant timestamp = null; @@ -159,17 +174,17 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { relatedDocIds.add(xcp.text()); } break; - case FEED_IDS_FIELD: + case IOC_WITH_FEED_IDS_FIELD: ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - feedIds.add(xcp.text()); + feedIds.add(IocWithFeeds.parse(xcp)); } break; - case IOC_SCAN_JOB_ID_FIELD: - iocScanJobId = xcp.textOrNull(); + case MONITOR_ID_FIELD: + monitorId = xcp.textOrNull(); break; - case IOC_SCAN_JOB_NAME_FIELD: - iocScanName = xcp.textOrNull(); + case MONITOR_NAME_FIELD: + monitorName = xcp.textOrNull(); break; case IOC_VALUE_FIELD: iocValue = xcp.textOrNull(); @@ -197,11 +212,11 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { } } - return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + return new IocFinding(id, relatedDocIds, feedIds, monitorId, monitorName, iocValue, iocType, timestamp, executionId); } - public static IoCMatch readFrom(StreamInput in) throws IOException { - return new IoCMatch(in); + public static IocFinding readFrom(StreamInput in) throws IOException { + return new IocFinding(in); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java b/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java new file mode 100644 index 000000000..d858619fc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java @@ -0,0 +1,111 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * container class to store a tuple of feed id, ioc id and index. + */ +public class IocWithFeeds implements Writeable, ToXContent { + + private static final String FEED_ID_FIELD = "feed_id"; + + private static final String IOC_ID_FIELD = "ioc_id"; + + private static final String INDEX_FIELD = "index"; + + private final String feedId; + + private final String iocId; + + private final String index; + + public IocWithFeeds(String iocId, String feedId, String index) { + this.iocId = iocId; + this.feedId = feedId; + this.index = index; + } + + public IocWithFeeds(StreamInput sin) throws IOException { + this.iocId = sin.readString(); + this.feedId = sin.readString(); + this.index = sin.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocId); + out.writeString(feedId); + out.writeString(index); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(IOC_ID_FIELD, iocId) + .field(FEED_ID_FIELD, feedId) + .field(INDEX_FIELD, index) + .endObject(); + return builder; + } + + public Map asTemplateArg() { + return Map.of( + FEED_ID_FIELD, feedId, + IOC_ID_FIELD, iocId, + INDEX_FIELD, index + ); + } + + public String getIocId() { + return iocId; + } + + public String getFeedId() { + return feedId; + } + + public String getIndex() { + return index; + } + + public static IocWithFeeds parse(XContentParser xcp) throws IOException { + String iocId = null; + String feedId = null; + String index = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_ID_FIELD: + iocId = xcp.text(); + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + case INDEX_FIELD: + index = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new IocWithFeeds(iocId, feedId, index); + } + + public static IocWithFeeds readFrom(StreamInput sin) throws IOException { + return new IocWithFeeds(sin); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 59bdfdf18..83bc8e567 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -31,6 +31,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.ioc_finding_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_rollover_period", TimeValue.timeValueHours(12), @@ -49,6 +55,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_max_age", new TimeValue(30, TimeUnit.DAYS), @@ -67,6 +79,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_MAX_DOCS = Setting.longSetting( "plugins.security_analytics.alert_history_max_docs", 1000L, @@ -88,6 +106,13 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.ioc_finding_history_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_retention_period", new TimeValue(60, TimeUnit.DAYS), @@ -106,6 +131,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( "plugins.security_analytics.request_timeout", TimeValue.timeValueSeconds(10), diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java new file mode 100644 index 000000000..f662ed1be --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +public class GetIocFindingsAction extends ActionType { + + public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/iocs/findings/get"; + + public GetIocFindingsAction() { + super(NAME, GetIocFindingsResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java new file mode 100644 index 000000000..1395cff1e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class GetIocFindingsRequest extends ActionRequest { + + private List findingIds; + + private List iocIds; + + private Instant startTime; + + private Instant endTime; + + private Table table; + + public GetIocFindingsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalStringList(), + sin.readOptionalStringList(), + sin.readOptionalInstant(), + sin.readOptionalInstant(), + Table.readFrom(sin) + ); + } + + public GetIocFindingsRequest(List findingIds, + List iocIds, + Instant startTime, + Instant endTime, + Table table) { + this.findingIds = findingIds; + this.iocIds = iocIds; + this.startTime = startTime; + this.endTime = endTime; + this.table = table; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (startTime != null && endTime != null && startTime.isAfter(endTime)) { + validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), + "startTime should be less than endTime"), validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalStringCollection(findingIds); + out.writeOptionalStringCollection(iocIds); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + table.writeTo(out); + } + + public List getFindingIds() { + return findingIds; + } + + public List getIocIds() { + return iocIds; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Table getTable() { + return table; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java new file mode 100644 index 000000000..4c0dea477 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.IocFinding; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetIocFindingsResponse extends ActionResponse implements ToXContentObject { + + private static final String TOTAL_IOC_FINDINGS_FIELD = "total_findings"; + + private static final String IOC_FINDINGS_FIELD = "ioc_findings"; + + private Integer totalFindings; + + private List iocFindings; + + public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { + super(); + this.totalFindings = totalFindings; + this.iocFindings = iocFindings; + } + + public GetIocFindingsResponse(StreamInput sin) throws IOException { + this( + sin.readInt(), + Collections.unmodifiableList(sin.readList(IocFinding::new)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalFindings); + out.writeCollection(iocFindings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(TOTAL_IOC_FINDINGS_FIELD, totalFindings) + .field(IOC_FINDINGS_FIELD, iocFindings); + return builder.endObject(); + } + + public Integer getTotalFindings() { + return totalFindings; + } + + public List getIocFindings() { + return iocFindings; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java new file mode 100644 index 000000000..0e1b955b1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -0,0 +1,215 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Data layer to perform CRUD operations for threat intel ioc match : store in system index. + */ +public class IocFindingService { + //TODO manage index rollover + public static final String IOC_FINDING_ALIAS_NAME = ".opensearch-sap-ioc-findings"; + + public static final String IOC_FINDING_INDEX_PATTERN = "<.opensearch-sap-ioc-findings-history-{now/d}-1>"; + + public static final String IOC_FINDING_INDEX_PATTERN_REGEXP = ".opensearch-sap-ioc-findings*"; + + private static final Logger log = LogManager.getLogger(IocFindingService.class); + private final Client client; + private final ClusterService clusterService; + + private final NamedXContentRegistry xContentRegistry; + + public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + public void indexIocMatches(List iocFindings, + final ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(IOC_FINDING_ALIAS_NAME); + for (int i = 0; i < iocFindings.size(); i++) { + IocFinding iocFinding = iocFindings.get(i); + try { + IndexRequest indexRequest = new IndexRequest(IOC_FINDING_ALIAS_NAME) + .source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for ioc match %s moving on to next"), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} Ioc Matches. Failure: {}", request.batchSize(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error("Failed to save ioc matches.", e); + } + } + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + public static String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_finding_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel ioc match index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc match index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + /** + * Index name: .opensearch-sap-iocmatch + * Mapping: /mappings/ioc_finding_mapping.json + * + * @param listener setup listener + */ + public void createIndexIfNotExists(final ActionListener listener) { + // check if job index exists + try { + if (clusterService.state().metadata().hasAlias(IOC_FINDING_ALIAS_NAME) == true) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(IOC_FINDING_INDEX_PATTERN).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING).alias(new Alias(IOC_FINDING_ALIAS_NAME)); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("Ioc match index created"); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.debug("index {} already exist", IOC_FINDING_INDEX_PATTERN); + listener.onResponse(null); + return; + } + log.error("Failed to create security analytics threat intel job index", e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failure in creating ioc_match index", e); + listener.onFailure(e); + } + } + + public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final ActionListener actionListener) { + createIndexIfNotExists(ActionListener.wrap( + r -> { + SearchRequest searchRequest = new SearchRequest() + .source(searchSourceBuilder) + .indices(IOC_FINDING_ALIAS_NAME); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; + List iocFindings = new ArrayList<>(); + + for (SearchHit hit: searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); + } + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); + } catch (Exception ex) { + this.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } + actionListener.onFailure(e); + } + }); + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java new file mode 100644 index 000000000..36927d35d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetIocFindingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_ioc_findings_action_sa"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String sortString = request.param("sortString", "timestamp"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + List findingIds = null; + if (request.param("findingIds") != null) { + findingIds = Arrays.asList(request.param("findingIds").split(",")); + } + List iocIds = null; + if (request.param("iocIds") != null) { + iocIds = Arrays.asList(request.param("iocIds").split(",")); + } + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + startTime = Instant.now(); // Default value or fallback + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + endTime = Instant.now(); // Default value or fallback + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetIocFindingsRequest getIocFindingsRequest = new GetIocFindingsRequest( + findingIds, + iocIds, + startTime, + endTime, + table + ); + return channel -> client.execute( + GetIocFindingsAction.INSTANCE, + getIocFindingsRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings" + "/_search")); + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java new file mode 100644 index 000000000..2c4792650 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -0,0 +1,144 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.util.List; + +public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { + + private final IocFindingService iocFindingService; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + @Inject + public TransportGetIocFindingsAction( + TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client + ) { + super(GetIocFindingsAction.NAME, transportService, actionFilters, GetIocFindingsRequest::new); + this.settings = settings; + this.clusterService = clusterService; + this.threadPool = client.threadPool(); + this.iocFindingService = new IocFindingService(client, this.clusterService, xContentRegistry); + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, GetIocFindingsRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isBlank()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()) + .fetchSource(new FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)) + .seqNoAndPrimaryTerm(true) + .version(true); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + List findingIds = request.getFindingIds(); + + if (findingIds != null && !findingIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("id", findingIds)); + } + + List iocIds = request.getIocIds(); + if (iocIds != null && !iocIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("ioc_feed_ids.ioc_id", iocIds)); + } + + Instant startTime = request.getStartTime(); + Instant endTime = request.getEndTime(); + if (startTime != null && endTime != null) { + long startTimeMillis = startTime.toEpochMilli(); + long endTimeMillis = endTime.toEpochMilli(); + QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp") + .from(startTimeMillis) // Greater than or equal to start time + .to(endTimeMillis); // Less than or equal to end time + queryBuilder.filter(timeRangeQuery); + } + + if (tableProp.getSearchString() != null && !tableProp.getSearchString().isBlank()) { + queryBuilder.should(QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + ).should( + QueryBuilders.nestedQuery( + "queries", + QueryBuilders.boolQuery() + .must( + QueryBuilders.queryStringQuery(tableProp.getSearchString()) + ), + ScoreMode.Avg + ) + ); + } + searchSourceBuilder.query(queryBuilder).trackTotalHits(true); + + this.threadPool.getThreadContext().stashContext(); + iocFindingService.searchIocMatches(searchSourceBuilder, actionListener); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/ioc_match_mapping.json b/src/main/resources/mappings/ioc_finding_mapping.json similarity index 63% rename from src/main/resources/mappings/ioc_match_mapping.json rename to src/main/resources/mappings/ioc_finding_mapping.json index f4573190e..2353bf14e 100644 --- a/src/main/resources/mappings/ioc_match_mapping.json +++ b/src/main/resources/mappings/ioc_finding_mapping.json @@ -7,16 +7,27 @@ "schema_version": { "type": "integer" }, - "feed_ids" : { - "type": "keyword" + "ioc_feed_ids" : { + "type": "object", + "properties": { + "feed_id": { + "type": "keyword" + }, + "ioc_id": { + "type": "keyword" + }, + "index": { + "type": "keyword" + } + } }, "related_doc_ids": { "type": "keyword" }, - "ioc_scan_job_id": { + "monitor_id": { "type": "keyword" }, - "ioc_scan_job_name": { + "monitor_name": { "type": "keyword" }, "id": { diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 68b485129..52f1d4b5d 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -61,6 +61,8 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.util.CorrelationIndices; @@ -671,6 +673,10 @@ protected HttpEntity toHttpEntity(ThreatIntelMonitorDto threatIntelMonitorDto) t return new StringEntity(toJsonString(threatIntelMonitorDto), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } + protected HttpEntity toHttpEntity(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { return new StringEntity(toJsonString(testS3ConnectionRequest), ContentType.APPLICATION_JSON); } @@ -728,6 +734,11 @@ private String toJsonString(ThreatIntelMonitorDto threatIntelMonitorDto) throws return IndexUtilsKt.string(shuffleXContent(threatIntelMonitorDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String toJsonString(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); return IndexUtilsKt.string(shuffleXContent(testS3ConnectionRequest.toXContent(builder, ToXContent.EMPTY_PARAMS))); @@ -1489,6 +1500,24 @@ public List getAlertIndices(String detectorType) throws IOException { return indices; } + public List getIocFindingIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + public List getQueryIndices(String detectorType) throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getRuleIndex(detectorType) + "*?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 132ad4123..3dccd142c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -28,7 +28,7 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.IoCMatch; +import org.opensearch.securityanalytics.model.IocFinding; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; @@ -809,9 +809,9 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } - public static String toJsonString(IoCMatch iocMatch) throws IOException { + public static String toJsonString(IocFinding iocFinding) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); - builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder = iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS); return BytesReference.bytes(builder).utf8ToString(); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java deleted file mode 100644 index 4b56c7eb5..000000000 --- a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.opensearch.securityanalytics.model; - -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.time.Instant; -import java.util.List; - -import static org.opensearch.securityanalytics.TestHelpers.toJsonString; - -public class IoCMatchTests extends OpenSearchTestCase { - - public void testIoCMatchAsAStream() throws IOException { - IoCMatch iocMatch = getRandomIoCMatch(); - String jsonString = toJsonString(iocMatch); - BytesStreamOutput out = new BytesStreamOutput(); - iocMatch.writeTo(out); - StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); - assertEquals(iocMatch.getId(), newIocMatch.getId()); - assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); - assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); - assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); - assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); - assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); - assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); - assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); - assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); - } - - public void testIoCMatchParse() throws IOException { - String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + - "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + - " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + - "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; - IoCMatch iocMatch = IoCMatch.parse((getParser(iocMatchString))); - BytesStreamOutput out = new BytesStreamOutput(); - iocMatch.writeTo(out); - StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); - assertEquals(iocMatch.getId(), newIocMatch.getId()); - assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); - assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); - assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); - assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); - assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); - assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); - assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); - assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); - } - - public XContentParser getParser(String xc) throws IOException { - XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); - parser.nextToken(); - return parser; - - } - - private static IoCMatch getRandomIoCMatch() { - return new IoCMatch( - randomAlphaOfLength(10), - List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), - List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - Instant.now(), - randomAlphaOfLength(10)); - } - - -} diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java new file mode 100644 index 000000000..8acf10744 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -0,0 +1,78 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IocFindingTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IocFinding iocFinding = getRandomIoCMatch(); + String jsonString = toJsonString(iocFinding); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getMonitorId(), newIocFinding.getMonitorId()); + assertEquals(iocFinding.getMonitorName(), newIocFinding.getMonitorName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public void testIoCMatchParse() throws IOException { + String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + + "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; + IocFinding iocFinding = IocFinding.parse((getParser(iocMatchString))); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getMonitorId(), newIocFinding.getMonitorId()); + assertEquals(iocFinding.getMonitorName(), newIocFinding.getMonitorName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static IocFinding getRandomIoCMatch() { + return new IocFinding( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } + + +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java new file mode 100644 index 000000000..5c66d50bb --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.model.IocWithFeeds; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.*; + +public class IocFindingServiceRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testGetIocFindings() throws IOException { + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(5, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + @SuppressWarnings("unchecked") + public void testGetIocFindingsWithIocIdFilter() throws IOException { + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + String iocId = iocFindings.stream().map(iocFinding -> iocFinding.getFeedIds().get(0).getIocId()).findFirst().get(); + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?iocIds=" + iocId, + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(1, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + public void testGetIocFindingsRolloverByMaxDocs() throws IOException, InterruptedException { + updateClusterSetting(IOC_FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(IOC_FINDING_HISTORY_MAX_DOCS.getKey(), "1"); + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(5); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + AtomicBoolean found = new AtomicBoolean(false); + OpenSearchTestCase.waitUntil(() -> { + try { + found.set(getIocFindingIndices().size() == 2); + return found.get(); + } catch (IOException e) { + return false; + } + }, 30000, TimeUnit.SECONDS); + Assert.assertTrue(found.get()); + } + + public void testGetIocFindingsRolloverByMaxAge() throws IOException, InterruptedException { + updateClusterSetting(IOC_FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(IOC_FINDING_HISTORY_MAX_DOCS.getKey(), "1000"); + updateClusterSetting(IOC_FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(5); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + AtomicBoolean found = new AtomicBoolean(false); + OpenSearchTestCase.waitUntil(() -> { + try { + found.set(getIocFindingIndices().size() == 2); + return found.get(); + } catch (IOException e) { + return false; + } + }, 30000, TimeUnit.SECONDS); + Assert.assertTrue(found.get()); + + updateClusterSetting(IOC_FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1000s"); + updateClusterSetting(IOC_FINDING_HISTORY_RETENTION_PERIOD.getKey(), "1s"); + + AtomicBoolean retFound = new AtomicBoolean(false); + OpenSearchTestCase.waitUntil(() -> { + try { + retFound.set(getIocFindingIndices().size() == 1); + return retFound.get(); + } catch (IOException e) { + return false; + } + }, 30000, TimeUnit.SECONDS); + Assert.assertTrue(retFound.get()); + } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file