From 4d5df3a93357593516b93bb6d24abd0b8707610d Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 25 Jun 2024 10:42:01 -0700 Subject: [PATCH 1/4] fix user mappings (#1095) Signed-off-by: Joanne Wang --- .../mappings/threat_intel_job_mapping.json | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index 4618ad9b1..bf237aded 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -27,7 +27,41 @@ "type": "text" }, "created_by_user": { - "type": "keyword" + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } }, "created_at": { "type": "date", @@ -92,7 +126,41 @@ "format": "strict_date_time||epoch_millis" }, "last_refreshed_user": { - "type": "keyword" + "properties": { + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "backend_roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "roles": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "custom_attribute_names": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } }, "enabled": { "type": "boolean" From 23a6b6d9678593802436dab51b17ce5c4d2e9ea4 Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 25 Jun 2024 14:09:38 -0700 Subject: [PATCH 2/4] Logic to delete old iocs and add ioc index rollover (#1094) * wip Signed-off-by: Joanne Wang * comments Signed-off-by: Joanne Wang * working Signed-off-by: Joanne Wang * delete ioc indices for delete api Signed-off-by: Joanne Wang * working rn Signed-off-by: Joanne Wang * cleanup Signed-off-by: Joanne Wang * comments Signed-off-by: Joanne Wang --------- Signed-off-by: Joanne Wang --- .../SecurityAnalyticsPlugin.java | 6 +- .../services/STIX2IOCFeedStore.java | 140 ++++++--- .../settings/SecurityAnalyticsSettings.java | 15 + .../SATIFSourceConfigManagementService.java | 287 ++++++++++++++++-- .../service/SATIFSourceConfigService.java | 75 ++++- .../resthandler/ListIOCsRestApiIT.java | 2 +- .../SATIFSourceConfigRestApiIT.java | 2 +- 7 files changed, 452 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 3b5a20ad6..c8168d428 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -294,7 +294,7 @@ public Collection createComponents(Client client, TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); saTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); STIX2IOCFetchService stix2IOCFetchService = new STIX2IOCFetchService(client, clusterService); - SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry); + SATIFSourceConfigManagementService saTifSourceConfigManagementService = new SATIFSourceConfigManagementService(saTifSourceConfigService, threatIntelLockService, stix2IOCFetchService, xContentRegistry, clusterService); SecurityAnalyticsRunner.getJobRunnerInstance(); TIFSourceConfigRunner.getJobRunnerInstance().initialize(clusterService, threatIntelLockService, threadPool, saTifSourceConfigManagementService, saTifSourceConfigService); TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); @@ -456,7 +456,9 @@ public List> getSettings() { SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL, SecurityAnalyticsSettings.BATCH_SIZE, - SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT + SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT, + SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD, + SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS ); } diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index d2945bdf0..aad65b24f 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -10,14 +10,18 @@ import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.admin.indices.rollover.RolloverRequest; +import org.opensearch.action.admin.indices.rollover.RolloverResponse; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.Streams; @@ -32,6 +36,7 @@ import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.util.IndexUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -50,10 +55,8 @@ public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID"; public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER; - - // TODO hurneyt implement history indexes + rollover logic - public static final String IOC_HISTORY_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE + "-history-write"; - public static final String IOC_HISTORY_INDEX_PATTERN = "<." + IOC_INDEX_NAME_BASE + "-history-{now/d{yyyy.MM.dd.hh.mm.ss|UTC}}-1>"; + public static final String IOC_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE + "-write"; + public static final String IOC_INDEX_PATTERN = "<" + IOC_INDEX_NAME_TEMPLATE + "-" + Instant.now().toEpochMilli() +"-000001>"; private final Logger log = LogManager.getLogger(STIX2IOCFeedStore.class); Instant startTime = Instant.now(); @@ -112,23 +115,79 @@ public void storeIOCs(Map actionToIOCs) { } public void indexIocs(List iocs) throws IOException { - String feedIndexName = getFeedConfigIndexName(saTifSourceConfig.getId()); - - // init index and add name to ioc map store only if index does not already exist, otherwise ioc map store will contain duplicate index names - if (feedIndexExists(feedIndexName) == false) { - initFeedIndex(feedIndexName); - saTifSourceConfig.getIocTypes().forEach(type -> { - String lowerCaseType = type.toLowerCase(Locale.ROOT); - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().putIfAbsent(lowerCaseType, new ArrayList<>()); - ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(feedIndexName); - }); + String iocAlias = getIocIndexAlias(saTifSourceConfig.getId()); + String iocPattern = getIocIndexRolloverPattern(saTifSourceConfig.getId()); + + if (iocIndexExists(iocAlias) == false) { + initFeedIndex(iocAlias, iocPattern, ActionListener.wrap( + r -> { + saTifSourceConfig.getIocTypes().forEach(type -> { + String writeIndex = IndexUtils.getWriteIndex(iocAlias, clusterService.state()); + String lowerCaseType = type.toLowerCase(Locale.ROOT); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().putIfAbsent(lowerCaseType, new ArrayList<>()); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(iocAlias); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(writeIndex); + }); + bulkIndexIocs(iocs, iocAlias); + }, e-> { + log.error("Failed to initialize the IOC index and save the IOCs", e); + baseListener.onFailure(e); + } + )); + } else { + rolloverIndex(iocAlias, iocPattern, ActionListener.wrap( + r -> { + saTifSourceConfig.getIocTypes().forEach(type -> { + String writeIndex = IndexUtils.getWriteIndex(iocAlias, clusterService.state()); + String lowerCaseType = type.toLowerCase(Locale.ROOT); + ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore().get(lowerCaseType).add(writeIndex); + }); + bulkIndexIocs(iocs, iocAlias); + }, e -> { + log.error("Failed to rollover the IOC index and save the IOCs", e); + baseListener.onFailure(e); + } + )); } + } + private void rolloverIndex( + String alias, + String pattern, + ActionListener listener + ) { + if (clusterService.state().metadata().hasAlias(alias) == false) { + listener.onFailure(new OpenSearchException("Alias not initialized")); + return; + } + // We have to pass null for newIndexName in order to get Elastic to increment the alias count. + RolloverRequest request = new RolloverRequest(alias, null); + request.getCreateIndexRequest().index(pattern) + .mapping(iocIndexMapping()) + .settings(Settings.builder().put("index.hidden", true).build()); + client.admin().indices().rolloverIndex( + request, + ActionListener.wrap( + rolloverResponse -> { + if (!rolloverResponse.isRolledOver()) { + log.info(alias + "not rolled over. Conditions were: " + rolloverResponse.getConditionStatus()); + } else { + listener.onResponse(rolloverResponse); + } + }, e -> { + log.error("rollover failed for alias [" + alias + "]."); + listener.onFailure(e); + } + ) + ); + } + + private void bulkIndexIocs(List iocs, String iocAlias) throws IOException { List bulkRequestList = new ArrayList<>(); BulkRequest bulkRequest = new BulkRequest(); for (STIX2IOC ioc : iocs) { - IndexRequest indexRequest = new IndexRequest(feedIndexName) + IndexRequest indexRequest = new IndexRequest(iocAlias) .opType(DocWriteRequest.OpType.INDEX) .source(ioc.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)); bulkRequest.add(indexRequest); @@ -154,11 +213,10 @@ public void indexIocs(List iocs) throws IOException { } idx++; } - long duration = Duration.between(startTime, Instant.now()).toMillis(); STIX2IOCFetchService.STIX2IOCFetchResponse output = new STIX2IOCFetchService.STIX2IOCFetchResponse(iocs, duration); baseListener.onResponse(output); - }, e -> { + }, e -> { log.error("Failed to index IOCs for config {}", saTifSourceConfig.getId(), e); baseListener.onFailure(e); }), bulkRequestList.size()); @@ -173,38 +231,35 @@ public void indexIocs(List iocs) throws IOException { } } - /** - * Checks whether the [IOC_INDEX_NAME_BASE]-related index exists. - * @param index The index to evaluate. - * @return TRUE if the index is an IOC-related system index, and exists; else returns FALSE. - */ - public boolean feedIndexExists(String index) { - return index.startsWith(IOC_INDEX_NAME_BASE) && this.clusterService.state().routingTable().hasIndex(index); + public boolean iocIndexExists(String alias) { + ClusterState clusterState = clusterService.state(); + return clusterState.metadata().hasAlias(alias); } - public static String getFeedConfigIndexName(String feedSourceConfigId) { - return IOC_INDEX_NAME_TEMPLATE.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); + public static String getIocIndexAlias(String feedSourceConfigId) { + return IOC_WRITE_INDEX_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); } - public void initFeedIndex(String feedIndexName) { + public static String getIocIndexRolloverPattern(String feedSourceConfigId) { + return IOC_INDEX_PATTERN.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); + } + + + public void initFeedIndex(String feedAliasName, String feedIndexName, ActionListener listener) { var indexRequest = new CreateIndexRequest(feedIndexName) .mapping(iocIndexMapping()) .settings(Settings.builder().put("index.hidden", true).build()); - - ActionListener createListener = new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse createIndexResponse) { - log.info("Created system index {}", feedIndexName); - } - - @Override - public void onFailure(Exception e) { - log.error("Failed to create system index {}", feedIndexName); - baseListener.onFailure(e); - } - }; - - client.admin().indices().create(indexRequest, createListener); + indexRequest.alias(new Alias(feedAliasName)); // set the alias + client.admin().indices().create(indexRequest, ActionListener.wrap( + r -> { + log.info("Created system index {}", feedIndexName); + listener.onResponse(r); + }, + e -> { + log.error("Failed to create system index {}", feedIndexName); + listener.onFailure(e); + } + )); } public String iocIndexMapping() { @@ -222,3 +277,4 @@ public SATIFSourceConfig getSaTifSourceConfig() { return saTifSourceConfig; } } + diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index fefe7c288..59bdfdf18 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -191,4 +191,19 @@ public static final List> settings() { return List.of(BATCH_SIZE, THREAT_INTEL_TIMEOUT, TIF_UPDATE_INTERVAL); } + // Threat Intel IOC Settings + public static final Setting IOC_INDEX_RETENTION_PERIOD = Setting.timeSetting( + "plugins.security_analytics.ioc.index_retention_period", + new TimeValue(30, TimeUnit.DAYS), + new TimeValue(1, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting IOC_MAX_INDICES_PER_ALIAS = Setting.intSetting( + "plugins.security_analytics.ioc.max_indices_per_alias", + 30, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 6477b308b..79a45bfe7 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -3,10 +3,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.action.StepListener; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.IndexAbstraction; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.routing.Preference; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; @@ -27,13 +33,23 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.IocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.IndexUtils; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; + +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getIocIndexAlias; + /** * Service class for threat intel feed source config object @@ -44,6 +60,7 @@ public class SATIFSourceConfigManagementService { private final TIFLockService lockService; //TODO: change to js impl lock private final STIX2IOCFetchService stix2IOCFetchService; private final NamedXContentRegistry xContentRegistry; + private final ClusterService clusterService; /** * Default constructor @@ -57,13 +74,14 @@ public SATIFSourceConfigManagementService( final SATIFSourceConfigService saTifSourceConfigService, final TIFLockService lockService, final STIX2IOCFetchService stix2IOCFetchService, - NamedXContentRegistry xContentRegistry - + final NamedXContentRegistry xContentRegistry, + final ClusterService clusterService ) { this.saTifSourceConfigService = saTifSourceConfigService; this.lockService = lockService; this.stix2IOCFetchService = stix2IOCFetchService; this.xContentRegistry = xContentRegistry; + this.clusterService = clusterService; } public void createOrUpdateTifSourceConfig( @@ -317,45 +335,49 @@ public void refreshTIFSourceConfig( } // REFRESH FLOW - log.info("Refreshing IOCs and updating threat intel source config"); // place holder - + log.debug("Refreshing IOCs and updating threat intel source config"); // place holder markSourceConfigAsAction(saTifSourceConfig, TIFJobState.REFRESHING, ActionListener.wrap( updatedSourceConfig -> { // TODO: download and save iocs listener should return the source config, sync up with @hurneyt downloadAndSaveIOCs(updatedSourceConfig, ActionListener.wrap( - // 1. call refresh IOC method (download and save IOCs) - // 1a. set state to refreshing - // 1b. delete old indices - // 1c. update or create iocs response -> { - // 2. update source config as succeeded - markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap( - r -> { - log.debug("Set threat intel source config as AVAILABLE for [{}]", updatedSourceConfig.getId()); - SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(updatedSourceConfig); - listener.onResponse(returnedSaTifSourceConfigDto); - }, ex -> { - log.error("Failed to set threat intel source config as AVAILABLE for [{}]", updatedSourceConfig.getId()); - listener.onFailure(ex); + // delete old IOCs and update the source config + deleteOldIocIndices(updatedSourceConfig, ActionListener.wrap( + newIocStoreConfig -> { + updatedSourceConfig.setIocStoreConfig(newIocStoreConfig); + // Update source config as succeeded, change state back to available + markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap( + r -> { + log.debug("Set threat intel source config as AVAILABLE for [{}]", updatedSourceConfig.getId()); + SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(updatedSourceConfig); + listener.onResponse(returnedSaTifSourceConfigDto); + }, ex -> { + log.error("Failed to set threat intel source config as AVAILABLE for [{}]", updatedSourceConfig.getId()); + listener.onFailure(ex); + } + )); + } , deleteIocIndicesError -> { + log.error("Failed to delete old IOC indices", deleteIocIndicesError); + listener.onFailure(deleteIocIndicesError); } )); - }, e -> { - // 3. update source config as failed + }, downloadAndSaveIocsError -> { + // Update source config as refresh failed log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId()); markSourceConfigAsAction(updatedSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap( r -> { log.debug("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId())); - }, ex -> { + }, e -> { log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()); - listener.onFailure(ex); + listener.onFailure(e); } )); - listener.onFailure(e); + listener.onFailure(downloadAndSaveIocsError); })); - }, ex -> { + }, e -> { log.error("Failed to set threat intel source config as REFRESHING for [{}]", saTifSourceConfig.getId()); - listener.onFailure(ex); + listener.onFailure(e); } )); }, e -> { @@ -395,6 +417,217 @@ public void deleteTIFSourceConfig( )); } + /** + * Deletes the old ioc indices based on retention age and number of indices per alias + * @param saTifSourceConfig + * @param listener + */ + public void deleteOldIocIndices ( + final SATIFSourceConfig saTifSourceConfig, + ActionListener listener + ) { + Map> iocToAliasMap = ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocMapStore(); + + // Grabbing the first ioc type since all the indices are stored in one index + String type = saTifSourceConfig.getIocTypes().get(0); + String alias = getIocIndexAlias(saTifSourceConfig.getId()); + + List iocIndicesDeleted = new ArrayList<>(); + StepListener> deleteIocIndicesByAgeListener = new StepListener<>(); + + List indicesWithoutAlias = new ArrayList<>(iocToAliasMap.get(type)); + indicesWithoutAlias.remove(alias); + checkAndDeleteOldIocIndicesByAge(indicesWithoutAlias, deleteIocIndicesByAgeListener, alias); + deleteIocIndicesByAgeListener.whenComplete( + iocIndicesDeletedByAge-> { + // remove indices deleted by age from the ioc map and add to ioc indices deleted list + iocToAliasMap.get(type).removeAll(iocIndicesDeletedByAge); + iocIndicesDeleted.addAll(iocIndicesDeletedByAge); + + List newIndicesWithoutAlias = new ArrayList<>(iocToAliasMap.get(type)); + newIndicesWithoutAlias.remove(alias); + checkAndDeleteOldIocIndicesBySize(newIndicesWithoutAlias, alias, ActionListener.wrap( + iocIndicesDeletedBySize -> { + iocToAliasMap.get(type).removeAll(iocIndicesDeletedBySize); + iocIndicesDeleted.addAll(iocIndicesDeletedBySize); + + // delete the ioc indices for other IOC types + saTifSourceConfig.getIocTypes() + .stream() + .filter(iocType -> iocType.equals(type) == false) + .forEach(iocType -> iocToAliasMap.get(iocType).removeAll(iocIndicesDeleted)); + listener.onResponse(new DefaultIocStoreConfig(iocToAliasMap)); + }, e -> { + log.error("Failed to check and delete ioc indices by size", e); + listener.onFailure(e); + } + )); + }, e -> { + log.error("Failed to check and delete ioc indices by age", e); + listener.onFailure(e); + }); + } + + /** + * Checks if any IOC index is greater than retention period and deletes it + * @param indices + * @param stepListener + * @param alias + */ + private void checkAndDeleteOldIocIndicesByAge( + List indices, + StepListener> stepListener, + String alias + ) { + log.debug("Delete old IOC indices by age"); + saTifSourceConfigService.getClusterState( + ActionListener.wrap( + clusterStateResponse -> { + List indicesToDelete = new ArrayList<>(); + if (!clusterStateResponse.getState().metadata().getIndices().isEmpty()) { + log.debug("Checking if we should delete indices: [" + indicesToDelete + "]"); + indicesToDelete = getIocIndicesToDeleteByAge(clusterStateResponse, alias); + if (indicesToDelete.isEmpty() == false) { + saTifSourceConfigService.deleteAllOldIocIndices(indicesToDelete); + } + } + stepListener.onResponse(indicesToDelete); + }, e -> { + log.error("Failed to get the cluster metadata"); + stepListener.onFailure(e); + } + ), indices.toArray(new String[0]) + ); + } + + /** + * Checks if number of allowed indices per alias is reached and delete old indices + * @param indices + * @param alias + * @param listener + */ + private void checkAndDeleteOldIocIndicesBySize( + List indices, + String alias, + ActionListener> listener + ) { + log.debug("Delete old IOC indices by size"); + saTifSourceConfigService.getClusterState( + ActionListener.wrap( + clusterStateResponse -> { + List indicesToDelete = new ArrayList<>(); + if (!clusterStateResponse.getState().metadata().getIndices().isEmpty()) { + Integer numIndicesToDelete = numOfIndicesToDelete(indices); + if (numIndicesToDelete > 0) { + indicesToDelete = getIocIndicesToDeleteBySize(clusterStateResponse, numIndicesToDelete, indices, alias); + if (indicesToDelete.isEmpty() == false) { + saTifSourceConfigService.deleteAllOldIocIndices(indicesToDelete); + } + } + } + listener.onResponse(indicesToDelete); + }, e -> { + log.error("Failed to get the cluster metadata"); + listener.onFailure(e); + } + ), indices.toArray(new String[0]) + ); + } + + /** + * Helper function to retrieve a list of IOC indices to delete based on retention age + * @param clusterStateResponse + * @param alias + * @return indicesToDelete + */ + private List getIocIndicesToDeleteByAge( + ClusterStateResponse clusterStateResponse, + String alias + ) { + List indicesToDelete = new ArrayList<>(); + String writeIndex = IndexUtils.getWriteIndex(alias, clusterStateResponse.getState()); + Long maxRetentionPeriod = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD).millis(); + + for (IndexMetadata indexMetadata : clusterStateResponse.getState().metadata().indices().values()) { + Long creationTime = indexMetadata.getCreationDate(); + if ((Instant.now().toEpochMilli() - creationTime) > maxRetentionPeriod) { + String indexToDelete = indexMetadata.getIndex().getName(); + // ensure index is not the current write index + if (indexToDelete.equals(writeIndex) == false) { + indicesToDelete.add(indexToDelete); + } + } + } + return indicesToDelete; + } + + /** + * Helper function to retrieve a list of IOC indices to delete based on number of indices associated with alias + * @param clusterStateResponse + * @param numOfIndices + * @param concreteIndices + * @param alias + * @return indicesToDelete + */ + private List getIocIndicesToDeleteBySize( + ClusterStateResponse clusterStateResponse, + Integer numOfIndices, + List concreteIndices, + String alias + ) { + List indicesToDelete = new ArrayList<>(); + String writeIndex = IndexUtils.getWriteIndex(alias, clusterStateResponse.getState()); + + for (int i = 0; i < numOfIndices; i++) { + String indexToDelete = getOldestIndexByCreationDate(concreteIndices, clusterStateResponse.getState(), indicesToDelete); + if (indexToDelete.equals(writeIndex) == false ) { + indicesToDelete.add(indexToDelete); + } + } + return indicesToDelete; + } + + /** + * Helper function to retrieve oldest index in a list of concrete indices + * @param concreteIndices + * @param clusterState + * @param indicesToDelete + * @return oldestIndex + */ + private static String getOldestIndexByCreationDate( + List concreteIndices, + ClusterState clusterState, + List indicesToDelete + ) { + final SortedMap lookup = clusterState.getMetadata().getIndicesLookup(); + long minCreationDate = Long.MAX_VALUE; + String oldestIndex = null; + for (String indexName : concreteIndices) { + IndexAbstraction index = lookup.get(indexName); + IndexMetadata indexMetadata = clusterState.getMetadata().index(indexName); + if(index != null && index.getType() == IndexAbstraction.Type.CONCRETE_INDEX) { + if (indexMetadata.getCreationDate() < minCreationDate && indicesToDelete.contains(indexName) == false) { + minCreationDate = indexMetadata.getCreationDate(); + oldestIndex = indexName; + } + } + } + return oldestIndex; + } + + /** + * Helper function to determine how many indices should be deleted based on setting for number of indices per alias + * @param concreteIndices + * @return + */ + private Integer numOfIndicesToDelete(List concreteIndices) { + Integer maxIndicesPerAlias = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_ALIAS); + if (concreteIndices.size() > maxIndicesPerAlias ) { + return concreteIndices.size() - maxIndicesPerAlias; + } + return 0; + } + private void onDeleteThreatIntelMonitors(String saTifSourceConfigId, ActionListener listener, SATIFSourceConfig saTifSourceConfig, Boolean isDeleted) { if (isDeleted == false) { listener.onFailure(new IllegalArgumentException("All threat intel monitors need to be deleted before deleting last threat intel source config")); @@ -405,7 +638,11 @@ private void onDeleteThreatIntelMonitors(String saTifSourceConfigId, ActionListe TIFJobState.DELETING, ActionListener.wrap( updateSaTifSourceConfigResponse -> { - // TODO: Delete all IOCs associated with source config then delete source config, sync up with @hurneyt + String type = updateSaTifSourceConfigResponse.getIocTypes().get(0); + DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) updateSaTifSourceConfigResponse.getIocStoreConfig(); + List indicesWithoutAlias = new ArrayList<>(iocStoreConfig.getIocMapStore().get(type)); + indicesWithoutAlias.remove(getIocIndexAlias(updateSaTifSourceConfigResponse.getId())); + saTifSourceConfigService.deleteAllOldIocIndices(indicesWithoutAlias); saTifSourceConfigService.deleteTIFSourceConfig(saTifSourceConfig, ActionListener.wrap( deleteResponse -> { log.debug("Successfully deleted threat intel source config [{}]", saTifSourceConfig.getId()); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 1124ce3f4..a6bb8b468 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -11,7 +11,10 @@ import org.opensearch.OpenSearchStatusException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.StepListener; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.get.GetRequest; @@ -19,6 +22,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; import org.opensearch.cluster.routing.Preference; @@ -38,8 +42,6 @@ import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.rest.BytesRestResponse; -import org.opensearch.rest.RestResponse; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.subphase.FetchSourceContext; @@ -49,7 +51,6 @@ import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; -import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.threadpool.ThreadPool; @@ -58,10 +59,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Locale; import java.util.stream.Collectors; -import static org.opensearch.core.rest.RestStatus.OK; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT; import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; @@ -329,6 +330,72 @@ public void deleteTIFSourceConfig( )); } + public void deleteAllOldIocIndices(List indicesToDelete) { + if (indicesToDelete.isEmpty() == false) { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + client.admin().indices().delete( + deleteIndexRequest, + ActionListener.wrap( + deleteIndicesResponse -> { + if (!deleteIndicesResponse.isAcknowledged()) { + log.error("Could not delete one or more IOC indices: [" + indicesToDelete + "]. Retrying one by one."); + deleteOldIocIndex(indicesToDelete); + } else { + log.info("Successfully deleted indices: [" + indicesToDelete + "]"); + } + }, e -> { + log.error("Delete for IOC Indices failed: [" + indicesToDelete + "]. Retrying one By one."); + deleteOldIocIndex(indicesToDelete); + } + ) + ); + } + } + + private void deleteOldIocIndex(List indicesToDelete) { + for (String index : indicesToDelete) { + final DeleteIndexRequest singleDeleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); + client.admin().indices().delete( + singleDeleteRequest, + ActionListener.wrap( + response -> { + if (!response.isAcknowledged()) { + log.error("Could not delete one or more IOC indices: " + index); + } + }, e -> { + log.debug("Exception: [" + e.getMessage() + "] while deleting the index " + index); + } + ) + ); + } + } + + public void getClusterState( + final ActionListener actionListener, + String... indices) + { + ClusterStateRequest clusterStateRequest = new ClusterStateRequest() + .clear() + .indices(indices) + .metadata(true) + .local(true) + .indicesOptions(IndicesOptions.strictExpand()); + client.admin().cluster().state( + clusterStateRequest, + ActionListener.wrap( + clusterStateResponse -> { + log.debug("Successfully retrieved cluster state"); + actionListener.onResponse(clusterStateResponse); + }, e -> { + log.error("Error fetching cluster state"); + actionListener.onFailure(e); + } + ) + ); + } + + + public void checkAndEnsureThreatIntelMonitorsDeleted( ActionListener listener ) { diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java index 3cd4d3d5c..59c01746c 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -87,7 +87,7 @@ public void cleanUp() throws IOException { public void test_retrievesIOCs() throws IOException { // Create index with mappings testFeedSourceConfigId = TestHelpers.randomLowerCaseString(); - indexName = STIX2IOCFeedStore.getFeedConfigIndexName(testFeedSourceConfigId); + indexName = STIX2IOCFeedStore.getIocIndexAlias(testFeedSourceConfigId); try { createIndex(indexName, Settings.EMPTY, indexMapping); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index 339e4fb9b..3afebdf5c 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -379,7 +379,7 @@ public void testRetrieveIOCsSuccessfully() throws IOException, InterruptedExcept }, 240, TimeUnit.SECONDS); // Confirm IOCs were ingested to system index for the feed - String indexName = STIX2IOCFeedStore.getFeedConfigIndexName(createdId); + String indexName = STIX2IOCFeedStore.getIocIndexAlias(createdId); String request = "{\n" + " \"query\" : {\n" + " \"match_all\":{\n" + From ad8002441d690f409645b6b91d30697a1a08134d Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Wed, 26 Jun 2024 00:50:25 -0700 Subject: [PATCH 3/4] fix threat intel monitor request in indexing flow Signed-off-by: Surya Sashank Nistala --- .../monitor/ThreatIntelMonitorDto.java | 44 ++++++++++++------- .../monitor/ThreatIntelTriggerDto.java | 4 +- .../util/ThreatIntelMonitorUtils.java | 36 +++++---------- .../resthandler/DetectorRestApiIT.java | 31 ++++++++----- .../resthandler/ListIOCsRestApiIT.java | 1 + .../ThreatIntelMonitorRestApiIT.java | 5 +-- 6 files changed, 66 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java index 23352581a..c4a0b9ed4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelMonitorDto.java @@ -1,6 +1,7 @@ package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; import org.apache.commons.lang3.StringUtils; +import org.opensearch.commons.alerting.model.CronSchedule; import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.commons.authuser.User; @@ -15,7 +16,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.UUID; public class ThreatIntelMonitorDto implements Writeable, ToXContentObject, ThreatIntelMonitorDtoInterface { @@ -33,17 +38,33 @@ public class ThreatIntelMonitorDto implements Writeable, ToXContentObject, Threa private final List indices; private final List triggers; - public ThreatIntelMonitorDto(String id, String name, List perIocTypeScanInputList, Schedule schedule, boolean enabled, User user, List indices, List triggers) { + public ThreatIntelMonitorDto(String id, String name, List perIocTypeScanInputList, Schedule schedule, boolean enabled, User user, List triggers) { this.id = StringUtils.isBlank(id) ? UUID.randomUUID().toString() : id; this.name = name; this.perIocTypeScanInputList = perIocTypeScanInputList; this.schedule = schedule; this.enabled = enabled; this.user = user; - this.indices = indices; + this.indices = getIndices(perIocTypeScanInputList); this.triggers = triggers; } + private List getIndices(List perIocTypeScanInputList) { + if (perIocTypeScanInputList == null) + return Collections.emptyList(); + List list = new ArrayList<>(); + Set uniqueValues = new HashSet<>(); + for (PerIocTypeScanInputDto dto : perIocTypeScanInputList) { + Map> indexToFieldsMap = dto.getIndexToFieldsMap() == null ? Collections.emptyMap() : dto.getIndexToFieldsMap(); + for (String s : indexToFieldsMap.keySet()) { + if (uniqueValues.add(s)) { + list.add(s); + } + } + } + return list; + } + public ThreatIntelMonitorDto(StreamInput sin) throws IOException { this( sin.readOptionalString(), @@ -52,7 +73,6 @@ public ThreatIntelMonitorDto(StreamInput sin) throws IOException { Schedule.readFrom(sin), sin.readBoolean(), sin.readBoolean() ? new User(sin) : null, - sin.readStringList(), sin.readList(ThreatIntelTriggerDto::new)); } @@ -66,9 +86,7 @@ public static ThreatIntelMonitorDto parse(XContentParser xcp, String id, Long ve Schedule schedule = null; Boolean enabled = null; User user = null; - List indices = new ArrayList<>(); List triggers = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); @@ -103,22 +121,13 @@ public static ThreatIntelMonitorDto parse(XContentParser xcp, String id, Long ve case Monitor.USER_FIELD: user = xcp.currentToken() == XContentParser.Token.VALUE_NULL ? null : User.parse(xcp); break; - - case INDICES: - List strings = new ArrayList<>(); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); - while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - strings.add(xcp.text()); - } - indices.addAll(strings); - break; default: xcp.skipChildren(); break; } } - return new ThreatIntelMonitorDto(id, name, inputs, schedule, enabled != null ? enabled : false, user, indices, triggers); + return new ThreatIntelMonitorDto(id, name, inputs, schedule, enabled != null ? enabled : false, user, triggers); } @Override @@ -126,6 +135,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(id); out.writeString(name); out.writeList(perIocTypeScanInputList); + if (schedule instanceof CronSchedule) { + out.writeEnum(Schedule.TYPE.CRON); + } else { + out.writeEnum(Schedule.TYPE.INTERVAL); + } schedule.writeTo(out); out.writeBoolean(enabled); user.writeTo(out); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java index 0fbb40d93..d82381b3d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java @@ -1,5 +1,6 @@ package org.opensearch.securityanalytics.threatIntel.sacommons.monitor; +import org.apache.commons.lang3.StringUtils; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -13,6 +14,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.UUID; public class ThreatIntelTriggerDto implements Writeable, ToXContentObject { @@ -35,7 +37,7 @@ public ThreatIntelTriggerDto(List dataSources, List iocTypes, Li this.iocTypes = iocTypes == null ? Collections.emptyList() : iocTypes; this.actions = actions; this.name = name; - this.id = id; + this.id = StringUtils.isBlank(id) ? UUID.randomUUID().toString() : id; this.severity = severity; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java index 81f148d88..4e149e1fd 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelMonitorUtils.java @@ -1,16 +1,12 @@ package org.opensearch.securityanalytics.threatIntel.util; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.model.Monitor; import org.opensearch.commons.alerting.model.Trigger; import org.opensearch.commons.alerting.model.remote.monitors.RemoteDocLevelMonitorInput; import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelInput; import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelTrigger; @@ -45,36 +41,25 @@ public static List buildThreatIntelTriggerDtos(List dataSources = new ArrayList<>(); - List iocTypes = new ArrayList<>(); - triggerDtos.add(new ThreatIntelTriggerDto(dataSources, - iocTypes, - remoteMonitorTrigger.getActions(), - remoteMonitorTrigger.getName(), - remoteMonitorTrigger.getId(), - remoteMonitorTrigger.getSeverity())); } return triggerDtos; } public static ThreatIntelTrigger getThreatIntelTriggerFromBytesReference(RemoteMonitorTrigger remoteMonitorTrigger, NamedXContentRegistry namedXContentRegistry) throws IOException { - String inputBytes = BytesReference.bytes(remoteMonitorTrigger.getTrigger().toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS)).utf8ToString(); - XContentParser parser = XContentType.JSON.xContent().createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, inputBytes); - parser.nextToken(); - return ThreatIntelTrigger.parse(parser); + StreamInput triggerSin = StreamInput.wrap(remoteMonitorTrigger.getTrigger().toBytesRef().bytes); + return new ThreatIntelTrigger(triggerSin); } - public static ThreatIntelInput getThreatIntelInputFromBytesReference(RemoteDocLevelMonitorInput input, NamedXContentRegistry namedXContentRegistry) throws IOException { - String inputBytes = BytesReference.bytes(input.toXContent(XContentBuilder.builder(XContentType.JSON.xContent()), ToXContent.EMPTY_PARAMS)).utf8ToString(); - XContentParser parser = XContentType.JSON.xContent().createParser(namedXContentRegistry, LoggingDeprecationHandler.INSTANCE, inputBytes); - parser.nextToken(); - return ThreatIntelInput.parse(parser); + public static ThreatIntelInput getThreatIntelInputFromBytesReference(BytesReference bytes) throws IOException { + StreamInput sin = StreamInput.wrap(bytes.toBytesRef().bytes); + ThreatIntelInput threatIntelInput = new ThreatIntelInput(sin); + return threatIntelInput; } public static ThreatIntelMonitorDto buildThreatIntelMonitorDto(String id, Monitor monitor, NamedXContentRegistry namedXContentRegistry) throws IOException { - RemoteDocLevelMonitorInput input = (RemoteDocLevelMonitorInput) monitor.getInputs().get(0); - List indices = input.getDocLevelMonitorInput().getIndices(); - ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(input, namedXContentRegistry); + RemoteDocLevelMonitorInput remoteDocLevelMonitorInput = (RemoteDocLevelMonitorInput) monitor.getInputs().get(0); + List indices = remoteDocLevelMonitorInput.getDocLevelMonitorInput().getIndices(); + ThreatIntelInput threatIntelInput = getThreatIntelInputFromBytesReference(remoteDocLevelMonitorInput.getInput()); return new ThreatIntelMonitorDto( id, monitor.getName(), @@ -82,7 +67,6 @@ public static ThreatIntelMonitorDto buildThreatIntelMonitorDto(String id, Monito monitor.getSchedule(), monitor.getEnabled(), monitor.getUser(), - indices, buildThreatIntelTriggerDtos(monitor.getTriggers(), namedXContentRegistry) ); } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 1418465ad..c2e80a0ac 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -4,11 +4,6 @@ */ package org.opensearch.securityanalytics.resthandler; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; - import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpStatus; @@ -19,11 +14,10 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor.MonitorType; -import org.opensearch.commons.alerting.model.ScheduledJob; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.search.SearchHit; @@ -33,17 +27,34 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; +import org.opensearch.securityanalytics.model.DetectorTrigger; import java.io.IOException; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; -import org.opensearch.securityanalytics.model.DetectorTrigger; -import static org.junit.Assert.assertNotNull; -import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; +import static org.opensearch.securityanalytics.TestHelpers.productIndexCountAggRule; +import static org.opensearch.securityanalytics.TestHelpers.productIndexMapping; +import static org.opensearch.securityanalytics.TestHelpers.randomDetector; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggersAndScheduleAndEnabled; +import static org.opensearch.securityanalytics.TestHelpers.randomDoc; +import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomProductDocument; +import static org.opensearch.securityanalytics.TestHelpers.randomProductDocumentWithTime; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; public class DetectorRestApiIT extends SecurityAnalyticsRestTestCase { diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java index 59c01746c..68c8ca0a1 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -144,6 +144,7 @@ public void test_retrievesIOCs() throws IOException { (String) hit.get(STIX2IOC.DESCRIPTION_FIELD), (List) hit.get(STIX2IOC.LABELS_FIELD), (String) hit.get(STIX2IOC.FEED_ID_FIELD), + (String) hit.get(STIX2IOC.FEED_NAME_FIELD), (String) hit.get(STIX2IOC.SPEC_VERSION_FIELD), Long.parseLong(String.valueOf(hit.get(STIX2IOC.VERSION_FIELD))) // TODO implement DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD check when GetFindings API is added diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 9261fc383..78036373f 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -75,11 +75,10 @@ private ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { return new ThreatIntelMonitorDto( Monitor.NO_ID, randomAlphaOfLength(10), - List.of(new PerIocTypeScanInputDto("IP", Map.of("abc", List.of("abc")))), + List.of(new PerIocTypeScanInputDto("IP", Map.of(index, List.of("abc")))), new org.opensearch.commons.alerting.model.IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), true, - null, - List.of(index), Collections.emptyList()); + null , Collections.emptyList()); } } From e47a6aca717ff8882aa45cf3d69882072065bb38 Mon Sep 17 00:00:00 2001 From: Subhobrata Dey Date: Wed, 26 Jun 2024 00:58:55 -0700 Subject: [PATCH 4/4] add search ioc findings api (#1093) * add search ioc findings api Signed-off-by: Subhobrata Dey add search ioc findings api Signed-off-by: Subhobrata Dey add search ioc findings api Signed-off-by: Subhobrata Dey add search ioc findings api Signed-off-by: Subhobrata Dey * fix review comments for ioc findings api Signed-off-by: Subhobrata Dey --------- Signed-off-by: Subhobrata Dey --- .../SecurityAnalyticsPlugin.java | 10 + .../DetectorIndexManagementService.java | 122 +++++++++- .../model/{IoCMatch.java => IocFinding.java} | 99 ++++---- .../securityanalytics/model/IocWithFeeds.java | 111 +++++++++ .../settings/SecurityAnalyticsSettings.java | 31 +++ .../action/GetIocFindingsAction.java | 17 ++ .../action/GetIocFindingsRequest.java | 91 ++++++++ .../action/GetIocFindingsResponse.java | 62 +++++ .../iocscan/dao/IocFindingService.java | 215 ++++++++++++++++++ .../resthandler/RestGetIocFindingsAction.java | 102 +++++++++ .../TransportGetIocFindingsAction.java | 144 ++++++++++++ ..._mapping.json => ioc_finding_mapping.json} | 19 +- .../SecurityAnalyticsRestTestCase.java | 29 +++ .../securityanalytics/TestHelpers.java | 6 +- .../model/IoCMatchTests.java | 78 ------- .../model/IocFindingTests.java | 78 +++++++ .../dao/IocFindingServiceRestApiIT.java | 140 ++++++++++++ 17 files changed, 1222 insertions(+), 132 deletions(-) rename src/main/java/org/opensearch/securityanalytics/model/{IoCMatch.java => IocFinding.java} (71%) create mode 100644 src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java rename src/main/resources/mappings/{ioc_match_mapping.json => ioc_finding_mapping.json} (63%) delete mode 100644 src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java create mode 100644 src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index c8168d428..e458afce4 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -119,6 +119,7 @@ import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SARefreshTIFSourceConfigAction; @@ -134,6 +135,7 @@ import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.model.monitor.SampleRemoteDocLevelMonitorRunner; import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportRemoteDocLevelMonitorFanOutAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestRefreshTIFSourceConfigAction; @@ -183,6 +185,7 @@ import org.opensearch.securityanalytics.transport.TransportTestS3ConnectionAction; import org.opensearch.securityanalytics.transport.TransportUpdateIndexMappingsAction; import org.opensearch.securityanalytics.transport.TransportValidateRulesAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.securityanalytics.util.CorrelationRuleIndices; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; @@ -352,6 +355,7 @@ public List getRestHandlers(Settings settings, new RestSearchThreatIntelMonitorAction(), new RestRefreshTIFSourceConfigAction(), new RestListIOCsAction(), + new RestGetIocFindingsAction(), new RestTestS3ConnectionAction() ); } @@ -449,6 +453,11 @@ public List> getSettings() { SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE, SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD, SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ENABLED, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_MAX_DOCS, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_INDEX_MAX_AGE, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_ROLLOVER_PERIOD, + SecurityAnalyticsSettings.IOC_FINDING_HISTORY_RETENTION_PERIOD, SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING, SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW, SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS, @@ -501,6 +510,7 @@ public List> getSettings() { new ActionHandler<>(SARefreshTIFSourceConfigAction.INSTANCE, TransportRefreshTIFSourceConfigAction.class), new ActionHandler<>(SampleRemoteDocLevelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportRemoteDocLevelMonitorFanOutAction.class), new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class), + new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class), new ActionHandler<>(TestS3ConnectionAction.INSTANCE, TransportTestS3ConnectionAction.class) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index f6630499f..d6cae5304 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -35,6 +35,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.util.CorrelationIndices; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -54,9 +55,13 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile Boolean alertHistoryEnabled; private volatile Boolean findingHistoryEnabled; + private volatile Boolean iocFindingHistoryEnabled; + private volatile Long alertHistoryMaxDocs; private volatile Long findingHistoryMaxDocs; + private volatile Long iocFindingHistoryMaxDocs; + private volatile Long correlationHistoryMaxDocs; private volatile TimeValue alertHistoryMaxAge; @@ -64,16 +69,22 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private volatile TimeValue correlationHistoryMaxAge; + private volatile TimeValue iocFindingHistoryMaxAge; + private volatile TimeValue alertHistoryRolloverPeriod; private volatile TimeValue findingHistoryRolloverPeriod; private volatile TimeValue correlationHistoryRolloverPeriod; + private volatile TimeValue iocFindingHistoryRolloverPeriod; + private volatile TimeValue alertHistoryRetentionPeriod; private volatile TimeValue findingHistoryRetentionPeriod; private volatile TimeValue correlationHistoryRetentionPeriod; + private volatile TimeValue iocFindingHistoryRetentionPeriod; + private volatile boolean isClusterManager = false; private Scheduler.Cancellable scheduledAlertsRollover = null; @@ -81,11 +92,15 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private Scheduler.Cancellable scheduledCorrelationHistoryRollover = null; + private Scheduler.Cancellable scheduledIocFindingHistoryRollover = null; + List alertHistoryIndices = new ArrayList<>(); List findingHistoryIndices = new ArrayList<>(); HistoryIndexInfo correlationHistoryIndex = null; + HistoryIndexInfo iocFindingHistoryIndex = null; + @Inject public DetectorIndexManagementService( Settings settings, @@ -161,6 +176,27 @@ public DetectorIndexManagementService( clusterService.getClusterSettings().addSettingsUpdateConsumer(CORRELATION_HISTORY_RETENTION_PERIOD, this::setCorrelationHistoryRetentionPeriod); + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_MAX_DOCS, maxDocs -> { + setIocFindingHistoryMaxDocs(maxDocs); + if (iocFindingHistoryIndex != null) { + iocFindingHistoryIndex.maxDocs = maxDocs; + } + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_INDEX_MAX_AGE, maxAge -> { + setIocFindingHistoryMaxAge(maxAge); + if (iocFindingHistoryIndex != null) { + iocFindingHistoryIndex.maxAge = maxAge; + } + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_ROLLOVER_PERIOD, timeValue -> { + DetectorIndexManagementService.this.iocFindingHistoryRolloverPeriod = timeValue; + rescheduleIocFindingHistoryRollover(); + }); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(IOC_FINDING_HISTORY_RETENTION_PERIOD, this::setIocFindingHistoryRetentionPeriod); + initFromClusterSettings(); } @@ -204,15 +240,19 @@ private void initFromClusterSettings() { alertHistoryMaxDocs = ALERT_HISTORY_MAX_DOCS.get(settings); findingHistoryMaxDocs = FINDING_HISTORY_MAX_DOCS.get(settings); correlationHistoryMaxDocs = CORRELATION_HISTORY_MAX_DOCS.get(settings); + iocFindingHistoryMaxDocs = IOC_FINDING_HISTORY_MAX_DOCS.get(settings); alertHistoryMaxAge = ALERT_HISTORY_INDEX_MAX_AGE.get(settings); findingHistoryMaxAge = FINDING_HISTORY_INDEX_MAX_AGE.get(settings); correlationHistoryMaxAge = CORRELATION_HISTORY_INDEX_MAX_AGE.get(settings); + iocFindingHistoryMaxAge = IOC_FINDING_HISTORY_INDEX_MAX_AGE.get(settings); alertHistoryRolloverPeriod = ALERT_HISTORY_ROLLOVER_PERIOD.get(settings); findingHistoryRolloverPeriod = FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); correlationHistoryRolloverPeriod = CORRELATION_HISTORY_ROLLOVER_PERIOD.get(settings); + iocFindingHistoryRolloverPeriod = IOC_FINDING_HISTORY_ROLLOVER_PERIOD.get(settings); alertHistoryRetentionPeriod = ALERT_HISTORY_RETENTION_PERIOD.get(settings); findingHistoryRetentionPeriod = FINDING_HISTORY_RETENTION_PERIOD.get(settings); correlationHistoryRetentionPeriod = CORRELATION_HISTORY_RETENTION_PERIOD.get(settings); + iocFindingHistoryRetentionPeriod = IOC_FINDING_HISTORY_RETENTION_PERIOD.get(settings); } @Override @@ -238,6 +278,9 @@ public void clusterChanged(ClusterChangedEvent event) { if (correlationHistoryIndex != null && correlationHistoryIndex.indexAlias != null) { correlationHistoryIndex.isInitialized = event.state().metadata().hasAlias(correlationHistoryIndex.indexAlias); } + if (iocFindingHistoryIndex != null && iocFindingHistoryIndex.indexAlias != null) { + iocFindingHistoryIndex.isInitialized = event.state().metadata().hasAlias(iocFindingHistoryIndex.indexAlias); + } } private void onMaster() { @@ -247,6 +290,7 @@ private void onMaster() { rolloverAndDeleteAlertHistoryIndices(); rolloverAndDeleteFindingHistoryIndices(); rolloverAndDeleteCorrelationHistoryIndices(); + rolloverAndDeleteIocFindingHistoryIndices(); }, TimeValue.timeValueSeconds(1), executorName()); // schedule the next rollover for approx MAX_AGE later scheduledAlertsRollover = threadPool @@ -255,11 +299,13 @@ private void onMaster() { .scheduleWithFixedDelay(() -> rolloverAndDeleteFindingHistoryIndices(), findingHistoryRolloverPeriod, executorName()); scheduledCorrelationHistoryRollover = threadPool .scheduleWithFixedDelay(() -> rolloverAndDeleteCorrelationHistoryIndices(), correlationHistoryRolloverPeriod, executorName()); + scheduledIocFindingHistoryRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName()); } catch (Exception e) { // This should be run on cluster startup logger.error( - "Error creating alert/finding/correlation indices. " + - "Alerts/Findings/Correlations can't be recorded until master node is restarted.", + "Error creating alert/finding/correlation/ioc finding indices. " + + "Alerts/Findings/Correlations/IOC Finding can't be recorded until master node is restarted.", e ); } @@ -275,6 +321,9 @@ private void offMaster() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } private String executorName() { @@ -327,6 +376,10 @@ private List getIndicesToDelete(ClusterStateResponse clusterStateRespons if (indexToDelete != null) { indicesToDelete.add(indexToDelete); } + indexToDelete = getHistoryIndexToDelete(indexMetaData, iocFindingHistoryRetentionPeriod.millis(), iocFindingHistoryIndex != null? List.of(iocFindingHistoryIndex): List.of(), true); + if (indexToDelete != null) { + indicesToDelete.add(indexToDelete); + } } return indicesToDelete; } @@ -371,7 +424,7 @@ private void deleteAllOldHistoryIndices(List indicesToDelete) { public void onResponse(AcknowledgedResponse deleteIndicesResponse) { if (!deleteIndicesResponse.isAcknowledged()) { logger.error( - "Could not delete one or more Alerting/Finding/Correlation history indices: [" + indicesToDelete + "]. Retrying one by one." + "Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: [" + indicesToDelete + "]. Retrying one by one." ); deleteOldHistoryIndex(indicesToDelete); } else { @@ -381,7 +434,7 @@ public void onResponse(AcknowledgedResponse deleteIndicesResponse) { @Override public void onFailure(Exception e) { - logger.error("Delete for Alerting/Finding/Correlation History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); + logger.error("Delete for Alerting/Finding/Correlation/IOC Finding History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); deleteOldHistoryIndex(indicesToDelete); } } @@ -399,7 +452,7 @@ private void deleteOldHistoryIndex(List indicesToDelete) { @Override public void onResponse(AcknowledgedResponse acknowledgedResponse) { if (!acknowledgedResponse.isAcknowledged()) { - logger.error("Could not delete one or more Alerting/Finding/Correlation history indices: " + index); + logger.error("Could not delete one or more Alerting/Finding/Correlation/IOC Finding history indices: " + index); } } @@ -455,6 +508,23 @@ private void rolloverAndDeleteCorrelationHistoryIndices() { } } + private void rolloverAndDeleteIocFindingHistoryIndices() { + try { + iocFindingHistoryIndex = new HistoryIndexInfo( + IocFindingService.IOC_FINDING_ALIAS_NAME, + IocFindingService.IOC_FINDING_INDEX_PATTERN, + IocFindingService.getIndexMapping(), + iocFindingHistoryMaxDocs, + iocFindingHistoryMaxAge, + clusterService.state().metadata().hasAlias(IocFindingService.IOC_FINDING_ALIAS_NAME) + ); + rolloverIocFindingHistoryIndices(); + deleteOldIndices("IOC Findings", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP); + } catch (Exception ex) { + logger.error("failed to construct ioc finding index info"); + } + } + private List getAllAlertsIndicesPatternForAllTypes(List logTypes) { return logTypes .stream() @@ -544,6 +614,20 @@ private void rolloverCorrelationHistoryIndices() { } } + private void rolloverIocFindingHistoryIndices() { + if (iocFindingHistoryIndex != null) { + rolloverIndex( + iocFindingHistoryIndex.isInitialized, + iocFindingHistoryIndex.indexAlias, + iocFindingHistoryIndex.indexPattern, + iocFindingHistoryIndex.indexMappings, + iocFindingHistoryIndex.maxDocs, + iocFindingHistoryIndex.maxAge, + true + ); + } + } + private void rescheduleAlertRollover() { if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) { if (scheduledAlertsRollover != null) { @@ -574,6 +658,16 @@ private void rescheduleCorrelationHistoryRollover() { } } + private void rescheduleIocFindingHistoryRollover() { + if (clusterService.state().getNodes().isLocalNodeElectedClusterManager()) { + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } + scheduledIocFindingHistoryRollover = threadPool + .scheduleWithFixedDelay(() -> rolloverAndDeleteIocFindingHistoryIndices(), iocFindingHistoryRolloverPeriod, executorName()); + } + } + private String alertMapping() { String alertMapping = null; try ( @@ -620,6 +714,10 @@ public void setCorrelationHistoryMaxDocs(Long correlationHistoryMaxDocs) { this.correlationHistoryMaxDocs = correlationHistoryMaxDocs; } + public void setIocFindingHistoryMaxDocs(Long iocFindingHistoryMaxDocs) { + this.iocFindingHistoryMaxDocs = iocFindingHistoryMaxDocs; + } + public void setAlertHistoryMaxAge(TimeValue alertHistoryMaxAge) { this.alertHistoryMaxAge = alertHistoryMaxAge; } @@ -632,6 +730,10 @@ public void setCorrelationHistoryMaxAge(TimeValue correlationHistoryMaxAge) { this.correlationHistoryMaxAge = correlationHistoryMaxAge; } + public void setIocFindingHistoryMaxAge(TimeValue iocFindingHistoryMaxAge) { + this.iocFindingHistoryMaxAge = iocFindingHistoryMaxAge; + } + public void setAlertHistoryRolloverPeriod(TimeValue alertHistoryRolloverPeriod) { this.alertHistoryRolloverPeriod = alertHistoryRolloverPeriod; } @@ -656,6 +758,10 @@ public void setCorrelationHistoryRetentionPeriod(TimeValue correlationHistoryRet this.correlationHistoryRetentionPeriod = correlationHistoryRetentionPeriod; } + public void setIocFindingHistoryRetentionPeriod(TimeValue iocFindingHistoryRetentionPeriod) { + this.iocFindingHistoryRetentionPeriod = iocFindingHistoryRetentionPeriod; + } + public void setClusterManager(boolean clusterManager) { isClusterManager = clusterManager; } @@ -676,6 +782,9 @@ protected void doStop() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } @Override @@ -689,6 +798,9 @@ protected void doClose() { if (scheduledCorrelationHistoryRollover != null) { scheduledCorrelationHistoryRollover.cancel(); } + if (scheduledIocFindingHistoryRollover != null) { + scheduledIocFindingHistoryRollover.cancel(); + } } private static class HistoryIndexInfo { diff --git a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java b/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java similarity index 71% rename from src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java rename to src/main/java/org/opensearch/securityanalytics/model/IocFinding.java index 04f54699f..6c34b2cb3 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java +++ b/src/main/java/org/opensearch/securityanalytics/model/IocFinding.java @@ -13,6 +13,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; @@ -20,13 +21,13 @@ * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job * It's the inverse of an IoC finding which maps a document to list of IoC's */ -public class IoCMatch implements Writeable, ToXContent { +public class IocFinding implements Writeable, ToXContent { //TODO implement IoC_Match interface from security-analytics-commons public static final String ID_FIELD = "id"; public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; - public static final String FEED_IDS_FIELD = "feed_ids"; - public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id"; - public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name"; + public static final String IOC_WITH_FEED_IDS_FIELD = "ioc_feed_ids"; + public static final String MONITOR_ID_FIELD = "monitor_id"; + public static final String MONITOR_NAME_FIELD = "monitor_name"; public static final String IOC_VALUE_FIELD = "ioc_value"; public static final String IOC_TYPE_FIELD = "ioc_type"; public static final String TIMESTAMP_FIELD = "timestamp"; @@ -34,34 +35,34 @@ public class IoCMatch implements Writeable, ToXContent { private final String id; private final List relatedDocIds; - private final List feedIds; - private final String iocScanJobId; - private final String iocScanJobName; + private final List iocWithFeeds; + private final String monitorId; + private final String monitorName; private final String iocValue; private final String iocType; private final Instant timestamp; private final String executionId; - public IoCMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, - String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { - validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); + public IocFinding(String id, List relatedDocIds, List iocWithFeeds, String monitorId, + String monitorName, String iocValue, String iocType, Instant timestamp, String executionId) { + validateIoCMatch(id, monitorId, monitorName, iocValue, timestamp, executionId, relatedDocIds); this.id = id; this.relatedDocIds = relatedDocIds; - this.feedIds = feedIds; - this.iocScanJobId = iocScanJobId; - this.iocScanJobName = iocScanJobName; + this.iocWithFeeds = iocWithFeeds; + this.monitorId = monitorId; + this.monitorName = monitorName; this.iocValue = iocValue; this.iocType = iocType; this.timestamp = timestamp; this.executionId = executionId; } - public IoCMatch(StreamInput in) throws IOException { + public IocFinding(StreamInput in) throws IOException { id = in.readString(); relatedDocIds = in.readStringList(); - feedIds = in.readStringList(); - iocScanJobId = in.readString(); - iocScanJobName = in.readString(); + iocWithFeeds = in.readList(IocWithFeeds::readFrom); + monitorId = in.readString(); + monitorName = in.readString(); iocValue = in.readString(); iocType = in.readString(); timestamp = in.readInstant(); @@ -72,23 +73,37 @@ public IoCMatch(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeStringCollection(relatedDocIds); - out.writeStringCollection(feedIds); - out.writeString(iocScanJobId); - out.writeString(iocScanJobName); + out.writeCollection(iocWithFeeds); + out.writeString(monitorId); + out.writeString(monitorName); out.writeString(iocValue); out.writeString(iocType); out.writeInstant(timestamp); out.writeOptionalString(executionId); } + public Map asTemplateArg() { + return Map.of( + ID_FIELD,id, + RELATED_DOC_IDS_FIELD, relatedDocIds, + IOC_WITH_FEED_IDS_FIELD, iocWithFeeds, + MONITOR_ID_FIELD, monitorId, + MONITOR_NAME_FIELD, monitorName, + IOC_VALUE_FIELD, iocValue, + IOC_TYPE_FIELD, iocType, + TIMESTAMP_FIELD, timestamp, + EXECUTION_ID_FIELD, executionId + ); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject() .field(ID_FIELD, id) .field(RELATED_DOC_IDS_FIELD, relatedDocIds) - .field(FEED_IDS_FIELD, feedIds) - .field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId) - .field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName) + .field(IOC_WITH_FEED_IDS_FIELD, iocWithFeeds) + .field(MONITOR_ID_FIELD, monitorId) + .field(MONITOR_NAME_FIELD, monitorName) .field(IOC_VALUE_FIELD, iocValue) .field(IOC_TYPE_FIELD, iocType) .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) @@ -105,16 +120,16 @@ public List getRelatedDocIds() { return relatedDocIds; } - public List getFeedIds() { - return feedIds; + public List getFeedIds() { + return iocWithFeeds; } - public String getIocScanJobId() { - return iocScanJobId; + public String getMonitorId() { + return monitorId; } - public String getIocScanJobName() { - return iocScanJobName; + public String getMonitorName() { + return monitorName; } public String getIocValue() { @@ -133,12 +148,12 @@ public String getExecutionId() { return executionId; } - public static IoCMatch parse(XContentParser xcp) throws IOException { + public static IocFinding parse(XContentParser xcp) throws IOException { String id = null; List relatedDocIds = new ArrayList<>(); - List feedIds = new ArrayList<>(); - String iocScanJobId = null; - String iocScanName = null; + List feedIds = new ArrayList<>(); + String monitorId = null; + String monitorName = null; String iocValue = null; String iocType = null; Instant timestamp = null; @@ -159,17 +174,17 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { relatedDocIds.add(xcp.text()); } break; - case FEED_IDS_FIELD: + case IOC_WITH_FEED_IDS_FIELD: ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - feedIds.add(xcp.text()); + feedIds.add(IocWithFeeds.parse(xcp)); } break; - case IOC_SCAN_JOB_ID_FIELD: - iocScanJobId = xcp.textOrNull(); + case MONITOR_ID_FIELD: + monitorId = xcp.textOrNull(); break; - case IOC_SCAN_JOB_NAME_FIELD: - iocScanName = xcp.textOrNull(); + case MONITOR_NAME_FIELD: + monitorName = xcp.textOrNull(); break; case IOC_VALUE_FIELD: iocValue = xcp.textOrNull(); @@ -197,11 +212,11 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { } } - return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + return new IocFinding(id, relatedDocIds, feedIds, monitorId, monitorName, iocValue, iocType, timestamp, executionId); } - public static IoCMatch readFrom(StreamInput in) throws IOException { - return new IoCMatch(in); + public static IocFinding readFrom(StreamInput in) throws IOException { + return new IocFinding(in); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java b/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java new file mode 100644 index 000000000..d858619fc --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocWithFeeds.java @@ -0,0 +1,111 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * container class to store a tuple of feed id, ioc id and index. + */ +public class IocWithFeeds implements Writeable, ToXContent { + + private static final String FEED_ID_FIELD = "feed_id"; + + private static final String IOC_ID_FIELD = "ioc_id"; + + private static final String INDEX_FIELD = "index"; + + private final String feedId; + + private final String iocId; + + private final String index; + + public IocWithFeeds(String iocId, String feedId, String index) { + this.iocId = iocId; + this.feedId = feedId; + this.index = index; + } + + public IocWithFeeds(StreamInput sin) throws IOException { + this.iocId = sin.readString(); + this.feedId = sin.readString(); + this.index = sin.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(iocId); + out.writeString(feedId); + out.writeString(index); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(IOC_ID_FIELD, iocId) + .field(FEED_ID_FIELD, feedId) + .field(INDEX_FIELD, index) + .endObject(); + return builder; + } + + public Map asTemplateArg() { + return Map.of( + FEED_ID_FIELD, feedId, + IOC_ID_FIELD, iocId, + INDEX_FIELD, index + ); + } + + public String getIocId() { + return iocId; + } + + public String getFeedId() { + return feedId; + } + + public String getIndex() { + return index; + } + + public static IocWithFeeds parse(XContentParser xcp) throws IOException { + String iocId = null; + String feedId = null; + String index = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case IOC_ID_FIELD: + iocId = xcp.text(); + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + case INDEX_FIELD: + index = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + return new IocWithFeeds(iocId, feedId, index); + } + + public static IocWithFeeds readFrom(StreamInput sin) throws IOException { + return new IocWithFeeds(sin); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index 59bdfdf18..83bc8e567 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -31,6 +31,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_ENABLED = Setting.boolSetting( + "plugins.security_analytics.ioc_finding_enabled", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_rollover_period", TimeValue.timeValueHours(12), @@ -49,6 +55,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_ROLLOVER_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_rollover_period", + TimeValue.timeValueHours(12), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_max_age", new TimeValue(30, TimeUnit.DAYS), @@ -67,6 +79,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_INDEX_MAX_AGE = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_max_age", + new TimeValue(30, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_MAX_DOCS = Setting.longSetting( "plugins.security_analytics.alert_history_max_docs", 1000L, @@ -88,6 +106,13 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_MAX_DOCS = Setting.longSetting( + "plugins.security_analytics.ioc_finding_history_max_docs", + 1000L, + 0L, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting ALERT_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( "plugins.security_analytics.alert_history_retention_period", new TimeValue(60, TimeUnit.DAYS), @@ -106,6 +131,12 @@ public class SecurityAnalyticsSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ); + public static final Setting IOC_FINDING_HISTORY_RETENTION_PERIOD = Setting.positiveTimeSetting( + "plugins.security_analytics.ioc_finding_history_retention_period", + new TimeValue(60, TimeUnit.DAYS), + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + public static final Setting REQUEST_TIMEOUT = Setting.positiveTimeSetting( "plugins.security_analytics.request_timeout", TimeValue.timeValueSeconds(10), diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java new file mode 100644 index 000000000..f662ed1be --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsAction.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionType; + +public class GetIocFindingsAction extends ActionType { + + public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction(); + public static final String NAME = "cluster:admin/opensearch/securityanalytics/iocs/findings/get"; + + public GetIocFindingsAction() { + super(NAME, GetIocFindingsResponse::new); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java new file mode 100644 index 000000000..1395cff1e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsRequest.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +public class GetIocFindingsRequest extends ActionRequest { + + private List findingIds; + + private List iocIds; + + private Instant startTime; + + private Instant endTime; + + private Table table; + + public GetIocFindingsRequest(StreamInput sin) throws IOException { + this( + sin.readOptionalStringList(), + sin.readOptionalStringList(), + sin.readOptionalInstant(), + sin.readOptionalInstant(), + Table.readFrom(sin) + ); + } + + public GetIocFindingsRequest(List findingIds, + List iocIds, + Instant startTime, + Instant endTime, + Table table) { + this.findingIds = findingIds; + this.iocIds = iocIds; + this.startTime = startTime; + this.endTime = endTime; + this.table = table; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (startTime != null && endTime != null && startTime.isAfter(endTime)) { + validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(), + "startTime should be less than endTime"), validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalStringCollection(findingIds); + out.writeOptionalStringCollection(iocIds); + out.writeOptionalInstant(startTime); + out.writeOptionalInstant(endTime); + table.writeTo(out); + } + + public List getFindingIds() { + return findingIds; + } + + public List getIocIds() { + return iocIds; + } + + public Instant getStartTime() { + return startTime; + } + + public Instant getEndTime() { + return endTime; + } + + public Table getTable() { + return table; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java new file mode 100644 index 000000000..4c0dea477 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/GetIocFindingsResponse.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.action; + +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.securityanalytics.model.IocFinding; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class GetIocFindingsResponse extends ActionResponse implements ToXContentObject { + + private static final String TOTAL_IOC_FINDINGS_FIELD = "total_findings"; + + private static final String IOC_FINDINGS_FIELD = "ioc_findings"; + + private Integer totalFindings; + + private List iocFindings; + + public GetIocFindingsResponse(Integer totalFindings, List iocFindings) { + super(); + this.totalFindings = totalFindings; + this.iocFindings = iocFindings; + } + + public GetIocFindingsResponse(StreamInput sin) throws IOException { + this( + sin.readInt(), + Collections.unmodifiableList(sin.readList(IocFinding::new)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalFindings); + out.writeCollection(iocFindings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(TOTAL_IOC_FINDINGS_FIELD, totalFindings) + .field(IOC_FINDINGS_FIELD, iocFindings); + return builder.endObject(); + } + + public Integer getTotalFindings() { + return totalFindings; + } + + public List getIocFindings() { + return iocFindings; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java new file mode 100644 index 000000000..0e1b955b1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingService.java @@ -0,0 +1,215 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.search.SearchHit; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Data layer to perform CRUD operations for threat intel ioc match : store in system index. + */ +public class IocFindingService { + //TODO manage index rollover + public static final String IOC_FINDING_ALIAS_NAME = ".opensearch-sap-ioc-findings"; + + public static final String IOC_FINDING_INDEX_PATTERN = "<.opensearch-sap-ioc-findings-history-{now/d}-1>"; + + public static final String IOC_FINDING_INDEX_PATTERN_REGEXP = ".opensearch-sap-ioc-findings*"; + + private static final Logger log = LogManager.getLogger(IocFindingService.class); + private final Client client; + private final ClusterService clusterService; + + private final NamedXContentRegistry xContentRegistry; + + public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) { + this.client = client; + this.clusterService = clusterService; + this.xContentRegistry = xContentRegistry; + } + + public void indexIocMatches(List iocFindings, + final ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(IOC_FINDING_ALIAS_NAME); + for (int i = 0; i < iocFindings.size(); i++) { + IocFinding iocFinding = iocFindings.get(i); + try { + IndexRequest indexRequest = new IndexRequest(IOC_FINDING_ALIAS_NAME) + .source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for ioc match %s moving on to next"), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} Ioc Matches. Failure: {}", request.batchSize(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error("Failed to save ioc matches.", e); + } + } + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + public static String getIndexMapping() { + try { + try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_finding_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel ioc match index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc match index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + /** + * Index name: .opensearch-sap-iocmatch + * Mapping: /mappings/ioc_finding_mapping.json + * + * @param listener setup listener + */ + public void createIndexIfNotExists(final ActionListener listener) { + // check if job index exists + try { + if (clusterService.state().metadata().hasAlias(IOC_FINDING_ALIAS_NAME) == true) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(IOC_FINDING_INDEX_PATTERN).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING).alias(new Alias(IOC_FINDING_ALIAS_NAME)); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("Ioc match index created"); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.debug("index {} already exist", IOC_FINDING_INDEX_PATTERN); + listener.onResponse(null); + return; + } + log.error("Failed to create security analytics threat intel job index", e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failure in creating ioc_match index", e); + listener.onFailure(e); + } + } + + public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final ActionListener actionListener) { + createIndexIfNotExists(ActionListener.wrap( + r -> { + SearchRequest searchRequest = new SearchRequest() + .source(searchSourceBuilder) + .indices(IOC_FINDING_ALIAS_NAME); + + client.search(searchRequest, new ActionListener<>() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value; + List iocFindings = new ArrayList<>(); + + for (SearchHit hit: searchResponse.getHits()) { + XContentParser xcp = XContentType.JSON.xContent() + .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); + IocFinding iocFinding = IocFinding.parse(xcp); + iocFindings.add(iocFinding); + } + actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings)); + } catch (Exception ex) { + this.onFailure(ex); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + actionListener.onResponse(new GetIocFindingsResponse(0, List.of())); + return; + } + actionListener.onFailure(e); + } + }); + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java new file mode 100644 index 000000000..36927d35d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestGetIocFindingsAction.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.resthandler; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.action.GetFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; + +import java.io.IOException; +import java.time.DateTimeException; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.opensearch.rest.RestRequest.Method.GET; + +public class RestGetIocFindingsAction extends BaseRestHandler { + + @Override + public String getName() { + return "get_ioc_findings_action_sa"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + String sortString = request.param("sortString", "timestamp"); + String sortOrder = request.param("sortOrder", "asc"); + String missing = request.param("missing"); + int size = request.paramAsInt("size", 20); + int startIndex = request.paramAsInt("startIndex", 0); + String searchString = request.param("searchString", ""); + + List findingIds = null; + if (request.param("findingIds") != null) { + findingIds = Arrays.asList(request.param("findingIds").split(",")); + } + List iocIds = null; + if (request.param("iocIds") != null) { + iocIds = Arrays.asList(request.param("iocIds").split(",")); + } + Instant startTime = null; + String startTimeParam = request.param("startTime"); + if (startTimeParam != null && !startTimeParam.isEmpty()) { + try { + startTime = Instant.ofEpochMilli(Long.parseLong(startTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + startTime = Instant.now(); // Default value or fallback + } + } + + Instant endTime = null; + String endTimeParam = request.param("endTime"); + if (endTimeParam != null && !endTimeParam.isEmpty()) { + try { + endTime = Instant.ofEpochMilli(Long.parseLong(endTimeParam)); + } catch (NumberFormatException | NullPointerException | DateTimeException e) { + // Handle the parsing error + // For example, log the error or provide a default value + endTime = Instant.now(); // Default value or fallback + } + } + + Table table = new Table( + sortOrder, + sortString, + missing, + size, + startIndex, + searchString + ); + + GetIocFindingsRequest getIocFindingsRequest = new GetIocFindingsRequest( + findingIds, + iocIds, + startTime, + endTime, + table + ); + return channel -> client.execute( + GetIocFindingsAction.INSTANCE, + getIocFindingsRequest, + new RestToXContentListener<>(channel) + ); + } + + @Override + public List routes() { + return singletonList(new Route(GET, SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings" + "/_search")); + } +} + diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java new file mode 100644 index 000000000..2c4792650 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -0,0 +1,144 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.transport; + +import org.apache.lucene.search.join.ScoreMode; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.sort.FieldSortBuilder; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; +import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.time.Instant; +import java.util.List; + +public class TransportGetIocFindingsAction extends HandledTransportAction implements SecureTransportAction { + + private final IocFindingService iocFindingService; + + private final ClusterService clusterService; + + private final Settings settings; + + private final ThreadPool threadPool; + + private volatile Boolean filterByEnabled; + + @Inject + public TransportGetIocFindingsAction( + TransportService transportService, + ActionFilters actionFilters, + ClusterService clusterService, + Settings settings, + NamedXContentRegistry xContentRegistry, + Client client + ) { + super(GetIocFindingsAction.NAME, transportService, actionFilters, GetIocFindingsRequest::new); + this.settings = settings; + this.clusterService = clusterService; + this.threadPool = client.threadPool(); + this.iocFindingService = new IocFindingService(client, this.clusterService, xContentRegistry); + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); + this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); + } + + @Override + protected void doExecute(Task task, GetIocFindingsRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + Table tableProp = request.getTable(); + FieldSortBuilder sortBuilder = SortBuilders + .fieldSort(tableProp.getSortString()) + .order(SortOrder.fromString(tableProp.getSortOrder())); + if (tableProp.getMissing() != null && !tableProp.getMissing().isBlank()) { + sortBuilder.missing(tableProp.getMissing()); + } + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .sort(sortBuilder) + .size(tableProp.getSize()) + .from(tableProp.getStartIndex()) + .fetchSource(new FetchSourceContext(true, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)) + .seqNoAndPrimaryTerm(true) + .version(true); + + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + List findingIds = request.getFindingIds(); + + if (findingIds != null && !findingIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("id", findingIds)); + } + + List iocIds = request.getIocIds(); + if (iocIds != null && !iocIds.isEmpty()) { + queryBuilder.filter(QueryBuilders.termsQuery("ioc_feed_ids.ioc_id", iocIds)); + } + + Instant startTime = request.getStartTime(); + Instant endTime = request.getEndTime(); + if (startTime != null && endTime != null) { + long startTimeMillis = startTime.toEpochMilli(); + long endTimeMillis = endTime.toEpochMilli(); + QueryBuilder timeRangeQuery = QueryBuilders.rangeQuery("timestamp") + .from(startTimeMillis) // Greater than or equal to start time + .to(endTimeMillis); // Less than or equal to end time + queryBuilder.filter(timeRangeQuery); + } + + if (tableProp.getSearchString() != null && !tableProp.getSearchString().isBlank()) { + queryBuilder.should(QueryBuilders + .queryStringQuery(tableProp.getSearchString()) + ).should( + QueryBuilders.nestedQuery( + "queries", + QueryBuilders.boolQuery() + .must( + QueryBuilders.queryStringQuery(tableProp.getSearchString()) + ), + ScoreMode.Avg + ) + ); + } + searchSourceBuilder.query(queryBuilder).trackTotalHits(true); + + this.threadPool.getThreadContext().stashContext(); + iocFindingService.searchIocMatches(searchSourceBuilder, actionListener); + } + + private void setFilterByEnabled(boolean filterByEnabled) { + this.filterByEnabled = filterByEnabled; + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/ioc_match_mapping.json b/src/main/resources/mappings/ioc_finding_mapping.json similarity index 63% rename from src/main/resources/mappings/ioc_match_mapping.json rename to src/main/resources/mappings/ioc_finding_mapping.json index f4573190e..2353bf14e 100644 --- a/src/main/resources/mappings/ioc_match_mapping.json +++ b/src/main/resources/mappings/ioc_finding_mapping.json @@ -7,16 +7,27 @@ "schema_version": { "type": "integer" }, - "feed_ids" : { - "type": "keyword" + "ioc_feed_ids" : { + "type": "object", + "properties": { + "feed_id": { + "type": "keyword" + }, + "ioc_id": { + "type": "keyword" + }, + "index": { + "type": "keyword" + } + } }, "related_doc_ids": { "type": "keyword" }, - "ioc_scan_job_id": { + "monitor_id": { "type": "keyword" }, - "ioc_scan_job_name": { + "monitor_name": { "type": "keyword" }, "id": { diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 68b485129..52f1d4b5d 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -61,6 +61,8 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.util.CorrelationIndices; @@ -671,6 +673,10 @@ protected HttpEntity toHttpEntity(ThreatIntelMonitorDto threatIntelMonitorDto) t return new StringEntity(toJsonString(threatIntelMonitorDto), ContentType.APPLICATION_JSON); } + protected HttpEntity toHttpEntity(IocFinding iocFinding) throws IOException { + return new StringEntity(toJsonString(iocFinding), ContentType.APPLICATION_JSON); + } + protected HttpEntity toHttpEntity(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { return new StringEntity(toJsonString(testS3ConnectionRequest), ContentType.APPLICATION_JSON); } @@ -728,6 +734,11 @@ private String toJsonString(ThreatIntelMonitorDto threatIntelMonitorDto) throws return IndexUtilsKt.string(shuffleXContent(threatIntelMonitorDto.toXContent(builder, ToXContent.EMPTY_PARAMS))); } + private String toJsonString(IocFinding iocFinding) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + return IndexUtilsKt.string(shuffleXContent(iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + private String toJsonString(TestS3ConnectionRequest testS3ConnectionRequest) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); return IndexUtilsKt.string(shuffleXContent(testS3ConnectionRequest.toXContent(builder, ToXContent.EMPTY_PARAMS))); @@ -1489,6 +1500,24 @@ public List getAlertIndices(String detectorType) throws IOException { return indices; } + public List getIocFindingIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + public List getQueryIndices(String detectorType) throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getRuleIndex(detectorType) + "*?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 132ad4123..3dccd142c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -28,7 +28,7 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.IoCMatch; +import org.opensearch.securityanalytics.model.IocFinding; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; @@ -809,9 +809,9 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } - public static String toJsonString(IoCMatch iocMatch) throws IOException { + public static String toJsonString(IocFinding iocFinding) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); - builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder = iocFinding.toXContent(builder, ToXContent.EMPTY_PARAMS); return BytesReference.bytes(builder).utf8ToString(); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java deleted file mode 100644 index 4b56c7eb5..000000000 --- a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.opensearch.securityanalytics.model; - -import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.time.Instant; -import java.util.List; - -import static org.opensearch.securityanalytics.TestHelpers.toJsonString; - -public class IoCMatchTests extends OpenSearchTestCase { - - public void testIoCMatchAsAStream() throws IOException { - IoCMatch iocMatch = getRandomIoCMatch(); - String jsonString = toJsonString(iocMatch); - BytesStreamOutput out = new BytesStreamOutput(); - iocMatch.writeTo(out); - StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); - assertEquals(iocMatch.getId(), newIocMatch.getId()); - assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); - assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); - assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); - assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); - assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); - assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); - assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); - assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); - } - - public void testIoCMatchParse() throws IOException { - String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + - "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + - " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + - "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; - IoCMatch iocMatch = IoCMatch.parse((getParser(iocMatchString))); - BytesStreamOutput out = new BytesStreamOutput(); - iocMatch.writeTo(out); - StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); - assertEquals(iocMatch.getId(), newIocMatch.getId()); - assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); - assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); - assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); - assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); - assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); - assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); - assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); - assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); - } - - public XContentParser getParser(String xc) throws IOException { - XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); - parser.nextToken(); - return parser; - - } - - private static IoCMatch getRandomIoCMatch() { - return new IoCMatch( - randomAlphaOfLength(10), - List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), - List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - randomAlphaOfLength(10), - Instant.now(), - randomAlphaOfLength(10)); - } - - -} diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java new file mode 100644 index 000000000..8acf10744 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocFindingTests.java @@ -0,0 +1,78 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IocFindingTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IocFinding iocFinding = getRandomIoCMatch(); + String jsonString = toJsonString(iocFinding); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getMonitorId(), newIocFinding.getMonitorId()); + assertEquals(iocFinding.getMonitorName(), newIocFinding.getMonitorName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public void testIoCMatchParse() throws IOException { + String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + + "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; + IocFinding iocFinding = IocFinding.parse((getParser(iocMatchString))); + BytesStreamOutput out = new BytesStreamOutput(); + iocFinding.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocFinding newIocFinding = new IocFinding(sin); + assertEquals(iocFinding.getId(), newIocFinding.getId()); + assertEquals(iocFinding.getMonitorId(), newIocFinding.getMonitorId()); + assertEquals(iocFinding.getMonitorName(), newIocFinding.getMonitorName()); + assertEquals(iocFinding.getIocValue(), newIocFinding.getIocValue()); + assertEquals(iocFinding.getIocType(), newIocFinding.getIocType()); + assertEquals(iocFinding.getTimestamp(), newIocFinding.getTimestamp()); + assertEquals(iocFinding.getExecutionId(), newIocFinding.getExecutionId()); + assertTrue(iocFinding.getFeedIds().containsAll(newIocFinding.getFeedIds())); + assertTrue(iocFinding.getRelatedDocIds().containsAll(newIocFinding.getRelatedDocIds())); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static IocFinding getRandomIoCMatch() { + return new IocFinding( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } + + +} diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java new file mode 100644 index 000000000..5c66d50bb --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocFindingServiceRestApiIT.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.model.IocFinding; +import org.opensearch.securityanalytics.model.IocWithFeeds; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.*; + +public class IocFindingServiceRestApiIT extends SecurityAnalyticsRestTestCase { + + @SuppressWarnings("unchecked") + public void testGetIocFindings() throws IOException { + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(5, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + @SuppressWarnings("unchecked") + public void testGetIocFindingsWithIocIdFilter() throws IOException { + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(10); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + String iocId = iocFindings.stream().map(iocFinding -> iocFinding.getFeedIds().get(0).getIocId()).findFirst().get(); + + Response response = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?iocIds=" + iocId, + Map.of(), null); + Map responseAsMap = responseAsMap(response); + Assert.assertEquals(1, ((List>) responseAsMap.get("ioc_findings")).size()); + } + + public void testGetIocFindingsRolloverByMaxDocs() throws IOException, InterruptedException { + updateClusterSetting(IOC_FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(IOC_FINDING_HISTORY_MAX_DOCS.getKey(), "1"); + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(5); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + AtomicBoolean found = new AtomicBoolean(false); + OpenSearchTestCase.waitUntil(() -> { + try { + found.set(getIocFindingIndices().size() == 2); + return found.get(); + } catch (IOException e) { + return false; + } + }, 30000, TimeUnit.SECONDS); + Assert.assertTrue(found.get()); + } + + public void testGetIocFindingsRolloverByMaxAge() throws IOException, InterruptedException { + updateClusterSetting(IOC_FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(IOC_FINDING_HISTORY_MAX_DOCS.getKey(), "1000"); + updateClusterSetting(IOC_FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); + makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search?startIndex=1&size=5", + Map.of(), null); + List iocFindings = generateIocMatches(5); + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + + AtomicBoolean found = new AtomicBoolean(false); + OpenSearchTestCase.waitUntil(() -> { + try { + found.set(getIocFindingIndices().size() == 2); + return found.get(); + } catch (IOException e) { + return false; + } + }, 30000, TimeUnit.SECONDS); + Assert.assertTrue(found.get()); + + updateClusterSetting(IOC_FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "1000s"); + updateClusterSetting(IOC_FINDING_HISTORY_RETENTION_PERIOD.getKey(), "1s"); + + AtomicBoolean retFound = new AtomicBoolean(false); + OpenSearchTestCase.waitUntil(() -> { + try { + retFound.set(getIocFindingIndices().size() == 1); + return retFound.get(); + } catch (IOException e) { + return false; + } + }, 30000, TimeUnit.SECONDS); + Assert.assertTrue(retFound.get()); + } + + private List generateIocMatches(int i) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> new IocWithFeeds(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10))), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } +} \ No newline at end of file