diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index b27d2f4c9..bb55bd770 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -57,7 +57,7 @@ public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID"; public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER; - public static final String IOC_ALL_INDEX_PATTERN_BY_ALIAS = IOC_INDEX_NAME_TEMPLATE + "-*"; + public static final String IOC_ALL_INDEX_PATTERN_BY_ID = IOC_INDEX_NAME_TEMPLATE + "-*"; public static final String IOC_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE; public static final String IOC_TIME_PLACEHOLDER = "TIME"; public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER; @@ -121,6 +121,7 @@ public void storeIOCs(Map actionToIOCs) { public void indexIocs(List iocs) throws IOException { String iocAlias = getIocIndexAlias(saTifSourceConfig.getId()); String iocPattern = getIocIndexRolloverPattern(saTifSourceConfig.getId()); + String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId()); if (iocIndexExists(iocAlias) == false) { initFeedIndex(iocAlias, iocPattern, ActionListener.wrap( @@ -132,7 +133,7 @@ public void indexIocs(List iocs) throws IOException { List listOfIocToIndexDetails = ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails(); DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails = - new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocAlias, writeIndex); + new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, writeIndex); listOfIocToIndexDetails.add(iocToIndexDetails); } }); @@ -167,7 +168,7 @@ public void indexIocs(List iocs) throws IOException { newIoctoIndexDetails.setWriteIndex(writeIndex); } else { DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails = - new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocAlias, writeIndex); + new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, writeIndex); listOfIocToIndexDetails.add(iocToIndexDetails); } @@ -273,8 +274,8 @@ public static String getIocIndexAlias(String feedSourceConfigId) { return IOC_WRITE_INDEX_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); } - public static String getAllIocIndexPatternByAlias(String feedSourceConfigId) { - return IOC_ALL_INDEX_PATTERN_BY_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); + public static String getAllIocIndexPatternById(String feedSourceConfigId) { + return IOC_ALL_INDEX_PATTERN_BY_ID.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT)); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java index 13c50a062..fd48a7a1d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java @@ -10,17 +10,15 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.securityanalytics.commons.model.IOCType; -import org.opensearch.securityanalytics.model.Value; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; /** * Model used for the default IOC store configuration - * Stores the IOC mapping in a map of string to list of strings + * Stores the IOC mapping in a list of IocToIndexDetails which contains the ioc type, index pattern, and write index */ public class DefaultIocStoreConfig extends IocStoreConfig implements Writeable, ToXContent { private static final Logger log = LogManager.getLogger(DefaultIocStoreConfig.class); @@ -90,15 +88,15 @@ public List getIocToIndexDetails() { public static class IocToIndexDetails implements Writeable, ToXContent { public static final String IOC_TYPE_FIELD = "ioc_type"; - public static final String ALIAS_FIELD = "alias"; + public static final String INDEX_PATTERN_FIELD = "index_pattern"; public static final String WRITE_INDEX_FIELD = "write_index"; IOCType iocType; - String alias; + String indexPattern; String writeIndex; - public IocToIndexDetails(IOCType iocType, String alias, String writeIndex) { + public IocToIndexDetails(IOCType iocType, String indexPattern, String writeIndex) { this.iocType = iocType; - this.alias = alias; + this.indexPattern = indexPattern; this.writeIndex = writeIndex; } @@ -110,7 +108,7 @@ public IocToIndexDetails(StreamInput sin) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeEnum(iocType); - out.writeString(alias); + out.writeString(indexPattern); out.writeString(writeIndex); } @@ -118,14 +116,14 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { return builder.startObject() .field(IOC_TYPE_FIELD, iocType) - .field(ALIAS_FIELD, alias) + .field(INDEX_PATTERN_FIELD, indexPattern) .field(WRITE_INDEX_FIELD, writeIndex) .endObject(); } public static IocToIndexDetails parse(XContentParser xcp) throws IOException { IOCType iocType = null; - String alias = null; + String indexPattern = null; String writeIndex = null; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); @@ -137,8 +135,8 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException { case IOC_TYPE_FIELD: iocType = toIocType(xcp.text()); break; - case ALIAS_FIELD: - alias = xcp.text(); + case INDEX_PATTERN_FIELD: + indexPattern = xcp.text(); break; case WRITE_INDEX_FIELD: writeIndex = xcp.text(); @@ -147,7 +145,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException { xcp.skipChildren(); } } - return new IocToIndexDetails(iocType, alias, writeIndex); + return new IocToIndexDetails(iocType, indexPattern, writeIndex); } public static IOCType toIocType(String name) { @@ -167,12 +165,12 @@ public void setIocType(IOCType iocType) { this.iocType = iocType; } - public String getAlias() { - return alias; + public String getIndexPattern() { + return indexPattern; } - public void setAlias(String alias) { - this.alias = alias; + public void setIndexPattern(String indexPattern) { + this.indexPattern = indexPattern; } public String getWriteIndex() { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 5d694725d..2da99f9c0 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -3,7 +3,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -31,7 +30,6 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; @@ -46,7 +44,6 @@ import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -57,7 +54,6 @@ import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternByAlias; import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD; /** @@ -350,6 +346,14 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene // Index the new iocs downloadAndSaveIOCs(updatedSaTifSourceConfig, stix2IOCList, ActionListener.wrap( downloadAndSaveIocsResponse -> { + + Set iocIndexPatterns = new HashSet<>(); + if (updatedSaTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + // get all the index patterns + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); + } + saTifSourceConfigService.getClusterState(ActionListener.wrap( clusterStateResponse -> { List iocTypes = updatedSaTifSourceConfig.getIocTypes(); @@ -392,7 +396,7 @@ private void storeAndDeleteIocIndices(List stix2IOCList, ActionListene log.error("Failed to get the cluster metadata"); listener.onFailure(e); } - ), getAllIocIndexPatternByAlias(updatedSaTifSourceConfig.getId())); + ), iocIndexPatterns.toArray(new String[0])); }, e -> { log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId()); @@ -552,10 +556,13 @@ public void deleteOldIocIndices( ) { Set writeIndices = new HashSet<>(); IocStoreConfig iocStoreConfig = saTifSourceConfig.getIocStoreConfig(); + Set iocIndexPatterns = new HashSet<>(); if (iocStoreConfig instanceof DefaultIocStoreConfig) { // get the write indices DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> writeIndices.add(e.getWriteIndex())); + // get all the index patterns + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); } saTifSourceConfigService.getClusterState(ActionListener.wrap( @@ -581,7 +588,7 @@ public void deleteOldIocIndices( log.error("Failed to get the cluster metadata"); listener.onFailure(e); } - ), getAllIocIndexPatternByAlias(saTifSourceConfig.getId())); + ), iocIndexPatterns.toArray(new String[0])); } /** @@ -685,6 +692,12 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList TIFJobState.DELETING, ActionListener.wrap( updateSaTifSourceConfigResponse -> { + Set iocIndexPatterns = new HashSet<>(); + if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { + // get all the index patterns + DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig(); + defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); + } saTifSourceConfigService.getClusterState(ActionListener.wrap( clusterStateResponse -> { Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); @@ -709,7 +722,7 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList log.error("Failed to get the cluster metadata"); listener.onFailure(e); } - ), getAllIocIndexPatternByAlias(updateSaTifSourceConfigResponse.getId())); + ), iocIndexPatterns.toArray(new String[0])); }, e -> { log.error("Failed to update threat intel source config with state as {}", TIFJobState.DELETING); listener.onFailure(e); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 954ccc81a..bea89ded0 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -73,7 +73,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternByAlias; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; @@ -490,6 +489,10 @@ public void checkAndEnsureThreatIntelMonitorsDeleted( } + /** + * Returns a map of ioc type to a list of active write indices + * @param listener + */ public void getIocTypeToIndices(ActionListener>> listener) { SearchRequest searchRequest = new SearchRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java index fd5239077..c90cde1b6 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java @@ -130,7 +130,9 @@ void start() { DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig(); for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails: iocStoreConfig.getIocToIndexDetails()) { String writeIndex = iocToindexDetails.getWriteIndex(); - iocIndices.add(writeIndex); + if (writeIndex != null) { + iocIndices.add(writeIndex); + } } } } diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index 8f9332b52..1ef52de95 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -181,7 +181,7 @@ "ioc_type": { "type": "keyword" }, - "alias": { + "index_pattern": { "type": "keyword" }, "write_index": {