Skip to content

Commit

Permalink
change alias to index pattern
Browse files Browse the repository at this point in the history 
Signed-off-by: Joanne Wang <[email protected]>
  • Loading branch information
@jowg-amazon
jowg-amazon committed Jul 3, 2024
1 parent bdd5669 commit 4b1d699
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 32 deletions.
11 changes: 6 additions & 5 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public class STIX2IOCFeedStore implements FeedStore {
public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*";
public static final String IOC_FEED_ID_PLACEHOLDER = "FEED_ID";
public static final String IOC_INDEX_NAME_TEMPLATE = IOC_INDEX_NAME_BASE + "-" + IOC_FEED_ID_PLACEHOLDER;
public static final String IOC_ALL_INDEX_PATTERN_BY_ALIAS = IOC_INDEX_NAME_TEMPLATE + "-*";
public static final String IOC_ALL_INDEX_PATTERN_BY_ID = IOC_INDEX_NAME_TEMPLATE + "-*";
public static final String IOC_WRITE_INDEX_ALIAS = IOC_INDEX_NAME_TEMPLATE;
public static final String IOC_TIME_PLACEHOLDER = "TIME";
public static final String IOC_INDEX_PATTERN = IOC_INDEX_NAME_TEMPLATE + "-" + IOC_TIME_PLACEHOLDER;
Expand Down Expand Up @@ -121,6 +121,7 @@ public void storeIOCs(Map<IOC, UpdateAction> actionToIOCs) {
public void indexIocs(List<STIX2IOC> iocs) throws IOException {
String iocAlias = getIocIndexAlias(saTifSourceConfig.getId());
String iocPattern = getIocIndexRolloverPattern(saTifSourceConfig.getId());
String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId());

if (iocIndexExists(iocAlias) == false) {
initFeedIndex(iocAlias, iocPattern, ActionListener.wrap(
Expand All @@ -132,7 +133,7 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocAlias, writeIndex);
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, writeIndex);
listOfIocToIndexDetails.add(iocToIndexDetails);
}
});
Expand Down Expand Up @@ -167,7 +168,7 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {
newIoctoIndexDetails.setWriteIndex(writeIndex);
} else {
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocAlias, writeIndex);
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, writeIndex);
listOfIocToIndexDetails.add(iocToIndexDetails);
}

Expand Down Expand Up @@ -273,8 +274,8 @@ public static String getIocIndexAlias(String feedSourceConfigId) {
return IOC_WRITE_INDEX_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT));
}

public static String getAllIocIndexPatternByAlias(String feedSourceConfigId) {
return IOC_ALL_INDEX_PATTERN_BY_ALIAS.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT));
public static String getAllIocIndexPatternById(String feedSourceConfigId) {
return IOC_ALL_INDEX_PATTERN_BY_ID.replace(IOC_FEED_ID_PLACEHOLDER, feedSourceConfigId.toLowerCase(Locale.ROOT));
}


Expand Down
32 changes: 15 additions & 17 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/model/DefaultIocStoreConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,15 @@
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.model.Value;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;

/**
* Model used for the default IOC store configuration
* Stores the IOC mapping in a map of string to list of strings
* Stores the IOC mapping in a list of IocToIndexDetails which contains the ioc type, index pattern, and write index
*/
public class DefaultIocStoreConfig extends IocStoreConfig implements Writeable, ToXContent {
private static final Logger log = LogManager.getLogger(DefaultIocStoreConfig.class);
Expand Down Expand Up @@ -90,15 +88,15 @@ public List<IocToIndexDetails> getIocToIndexDetails() {

public static class IocToIndexDetails implements Writeable, ToXContent {
public static final String IOC_TYPE_FIELD = "ioc_type";
public static final String ALIAS_FIELD = "alias";
public static final String INDEX_PATTERN_FIELD = "index_pattern";
public static final String WRITE_INDEX_FIELD = "write_index";
IOCType iocType;
String alias;
String indexPattern;
String writeIndex;

public IocToIndexDetails(IOCType iocType, String alias, String writeIndex) {
public IocToIndexDetails(IOCType iocType, String indexPattern, String writeIndex) {
this.iocType = iocType;
this.alias = alias;
this.indexPattern = indexPattern;
this.writeIndex = writeIndex;
}

Expand All @@ -110,22 +108,22 @@ public IocToIndexDetails(StreamInput sin) throws IOException {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeEnum(iocType);
out.writeString(alias);
out.writeString(indexPattern);
out.writeString(writeIndex);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return builder.startObject()
.field(IOC_TYPE_FIELD, iocType)
.field(ALIAS_FIELD, alias)
.field(INDEX_PATTERN_FIELD, indexPattern)
.field(WRITE_INDEX_FIELD, writeIndex)
.endObject();
}

public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
IOCType iocType = null;
String alias = null;
String indexPattern = null;
String writeIndex = null;

XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp);
Expand All @@ -137,8 +135,8 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
case IOC_TYPE_FIELD:
iocType = toIocType(xcp.text());
break;
case ALIAS_FIELD:
alias = xcp.text();
case INDEX_PATTERN_FIELD:
indexPattern = xcp.text();
break;
case WRITE_INDEX_FIELD:
writeIndex = xcp.text();
Expand All @@ -147,7 +145,7 @@ public static IocToIndexDetails parse(XContentParser xcp) throws IOException {
xcp.skipChildren();
}
}
return new IocToIndexDetails(iocType, alias, writeIndex);
return new IocToIndexDetails(iocType, indexPattern, writeIndex);
}

public static IOCType toIocType(String name) {
Expand All @@ -167,12 +165,12 @@ public void setIocType(IOCType iocType) {
this.iocType = iocType;
}

public String getAlias() {
return alias;
public String getIndexPattern() {
return indexPattern;
}

public void setAlias(String alias) {
this.alias = alias;
public void setIndexPattern(String indexPattern) {
this.indexPattern = indexPattern;
}

public String getWriteIndex() {
Expand Down
27 changes: 20 additions & 7 deletions .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchException;
import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
import org.opensearch.action.delete.DeleteResponse;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
Expand Down Expand Up @@ -31,7 +30,6 @@
import org.opensearch.search.SearchHit;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.commons.model.IOCType;
import org.opensearch.securityanalytics.model.STIX2IOC;
import org.opensearch.securityanalytics.model.STIX2IOCDto;
import org.opensearch.securityanalytics.services.STIX2IOCFetchService;
Expand All @@ -46,7 +44,6 @@

import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
Expand All @@ -57,7 +54,6 @@

import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternByAlias;
import static org.opensearch.securityanalytics.threatIntel.common.SourceConfigType.IOC_UPLOAD;

/**
Expand Down Expand Up @@ -350,6 +346,14 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
// Index the new iocs
downloadAndSaveIOCs(updatedSaTifSourceConfig, stix2IOCList, ActionListener.wrap(
downloadAndSaveIocsResponse -> {

Set<String> iocIndexPatterns = new HashSet<>();
if (updatedSaTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
// get all the index patterns
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig();
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern()));
}

saTifSourceConfigService.getClusterState(ActionListener.wrap(
clusterStateResponse -> {
List<String> iocTypes = updatedSaTifSourceConfig.getIocTypes();
Expand Down Expand Up @@ -392,7 +396,7 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
log.error("Failed to get the cluster metadata");
listener.onFailure(e);
}
), getAllIocIndexPatternByAlias(updatedSaTifSourceConfig.getId()));
), iocIndexPatterns.toArray(new String[0]));
},
e -> {
log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId());
Expand Down Expand Up @@ -552,10 +556,13 @@ public void deleteOldIocIndices(
) {
Set<String> writeIndices = new HashSet<>();
IocStoreConfig iocStoreConfig = saTifSourceConfig.getIocStoreConfig();
Set<String> iocIndexPatterns = new HashSet<>();
if (iocStoreConfig instanceof DefaultIocStoreConfig) {
// get the write indices
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig();
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> writeIndices.add(e.getWriteIndex()));
// get all the index patterns
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern()));
}

saTifSourceConfigService.getClusterState(ActionListener.wrap(
Expand All @@ -581,7 +588,7 @@ public void deleteOldIocIndices(
log.error("Failed to get the cluster metadata");
listener.onFailure(e);
}
), getAllIocIndexPatternByAlias(saTifSourceConfig.getId()));
), iocIndexPatterns.toArray(new String[0]));
}

/**
Expand Down Expand Up @@ -685,6 +692,12 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList
TIFJobState.DELETING,
ActionListener.wrap(
updateSaTifSourceConfigResponse -> {
Set<String> iocIndexPatterns = new HashSet<>();
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
// get all the index patterns
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig();
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern()));
}
saTifSourceConfigService.getClusterState(ActionListener.wrap(
clusterStateResponse -> {
Set<String> concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse);
Expand All @@ -709,7 +722,7 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList
log.error("Failed to get the cluster metadata");
listener.onFailure(e);
}
), getAllIocIndexPatternByAlias(updateSaTifSourceConfigResponse.getId()));
), iocIndexPatterns.toArray(new String[0]));
}, e -> {
log.error("Failed to update threat intel source config with state as {}", TIFJobState.DELETING);
listener.onFailure(e);
Expand Down
5 changes: 4 additions & 1 deletion ...n/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@
import java.util.Set;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternByAlias;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT;
import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE;
import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING;
Expand Down Expand Up @@ -490,6 +489,10 @@ public void checkAndEnsureThreatIntelMonitorsDeleted(

}

/**
* Returns a map of ioc type to a list of active write indices
* @param listener
*/
public void getIocTypeToIndices(ActionListener<Map<String, List<String>>> listener) {
SearchRequest searchRequest = new SearchRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME);

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ void start() {
DefaultIocStoreConfig iocStoreConfig = (DefaultIocStoreConfig) config.getIocStoreConfig();
for (DefaultIocStoreConfig.IocToIndexDetails iocToindexDetails: iocStoreConfig.getIocToIndexDetails()) {
String writeIndex = iocToindexDetails.getWriteIndex();
iocIndices.add(writeIndex);
if (writeIndex != null) {
iocIndices.add(writeIndex);
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/resources/mappings/threat_intel_job_mapping.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@
"ioc_type": {
"type": "keyword"
},
"alias": {
"index_pattern": {
"type": "keyword"
},
"write_index": {
Expand Down

0 comments on commit 4b1d699

Please sign in to comment.