Skip to content

Commit

Permalink
Fix ioc upload update behavior and change error response (opensearch-…
Browse files Browse the repository at this point in the history 
…project#1192)

* fix ioc upload update behavior and change error response

Signed-off-by: Joanne Wang <[email protected]>

* update test

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
  • Loading branch information
@jowg-amazon
jowg-amazon authored Jul 23, 2024
1 parent 04b7719 commit f1e9d20
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 90 deletions.
93 changes: 5 additions & 88 deletions .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,14 +311,10 @@ public void updateIocAndTIFSourceConfig(
log.info("Set threat intel source config as REFRESHING for [{}]", updatedSaTifSourceConfig.getId());
switch (updatedSaTifSourceConfig.getType()) {
case S3_CUSTOM:
downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig);
downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, null);
break;
case IOC_UPLOAD:
storeAndDeleteIocIndices(
iocs,
listener,
updatedSaTifSourceConfig
);
downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, iocs);
break;
}
}, e -> {
Expand All @@ -337,78 +333,6 @@ public void updateIocAndTIFSourceConfig(
}
}

private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListener<SATIFSourceConfigDto> listener, SATIFSourceConfig updatedSaTifSourceConfig) {
// Index the new iocs
downloadAndSaveIOCs(updatedSaTifSourceConfig, stix2IOCList, ActionListener.wrap(
downloadAndSaveIocsResponse -> {

Set<String> iocIndexPatterns = new HashSet<>();
if (updatedSaTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
// get all the index patterns
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig();
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern()));
}

saTifSourceConfigService.getClusterState(ActionListener.wrap(
clusterStateResponse -> {
IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig();
Set<String> activeIndices = new HashSet<>();
Set<String> indicesToDelete = new HashSet<>();

if (iocStoreConfig instanceof DefaultIocStoreConfig) {
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) iocStoreConfig;
Set<String> concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse);

// remove ioc types not specified in list
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));

// get the active indices
defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex()));

for (String index : concreteIndices) {
if (false == activeIndices.contains(index)) {
indicesToDelete.add(index);
}
}
}

// delete the old indices
saTifSourceConfigService.deleteAllIocIndices(indicesToDelete, true, null);
markSourceConfigAsAction(
updatedSaTifSourceConfig,
TIFJobState.AVAILABLE,
ActionListener.wrap(
saTifSourceConfigResponse -> {
SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(saTifSourceConfigResponse);
listener.onResponse(returnedSaTifSourceConfigDto);
}, e -> {
log.error("Failed to index threat intel source config with id [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(e);
}
));
}, e -> {
log.error("Failed to get the cluster metadata");
listener.onFailure(e);
}
), iocIndexPatterns.toArray(new String[0]));
},
e -> {
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e);
markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
r -> {
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()),
e)));
}, ex -> {
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(ex);
}
));
})
);
}

public void internalUpdateTIFSourceConfig(
final SATIFSourceConfig saTifSourceConfig,
final ActionListener<SATIFSourceConfig> listener
Expand Down Expand Up @@ -455,7 +379,7 @@ public void refreshTIFSourceConfig(
saTifSourceConfig.setLastRefreshedTime(Instant.now());
markSourceConfigAsAction(saTifSourceConfig, TIFJobState.REFRESHING, ActionListener.wrap(
updatedSourceConfig -> {
downloadAndSaveIocsToRefresh(listener, updatedSourceConfig);
downloadAndSaveIocsToRefresh(listener, updatedSourceConfig, null);
}, e -> {
log.error("Failed to set threat intel source config as REFRESHING for [{}]", saTifSourceConfig.getId());
listener.onFailure(e);
Expand All @@ -468,19 +392,12 @@ public void refreshTIFSourceConfig(
));
}

private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> listener, SATIFSourceConfig updatedSourceConfig) {
downloadAndSaveIOCs(updatedSourceConfig, null, ActionListener.wrap(
private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> listener, SATIFSourceConfig updatedSourceConfig, List<STIX2IOC> stix2IOCList) {
downloadAndSaveIOCs(updatedSourceConfig, stix2IOCList, ActionListener.wrap(
response -> {
// delete old IOCs and update the source config
deleteOldIocIndices(updatedSourceConfig, ActionListener.wrap(
newIocStoreConfig -> {
List<String> iocTypes = updatedSourceConfig.getIocTypes();
if (newIocStoreConfig instanceof DefaultIocStoreConfig) {
DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig;
// remove ioc types not specified in list
defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString()));
updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig);
}
// Update source config as succeeded, change state back to available
markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap(
r -> {
Expand Down
4 changes: 2 additions & 2 deletions ...n/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ private String getIndexMapping() {
}
} catch (IOException e) {
log.error("Failed to get the threat intel index mapping", e);
throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e);
throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.BAD_REQUEST, e);
}
}

Expand Down Expand Up @@ -409,7 +409,7 @@ private void deleteIocIndex(Set<String> indicesToDelete, Boolean backgroundJob,
if (!response.isAcknowledged()) {
log.error("Could not delete one or more IOC indices: " + index);
if (backgroundJob == false) {
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR)));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.BAD_REQUEST)));
}
} else {
log.debug("Successfully deleted one or more IOC indices:" + index);
Expand Down
19 changes: 19 additions & 0 deletions src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@
import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule;
import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings;
import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping;
import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS;
import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD;
Expand Down Expand Up @@ -1539,6 +1540,24 @@ public List<String> getIocFindingIndices() throws IOException {
return indices;
}

public List<String> getIocIndices() throws IOException {
Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IOC_ALL_INDEX_PATTERN + "?format=json"));
XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent());
List<Object> responseList = xcp.list();
List<String> indices = new ArrayList<>();
for (Object o : responseList) {
if (o instanceof Map) {
((Map<?, ?>) o).forEach((BiConsumer<Object, Object>)
(o1, o2) -> {
if (o1.equals("index")) {
indices.add((String) o2);
}
});
}
}
return indices;
}

public List<String> getQueryIndices(String detectorType) throws IOException {
Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getRuleIndex(detectorType) + "*?format=json"));
XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent());
Expand Down
4 changes: 4 additions & 0 deletions ...est/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,10 @@ public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedExc
response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto));
Assert.assertEquals(RestStatus.OK, restStatus(response));

// Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config)
List<String> findingIndices = getIocIndices();
Assert.assertEquals(3, findingIndices.size());

// Retrieve all IOCs by feed Ids
iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null);
Assert.assertEquals(RestStatus.OK, restStatus(iocResponse));
Expand Down

0 comments on commit f1e9d20

Please sign in to comment.