Skip to content

Commit

Permalink
Ioc upload integ tests and fix update (#1162)
Browse files Browse the repository at this point in the history 
* add tests and fix error msg

Signed-off-by: Joanne Wang <[email protected]>

* fix feeds store

Signed-off-by: Joanne Wang <[email protected]>

* fix msg

Signed-off-by: Joanne Wang <[email protected]>

* add return so flushIOCs isn't called on failure

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
  • Loading branch information
@jowg-amazon
jowg-amazon authored Jul 17, 2024
1 parent 105e590 commit c8ad00f
Show file tree
Hide file tree
Showing 5 changed files with 492 additions and 40 deletions.
14 changes: 8 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,17 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {

initFeedIndex(newActiveIndex, ActionListener.wrap(
r -> {
// reset the store configs
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().clear();
}

// recreate the store configs
saTifSourceConfig.getIocTypes().forEach(type -> {
IOCType iocType = new IOCType(type);
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
listOfIocToIndexDetails.removeIf(iocToIndexDetails -> iocToIndexDetails.getIocType() == iocType);
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, newActiveIndex);
listOfIocToIndexDetails.add(iocToIndexDetails);
new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex);
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails);
}
});
bulkIndexIocs(iocs, newActiveIndex);
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionList
} catch (Exception e) {
log.error("Failed to download IOCs.", e);
listener.onFailure(e);
return;
}

try {
Expand Down
55 changes: 34 additions & 21 deletions .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,15 @@
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
Expand Down Expand Up @@ -158,7 +160,7 @@ public void createIocAndTIFSourceConfig(
));
},
e -> {
log.error("Failed to download and save IOCs for source config [{}]", indexSaTifSourceConfigResponse.getId());
log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId());
saTifSourceConfigService.deleteTIFSourceConfig(indexSaTifSourceConfigResponse, ActionListener.wrap(
deleteResponse -> {
log.debug("Successfully deleted threat intel source config [{}]", indexSaTifSourceConfigResponse.getId());
Expand Down Expand Up @@ -204,15 +206,15 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
validStix2IocList.add(stix2IOC);
} else {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
log.error("{} is not a supported Ioc type for threat intel source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().toString(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
);
}
}
if (validStix2IocList.isEmpty()) {
log.error("No supported IOCs to index");
actionListener.onFailure(new OpenSearchStatusException("No compatible Iocs were uploaded for config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)));
return;
}
stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener);
Expand Down Expand Up @@ -274,14 +276,18 @@ public void updateIocAndTIFSourceConfig(
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap(
retrievedSaTifSourceConfig -> {
if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) {
log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState());
listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()));
log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()),
RestStatus.BAD_REQUEST)));
return;
}

if (false == saTifSourceConfigDto.getType().equals(retrievedSaTifSourceConfig.getType())) {
log.error("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType());
listener.onFailure(new OpenSearchException("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()));
log.error("Unable to update threat intel source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Unable to update threat intel source config, type cannot change from %s to %s", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()),
RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -345,7 +351,6 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene

saTifSourceConfigService.getClusterState(ActionListener.wrap(
clusterStateResponse -> {
List<String> iocTypes = updatedSaTifSourceConfig.getIocTypes();
IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig();
Set<String> activeIndices = new HashSet<>();
Set<String> indicesToDelete = new HashSet<>();
Expand Down Expand Up @@ -388,11 +393,13 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
), iocIndexPatterns.toArray(new String[0]));
},
e -> {
log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId());
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e);
markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
r -> {
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()),
e)));
}, ex -> {
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(ex);
Expand Down Expand Up @@ -423,14 +430,18 @@ public void refreshTIFSourceConfig(
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap(
saTifSourceConfig -> {
if (saTifSourceConfig.getType() == IOC_UPLOAD) {
log.error("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
listener.onFailure(new OpenSearchException("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD));
log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), IOC_UPLOAD),
RestStatus.BAD_REQUEST)));
return;
}

if (TIFJobState.AVAILABLE.equals(saTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(saTifSourceConfig.getState()) == false) {
log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState());
listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()));
log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()),
RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -488,14 +499,16 @@ private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> l
));
}, downloadAndSaveIocsError -> {
// Update source config as refresh failed
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId());
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId(), downloadAndSaveIocsError);
markSourceConfigAsAction(updatedSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
r -> {
log.debug("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()));
}, e -> {
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSourceConfig.getId()),
downloadAndSaveIocsError)));
}, ex -> {
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
listener.onFailure(e);
listener.onFailure(ex);
}
));
}));
Expand Down Expand Up @@ -528,7 +541,7 @@ public void deleteTIFSourceConfig(
}, e -> {
log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId);
if (e instanceof IndexNotFoundException) {
listener.onFailure(new OpenSearchException("Threat intel source config [{}] not found", saTifSourceConfigId));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND)));
} else {
listener.onFailure(e);
}
Expand Down Expand Up @@ -708,7 +721,7 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList
}
));
}, e -> {
log.error("Failed to delete IOC indices for source config [{}]", updateSaTifSourceConfigResponse.getId());
log.error("Failed to delete IOC indices for threat intel source config [{}]", updateSaTifSourceConfigResponse.getId());
listener.onFailure(e);
}
));
Expand Down
11 changes: 5 additions & 6 deletions ...n/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchException;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.ResourceAlreadyExistsException;
import org.opensearch.action.StepListener;
Expand Down Expand Up @@ -226,7 +225,7 @@ public void getTIFSourceConfig(
client.get(getRequest, ActionListener.wrap(
getResponse -> {
if (!getResponse.isExists()) {
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND)));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND)));
return;
}
SATIFSourceConfig saTifSourceConfig = null;
Expand All @@ -238,7 +237,7 @@ public void getTIFSourceConfig(
saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion());
}
if (saTifSourceConfig == null) {
actionListener.onFailure(new OpenSearchException("No threat intel source config exists [{}]", tifSourceConfigId));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST)));
} else {
log.debug("Threat intel source config with id [{}] fetched", getResponse.getId());
actionListener.onResponse(saTifSourceConfig);
Expand All @@ -258,7 +257,7 @@ public void searchTIFSourceConfigs(

// Check to make sure the job index exists
if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) {
actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist"));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -350,7 +349,7 @@ public void deleteTIFSourceConfig(
) {
// check to make sure the job index exists
if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) {
actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist"));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -410,7 +409,7 @@ private void deleteIocIndex(Set<String> indicesToDelete, Boolean backgroundJob,
if (!response.isAcknowledged()) {
log.error("Could not delete one or more IOC indices: " + index);
if (backgroundJob == false) {
listener.onFailure(new OpenSearchException("Could not delete one or more IOC indices: " + index));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR)));
}
} else {
log.debug("Successfully deleted one or more IOC indices:" + index);
Expand Down
Loading

0 comments on commit c8ad00f

Please sign in to comment.