diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 9c8f81025..de483deea 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -311,14 +311,10 @@ public void updateIocAndTIFSourceConfig( log.info("Set threat intel source config as REFRESHING for [{}]", updatedSaTifSourceConfig.getId()); switch (updatedSaTifSourceConfig.getType()) { case S3_CUSTOM: - downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig); + downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, null); break; case IOC_UPLOAD: - storeAndDeleteIocIndices( - iocs, - listener, - updatedSaTifSourceConfig - ); + downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, iocs); break; } }, e -> { @@ -337,78 +333,6 @@ public void updateIocAndTIFSourceConfig( } } - private void storeAndDeleteIocIndices(List stix2IOCList, ActionListener listener, SATIFSourceConfig updatedSaTifSourceConfig) { - // Index the new iocs - downloadAndSaveIOCs(updatedSaTifSourceConfig, stix2IOCList, ActionListener.wrap( - downloadAndSaveIocsResponse -> { - - Set iocIndexPatterns = new HashSet<>(); - if (updatedSaTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { - // get all the index patterns - DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig(); - defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); - } - - saTifSourceConfigService.getClusterState(ActionListener.wrap( - clusterStateResponse -> { - IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig(); - Set activeIndices = new HashSet<>(); - Set indicesToDelete = new HashSet<>(); - - if (iocStoreConfig instanceof DefaultIocStoreConfig) { - DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) iocStoreConfig; - Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); - - // remove ioc types not specified in list - defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString())); - - // get the active indices - defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex())); - - for (String index : concreteIndices) { - if (false == activeIndices.contains(index)) { - indicesToDelete.add(index); - } - } - } - - // delete the old indices - saTifSourceConfigService.deleteAllIocIndices(indicesToDelete, true, null); - markSourceConfigAsAction( - updatedSaTifSourceConfig, - TIFJobState.AVAILABLE, - ActionListener.wrap( - saTifSourceConfigResponse -> { - SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(saTifSourceConfigResponse); - listener.onResponse(returnedSaTifSourceConfigDto); - }, e -> { - log.error("Failed to index threat intel source config with id [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(e); - } - )); - }, e -> { - log.error("Failed to get the cluster metadata"); - listener.onFailure(e); - } - ), iocIndexPatterns.toArray(new String[0])); - }, - e -> { - log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e); - markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap( - r -> { - log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException( - String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()), - e))); - }, ex -> { - log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(ex); - } - )); - }) - ); - } - public void internalUpdateTIFSourceConfig( final SATIFSourceConfig saTifSourceConfig, final ActionListener listener @@ -455,7 +379,7 @@ public void refreshTIFSourceConfig( saTifSourceConfig.setLastRefreshedTime(Instant.now()); markSourceConfigAsAction(saTifSourceConfig, TIFJobState.REFRESHING, ActionListener.wrap( updatedSourceConfig -> { - downloadAndSaveIocsToRefresh(listener, updatedSourceConfig); + downloadAndSaveIocsToRefresh(listener, updatedSourceConfig, null); }, e -> { log.error("Failed to set threat intel source config as REFRESHING for [{}]", saTifSourceConfig.getId()); listener.onFailure(e); @@ -468,19 +392,12 @@ public void refreshTIFSourceConfig( )); } - private void downloadAndSaveIocsToRefresh(ActionListener listener, SATIFSourceConfig updatedSourceConfig) { - downloadAndSaveIOCs(updatedSourceConfig, null, ActionListener.wrap( + private void downloadAndSaveIocsToRefresh(ActionListener listener, SATIFSourceConfig updatedSourceConfig, List stix2IOCList) { + downloadAndSaveIOCs(updatedSourceConfig, stix2IOCList, ActionListener.wrap( response -> { // delete old IOCs and update the source config deleteOldIocIndices(updatedSourceConfig, ActionListener.wrap( newIocStoreConfig -> { - List iocTypes = updatedSourceConfig.getIocTypes(); - if (newIocStoreConfig instanceof DefaultIocStoreConfig) { - DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig; - // remove ioc types not specified in list - defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString())); - updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig); - } // Update source config as succeeded, change state back to available markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap( r -> { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 35571facd..34be1e58e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -179,7 +179,7 @@ private String getIndexMapping() { } } catch (IOException e) { log.error("Failed to get the threat intel index mapping", e); - throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.BAD_REQUEST, e); } } @@ -409,7 +409,7 @@ private void deleteIocIndex(Set indicesToDelete, Boolean backgroundJob, if (!response.isAcknowledged()) { log.error("Could not delete one or more IOC indices: " + index); if (backgroundJob == false) { - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR))); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.BAD_REQUEST))); } } else { log.debug("Successfully deleted one or more IOC indices:" + index); diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 81e3498bc..f485d3134 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -111,6 +111,7 @@ import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; @@ -1539,6 +1540,24 @@ public List getIocFindingIndices() throws IOException { return indices; } + public List getIocIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IOC_ALL_INDEX_PATTERN + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + public List getQueryIndices(String detectorType) throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getRuleIndex(detectorType) + "*?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index 3989f38f3..f00c8e9a8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -341,6 +341,10 @@ public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedExc response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); Assert.assertEquals(RestStatus.OK, restStatus(response)); + // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) + List findingIndices = getIocIndices(); + Assert.assertEquals(3, findingIndices.size()); + // Retrieve all IOCs by feed Ids iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); Assert.assertEquals(RestStatus.OK, restStatus(iocResponse));