diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 0d28bce4d..20cc34ffd 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -10,6 +10,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -69,8 +70,11 @@ public void ensureLogTypesLoaded() { private List loadBuiltinLogTypes() throws URISyntaxException, IOException { List logTypes = new ArrayList<>(); - final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); + String pathurl = Paths.get(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH).toURI()).toString(); + final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); + logger.error("SASHANK Path url is {}", pathurl); + logger.error("SASHANK currently used url is {}", url); Path dirPath = null; if (url.contains("!")) { final String[] paths = url.split("!"); diff --git a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java similarity index 95% rename from src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java rename to src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java index 04f54699f..037541741 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java +++ b/src/main/java/org/opensearch/securityanalytics/model/threatintel/IocMatch.java @@ -1,4 +1,4 @@ -package org.opensearch.securityanalytics.model; +package org.opensearch.securityanalytics.model.threatintel; import org.apache.commons.lang3.StringUtils; import org.opensearch.core.common.io.stream.StreamInput; @@ -20,7 +20,7 @@ * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job * It's the inverse of an IoC finding which maps a document to list of IoC's */ -public class IoCMatch implements Writeable, ToXContent { +public class IocMatch implements Writeable, ToXContent { //TODO implement IoC_Match interface from security-analytics-commons public static final String ID_FIELD = "id"; public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; @@ -42,7 +42,7 @@ public class IoCMatch implements Writeable, ToXContent { private final Instant timestamp; private final String executionId; - public IoCMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, + public IocMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); this.id = id; @@ -56,7 +56,7 @@ public IoCMatch(String id, List relatedDocIds, List feedIds, Str this.executionId = executionId; } - public IoCMatch(StreamInput in) throws IOException { + public IocMatch(StreamInput in) throws IOException { id = in.readString(); relatedDocIds = in.readStringList(); feedIds = in.readStringList(); @@ -133,7 +133,7 @@ public String getExecutionId() { return executionId; } - public static IoCMatch parse(XContentParser xcp) throws IOException { + public static IocMatch parse(XContentParser xcp) throws IOException { String id = null; List relatedDocIds = new ArrayList<>(); List feedIds = new ArrayList<>(); @@ -197,11 +197,11 @@ public static IoCMatch parse(XContentParser xcp) throws IOException { } } - return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + return new IocMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); } - public static IoCMatch readFrom(StreamInput in) throws IOException { - return new IoCMatch(in); + public static IocMatch readFrom(StreamInput in) throws IOException { + return new IocMatch(in); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchService.java new file mode 100644 index 000000000..9bf92a985 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchService.java @@ -0,0 +1,155 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.bulk.BulkResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.support.GroupedActionListener; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.threadpool.ThreadPool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Data layer to perform CRUD operations for threat intel ioc match : store in system index. + */ +public class IocMatchService { + //TODO manage index rollover + public static final String INDEX_NAME = ".opensearch-sap-iocmatch"; + private static final Logger log = LogManager.getLogger(IocMatchService.class); + private final Client client; + private final ClusterService clusterService; + + public IocMatchService(final Client client, final ClusterService clusterService) { + this.client = client; + this.clusterService = clusterService; + } + + public void indexIocMatches(List iocMatches, + final ActionListener actionListener) { + try { + Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); + createIndexIfNotExists(ActionListener.wrap( + r -> { + List bulkRequestList = new ArrayList<>(); + BulkRequest bulkRequest = new BulkRequest(INDEX_NAME); + for (int i = 0; i < iocMatches.size(); i++) { + IocMatch iocMatch = iocMatches.get(i); + try { + IndexRequest indexRequest = new IndexRequest(INDEX_NAME) + .source(iocMatch.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .opType(DocWriteRequest.OpType.CREATE); + bulkRequest.add(indexRequest); + if ( + bulkRequest.requests().size() == batchSize + && i != iocMatches.size() - 1 // final bulk request will be added outside for loop with refresh policy none + ) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); + bulkRequestList.add(bulkRequest); + bulkRequest = new BulkRequest(); + } + } catch (IOException e) { + log.error(String.format("Failed to create index request for ioc match %s moving on to next"), e); + } + } + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulkRequestList.add(bulkRequest); + GroupedActionListener groupedListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> { + int idx = 0; + for (BulkResponse response : bulkResponses) { + BulkRequest request = bulkRequestList.get(idx); + if (response.hasFailures()) { + log.error("Failed to bulk index {} Ioc Matches. Failure: {}", request.batchSize(), response.buildFailureMessage()); + } + } + actionListener.onResponse(null); + }, actionListener::onFailure), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { + try { + client.bulk(req, groupedListener); //todo why stash context here? + } catch (Exception e) { + log.error("Failed to save ioc matches.", e); + } + } + }, e -> { + log.error("Failed to create System Index"); + actionListener.onFailure(e); + })); + + + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + } + + private String getIndexMapping() { + try { + try (InputStream is = IocMatchService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); + } + } + } catch (IOException e) { + log.error("Failed to get the threat intel ioc match index mapping", e); + throw new SecurityAnalyticsException("Failed to get the threat intel ioc match index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + /** + * Index name: .opensearch-sap-iocmatch + * Mapping: /mappings/ioc_match_mapping.json + * + * @param listener setup listener + */ + public void createIndexIfNotExists(final ActionListener listener) { + // check if job index exists + try { + if (clusterService.state().metadata().hasIndex(INDEX_NAME) == true) { + listener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("Ioc match index created"); + listener.onResponse(null); + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.debug("index {} already exist", INDEX_NAME); + listener.onResponse(null); + return; + } + log.error("Failed to create security analytics threat intel job index", e); + listener.onFailure(e); + } + )); + } catch (Exception e) { + log.error("Failure in creating ioc_match index", e); + listener.onFailure(e); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java new file mode 100644 index 000000000..3e77f2d28 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/IocScanContext.java @@ -0,0 +1,23 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import org.opensearch.securityanalytics.threatIntel.iocscan.model.IocScanMonitor; + +import java.util.List; + +public class IocScanContext { + IocScanMonitor monitor; + boolean dryRun; + List data; + + public IocScanMonitor getMonitor() { + return monitor; + } + + public boolean isDryRun() { + return dryRun; + } + + public List getData() { + return data; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java new file mode 100644 index 000000000..e101d2c5e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeFieldMappings.java @@ -0,0 +1,26 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dto; + +import java.util.List; +import java.util.Map; + +/** + * DTO that contains information about an Ioc type and the list of fields in each index that map to the given + */ +public class PerIocTypeFieldMappings { + + private final String iocType; + private final Map> indexToFieldsMap; + + public PerIocTypeFieldMappings(String iocType, Map> indexToFieldsMap) { + this.iocType = iocType; + this.indexToFieldsMap = indexToFieldsMap; + } + + public String getIocType() { + return iocType; + } + + public Map> getIndexToFieldsMap() { + return indexToFieldsMap; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java new file mode 100644 index 000000000..5e83b7157 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/Ioc.java @@ -0,0 +1,26 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.model; + +public class Ioc { + + private final String feedId; + private final String iocValue; + private final String iocType; + + public Ioc(String feedId, String iocValue, String iocType) { + this.feedId = feedId; + this.iocValue = iocValue; + this.iocType = iocType; + } + + public String getFeedId() { + return feedId; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java new file mode 100644 index 000000000..17b1c3e48 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/model/IocScanMonitor.java @@ -0,0 +1,30 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.model; + +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings; + +import java.util.List; +import java.util.Map; + + +public class IocScanMonitor { + String id; + String name; + List iocTypeToIndexFieldMappings; + Map> perIoCTypeThreatIntelIndices; + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public List getIocTypeToIndexFieldMappings() { + return iocTypeToIndexFieldMappings; + } + + public Map> getPerIoCTypeThreatIntelIndices() { + return perIoCTypeThreatIntelIndices; + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java new file mode 100644 index 000000000..eb8c14552 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -0,0 +1,235 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.commons.alerting.model.Finding; +import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeFieldMappings; +import org.opensearch.securityanalytics.threatIntel.iocscan.model.Ioc; +import org.opensearch.securityanalytics.threatIntel.iocscan.model.IocScanMonitor; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; + + +public abstract class IoCScanService implements IoCScanServiceInterface { + private static final Logger log = LogManager.getLogger(IoCScanService.class); + + @Override + public void scanIoCs(IocScanContext iocScanContext, + BiConsumer scanCallback + ) { + List data = iocScanContext.getData(); + IocScanMonitor iocScanMonitor = iocScanContext.getMonitor(); + + long start = System.currentTimeMillis(); + // log.debug("beginning to scan IoC's") + IocLookupDtos iocLookupDtos = extractIocPerTypeSet(data, iocScanMonitor.getIocTypeToIndexFieldMappings()); + BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { + if (e == null) { + createIoCMatches(maliciousIocs, iocLookupDtos.iocValueToDocIdMap, iocScanContext, + new BiConsumer, Exception>() { + @Override + public void accept(List iocs, Exception e) { + createFindings(maliciousIocs, iocLookupDtos.docIdToIocsMap, iocScanMonitor); + } + } + ); + + } else { + // onIocMatchFailure(e, iocScanMonitor); + + } + }; + matchAgainstThreatIntelAndReturnMaliciousIocs(iocLookupDtos.getIocsPerIocTypeMap(), iocScanMonitor, iocScanResultConsumer); + } + + abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( + Map> iocPerTypeSet, + IocScanMonitor iocScanMonitor, + BiConsumer, Exception> callback); + + /** + * For each doc, we extract the list of + */ + private IocLookupDtos extractIocPerTypeSet(List data, List iocTypeToIndexFieldMappings) { + Map> iocsPerIocTypeMap = new HashMap<>(); + Map> iocValueToDocIdMap = new HashMap<>(); + Map> docIdToIocsMap = new HashMap<>(); + for (Data datum : data) { + for (PerIocTypeFieldMappings iocTypeToIndexFieldMapping : iocTypeToIndexFieldMappings) { + String iocType = iocTypeToIndexFieldMapping.getIocType(); + String index = getIndexName(datum); + List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + for (String field : fields) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String indexName = getIndexName(datum); + String docId = id + ":" + indexName; + Set iocs = docIdToIocsMap.getOrDefault(docIdToIocsMap.get(docId), new HashSet<>()); + iocs.addAll(vals); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(iocValueToDocIdMap.get(ioc), new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } + } + } + } + return new IocLookupDtos(iocsPerIocTypeMap, iocValueToDocIdMap, docIdToIocsMap); + } + + public abstract List getValuesAsStringList(Data datum, String field); + + public abstract String getIndexName(Data datum); + + public abstract String getId(Data datum); + + public void createIoCMatches(List iocs, Map> iocValueToDocIdMap, IocScanContext iocScanContext, BiConsumer, Exception> callback) { + try { + Instant timestamp = Instant.now(); + IocScanMonitor iocScanMonitor = iocScanContext.getMonitor(); + // Map to collect unique IocValue with their respective FeedIds + Map> iocValueToFeedIds = new HashMap<>(); + + for (Ioc ioc : iocs) { + String iocValue = ioc.getIocValue(); + iocValueToFeedIds + .computeIfAbsent(iocValue, k -> new HashSet<>()) + .add(ioc.getFeedId()); + } + + List iocMatches = new ArrayList<>(); + + for (Map.Entry> entry : iocValueToFeedIds.entrySet()) { + String iocValue = entry.getKey(); + Set feedIds = entry.getValue(); + + List relatedDocIds = new ArrayList<>(iocValueToDocIdMap.getOrDefault(iocValue, new HashSet<>())); + List feedIdsList = new ArrayList<>(feedIds); + try { + IocMatch iocMatch = new IocMatch( + UUID.randomUUID().toString(), // Generating a unique ID + relatedDocIds, + feedIdsList, + iocScanMonitor.getId(), + iocScanMonitor.getName(), + iocValue, + iocs.stream().filter(i -> i.getIocValue().equals(iocValue)).findFirst().orElseThrow().getIocType(), + timestamp, + UUID.randomUUID().toString() // TODO execution ID + ); + iocMatches.add(iocMatch); + } catch (Exception e) { + log.error(String.format("skipping creating ioc match for %s due to unexpected failure.", entry.getKey()), e); + } + } + saveIocs(iocs, callback); + } catch (Exception e) { + log.error(() -> new ParameterizedMessage("Failed to create ioc matches due to unexpected error {}", iocScanContext.getMonitor().getId()), e); + callback.accept(null, e); + } + } + + abstract void saveIocs(List iocs, BiConsumer, Exception> callback); + + public List createFindings(List iocs, Map> docIdToIocsMap, IocScanMonitor iocScanMonitor) { + List findings = new ArrayList<>(); + + for (Map.Entry> entry : docIdToIocsMap.entrySet()) { + String docId = entry.getKey(); + Set iocValues = entry.getValue(); + + List iocStrings = new ArrayList<>(iocValues); + + Finding finding = new Finding( + UUID.randomUUID().toString(), // Generating a unique ID + Collections.singletonList(docId), // Singleton list for relatedDocIds + Collections.emptyList(), // Empty list for correlatedDocIds + iocScanMonitor.getId(), + iocScanMonitor.getName(), + "", // Index value, you may need to provide an actual value + Collections.emptyList(), // Empty list for docLevelQueries +// iocStrings, TODO add field in findings + Instant.now(), // Current timestamp + null // Setting executionId as null, you may adjust accordingly + ); + + findings.add(finding); + } + + return findings; + } + + + private static class IocMatchDto { + private final String iocValue; + private final String iocType; + private final List iocs; + private final List docIdsContainingIoc; + + public IocMatchDto(String iocValue, String iocType, List iocs, List docIdsContainingIoc) { + this.iocValue = iocValue; + this.iocType = iocType; + this.iocs = iocs; + this.docIdsContainingIoc = docIdsContainingIoc; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public List getIocs() { + return iocs; + } + + public List getDocIdsContainingIoc() { + return docIdsContainingIoc; + } + } + + private static class IocLookupDtos { + private final Map> iocsPerIocTypeMap; + private final Map> iocValueToDocIdMap; + private final Map> docIdToIocsMap; + + public IocLookupDtos(Map> iocsPerIocTypeMap, Map> iocValueToDocIdMap, Map> docIdToIocsMap) { + this.iocsPerIocTypeMap = iocsPerIocTypeMap; + this.iocValueToDocIdMap = iocValueToDocIdMap; + this.docIdToIocsMap = docIdToIocsMap; + } + + public Map> getIocsPerIocTypeMap() { + return iocsPerIocTypeMap; + } + + public Map> getIocValueToDocIdMap() { + return iocValueToDocIdMap; + } + + public Map> getDocIdToIocsMap() { + return docIdToIocsMap; + } + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java new file mode 100644 index 000000000..0746eddf4 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanServiceInterface.java @@ -0,0 +1,16 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.service; + +import org.opensearch.commons.alerting.model.Finding; +import org.opensearch.securityanalytics.model.threatintel.IocMatch; +import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; + +import java.util.List; +import java.util.function.BiConsumer; + +public interface IoCScanServiceInterface { + + void scanIoCs( + IocScanContext iocScanContext, + BiConsumer scanCallback + ); +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelMonitorRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelMonitorRunner.java new file mode 100644 index 000000000..9e81ef484 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelMonitorRunner.java @@ -0,0 +1,11 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.opensearch.client.Client; +import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; + +public class ThreatIntelMonitorRunner { + + Client client; + SATIFSourceConfigService service; + +} diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsIntegTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsIntegTestCase.java new file mode 100644 index 000000000..a85196663 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsIntegTestCase.java @@ -0,0 +1,23 @@ +package org.opensearch.securityanalytics; + +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; + +public class SecurityAnalyticsIntegTestCase extends OpenSearchIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + SecurityAnalyticsPlugin.class + ); + } + + @Override + protected boolean ignoreExternalCluster() { + return true; + } + +} diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 9c1e659bf..f5bcd77bf 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -6,19 +6,19 @@ import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import org.apache.lucene.tests.util.LuceneTestCase; -import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.commons.alerting.model.action.Throttle; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.securityanalytics.model.CorrelationQuery; @@ -28,10 +28,10 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.IoCMatch; import org.opensearch.securityanalytics.model.IocDao; import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; +import org.opensearch.securityanalytics.model.threatintel.IocMatch; import org.opensearch.securityanalytics.threatIntel.common.FeedType; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; @@ -42,9 +42,7 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -809,7 +807,7 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } - public static String toJsonString(IoCMatch iocMatch) throws IOException { + public static String toJsonString(IocMatch iocMatch) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); return BytesReference.bytes(builder).utf8ToString(); diff --git a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocMatchTests.java similarity index 89% rename from src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java rename to src/test/java/org/opensearch/securityanalytics/model/IocMatchTests.java index 4b56c7eb5..298d3f7e4 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/IocMatchTests.java @@ -5,6 +5,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.model.threatintel.IocMatch; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -13,15 +14,15 @@ import static org.opensearch.securityanalytics.TestHelpers.toJsonString; -public class IoCMatchTests extends OpenSearchTestCase { +public class IocMatchTests extends OpenSearchTestCase { public void testIoCMatchAsAStream() throws IOException { - IoCMatch iocMatch = getRandomIoCMatch(); + IocMatch iocMatch = getRandomIoCMatch(); String jsonString = toJsonString(iocMatch); BytesStreamOutput out = new BytesStreamOutput(); iocMatch.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); + IocMatch newIocMatch = new IocMatch(sin); assertEquals(iocMatch.getId(), newIocMatch.getId()); assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); @@ -38,11 +39,11 @@ public void testIoCMatchParse() throws IOException { "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; - IoCMatch iocMatch = IoCMatch.parse((getParser(iocMatchString))); + IocMatch iocMatch = IocMatch.parse((getParser(iocMatchString))); BytesStreamOutput out = new BytesStreamOutput(); iocMatch.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); - IoCMatch newIocMatch = new IoCMatch(sin); + IocMatch newIocMatch = new IocMatch(sin); assertEquals(iocMatch.getId(), newIocMatch.getId()); assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); @@ -61,8 +62,8 @@ public XContentParser getParser(String xc) throws IOException { } - private static IoCMatch getRandomIoCMatch() { - return new IoCMatch( + private static IocMatch getRandomIoCMatch() { + return new IocMatch( randomAlphaOfLength(10), List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchServiceIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchServiceIT.java new file mode 100644 index 000000000..fc24b5b76 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/IocMatchServiceIT.java @@ -0,0 +1,67 @@ +package org.opensearch.securityanalytics.threatIntel.iocscan.dao; + +import org.opensearch.action.LatchedActionListener; +import org.opensearch.action.StepListener; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.util.concurrent.CountDown; +import org.opensearch.core.action.ActionListener; +import org.opensearch.securityanalytics.SecurityAnalyticsIntegTestCase; +import org.opensearch.securityanalytics.model.threatintel.IocMatch; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; + + +public class IocMatchServiceIT extends SecurityAnalyticsIntegTestCase { + + public void test_indexIocMatches() throws InterruptedException { + IocMatchService service = new IocMatchService(client(), clusterService()); + List iocMatches = generateIocMatches(10); + CountDown countdown = new CountDown(1); + service.indexIocMatches(iocMatches, ActionListener.wrap(r -> { + countdown.countDown(); + }, e -> { + logger.error("failed to index ioc matches", e); + fail(); + countdown.countDown(); + })); + SearchRequest request = new SearchRequest(IocMatchService.INDEX_NAME); + request.source().size(10); + CountDown countDownLatch1 = new CountDown(1); + client().search(request, ActionListener.wrap( + response -> { + assertEquals(response.getHits().getHits().length, 10); + countDownLatch1.countDown(); + }, + e -> { + logger.error("failed to search indexed ioc matches", e); + fail(); + countDownLatch1.countDown(); + } + + )); + countDownLatch1.isCountedDown(); + } + + private List generateIocMatches(int i) { + List iocMatches = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i1 = 0; i1 < i; i1++) { + iocMatches.add(new IocMatch( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docids + randomList(1, 10, () -> randomAlphaOfLength(10)), //feedids + monitorId, + monitorName, + randomAlphaOfLength(10), + "IP", + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocMatches; + } +} \ No newline at end of file