Skip to content

Commit

Permalink
add search ioc findings api
Browse files Browse the repository at this point in the history
Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
sbcd90 authored and eirsep committed Jun 20, 2024
1 parent ec14132 commit 3015ee1
Show file tree
Hide file tree
Showing 15 changed files with 586 additions and 76 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@
import org.opensearch.securityanalytics.resthandler.RestValidateRulesAction;
import org.opensearch.securityanalytics.services.STIX2IOCFetchService;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction;
Expand All @@ -132,6 +133,7 @@
import org.opensearch.securityanalytics.threatIntel.model.monitor.ThreatIntelMonitorRunner;
import org.opensearch.securityanalytics.threatIntel.model.monitor.TransportThreatIntelMonitorFanOutAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestDeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.resthandler.RestSearchTIFSourceConfigsAction;
Expand All @@ -145,6 +147,7 @@
import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService;
import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService;
import org.opensearch.securityanalytics.threatIntel.transport.TransportDeleteTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetIocFindingsAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportIndexTIFSourceConfigAction;
import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction;
Expand Down Expand Up @@ -342,7 +345,8 @@ public List<RestHandler> getRestHandlers(Settings settings,
new RestIndexThreatIntelMonitorAction(),
new RestDeleteThreatIntelMonitorAction(),
new RestSearchThreatIntelMonitorAction(),
new RestListIOCsAction()
new RestListIOCsAction(),
new RestGetIocFindingsAction()
);
}

Expand Down Expand Up @@ -486,7 +490,8 @@ public List<Setting<?>> getSettings() {
new ActionHandler<>(SADeleteTIFSourceConfigAction.INSTANCE, TransportDeleteTIFSourceConfigAction.class),
new ActionHandler<>(SASearchTIFSourceConfigsAction.INSTANCE, TransportSearchTIFSourceConfigsAction.class),
new ActionHandler<>(ThreatIntelMonitorRunner.REMOTE_DOC_LEVEL_MONITOR_ACTION_INSTANCE, TransportThreatIntelMonitorFanOutAction.class),
new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class)
new ActionHandler<>(ListIOCsAction.INSTANCE, TransportListIOCsAction.class),
new ActionHandler<>(GetIocFindingsAction.INSTANCE, TransportGetIocFindingsAction.class)
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
* IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job
* It's the inverse of an IoC finding which maps a document to list of IoC's
*/
public class IocMatch implements Writeable, ToXContent {
public class IocFinding implements Writeable, ToXContent {
//TODO implement IoC_Match interface from security-analytics-commons
public static final String ID_FIELD = "id";
public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids";
Expand All @@ -42,8 +42,8 @@ public class IocMatch implements Writeable, ToXContent {
private final Instant timestamp;
private final String executionId;

public IocMatch(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
public IocFinding(String id, List<String> relatedDocIds, List<String> feedIds, String iocScanJobId,
String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) {
validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds);
this.id = id;
this.relatedDocIds = relatedDocIds;
Expand All @@ -56,7 +56,7 @@ public IocMatch(String id, List<String> relatedDocIds, List<String> feedIds, Str
this.executionId = executionId;
}

public IocMatch(StreamInput in) throws IOException {
public IocFinding(StreamInput in) throws IOException {
id = in.readString();
relatedDocIds = in.readStringList();
feedIds = in.readStringList();
Expand Down Expand Up @@ -133,7 +133,7 @@ public String getExecutionId() {
return executionId;
}

public static IocMatch parse(XContentParser xcp) throws IOException {
public static IocFinding parse(XContentParser xcp) throws IOException {
String id = null;
List<String> relatedDocIds = new ArrayList<>();
List<String> feedIds = new ArrayList<>();
Expand Down Expand Up @@ -197,11 +197,11 @@ public static IocMatch parse(XContentParser xcp) throws IOException {
}
}

return new IocMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
return new IocFinding(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId);
}

public static IocMatch readFrom(StreamInput in) throws IOException {
return new IocMatch(in);
public static IocFinding readFrom(StreamInput in) throws IOException {
return new IocFinding(in);
}


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionType;

public class GetIocFindingsAction extends ActionType<GetIocFindingsResponse> {

public static final GetIocFindingsAction INSTANCE = new GetIocFindingsAction();
public static final String NAME = "cluster:admin/opensearch/securityanalytics/ioc/findings/get";

public GetIocFindingsAction() {
super(NAME, GetIocFindingsResponse::new);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionRequestValidationException;
import org.opensearch.action.ValidateActions;
import org.opensearch.commons.alerting.model.Table;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;

import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Locale;

public class GetIocFindingsRequest extends ActionRequest {

private List<String> findingIds;

private Instant startTime;

private Instant endTime;

private Table table;

public GetIocFindingsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalStringList(),
sin.readOptionalInstant(),
sin.readOptionalInstant(),
Table.readFrom(sin)
);
}

public GetIocFindingsRequest(List<String> findingIds,
Instant startTime,
Instant endTime,
Table table) {
this.findingIds = findingIds;
this.startTime = startTime;
this.endTime = endTime;
this.table = table;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (startTime != null && endTime != null && startTime.isAfter(endTime)) {
validationException = ValidateActions.addValidationError(String.format(Locale.getDefault(),
"startTime should be less than endTime"), validationException);
}
return validationException;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalStringCollection(findingIds);
out.writeOptionalInstant(startTime);
out.writeOptionalInstant(endTime);
table.writeTo(out);
}

public List<String> getFindingIds() {
return findingIds;
}

public Instant getStartTime() {
return startTime;
}

public Instant getEndTime() {
return endTime;
}

public Table getTable() {
return table;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/
package org.opensearch.securityanalytics.threatIntel.action;

import org.opensearch.core.action.ActionResponse;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentObject;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.model.threatintel.IocFinding;

import java.io.IOException;
import java.util.Collections;
import java.util.List;

public class GetIocFindingsResponse extends ActionResponse implements ToXContentObject {

private static final String TOTAL_IOC_FINDINGS_FIELD = "total_findings";

private static final String IOC_FINDINGS_FIELD = "ioc_findings";

private Integer totalFindings;

private List<IocFinding> iocFindings;

public GetIocFindingsResponse(Integer totalFindings, List<IocFinding> iocFindings) {
super();
this.totalFindings = totalFindings;
this.iocFindings = iocFindings;
}

public GetIocFindingsResponse(StreamInput sin) throws IOException {
this(
sin.readInt(),
Collections.unmodifiableList(sin.readList(IocFinding::new))
);
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(totalFindings);
out.writeCollection(iocFindings);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject()
.field(TOTAL_IOC_FINDINGS_FIELD, totalFindings)
.field(IOC_FINDINGS_FIELD, iocFindings);
return builder.endObject();
}

public Integer getTotalFindings() {
return totalFindings;
}

public List<IocFinding> getIocFindings() {
return iocFindings;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,31 @@
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
import org.opensearch.action.bulk.BulkRequest;
import org.opensearch.action.bulk.BulkResponse;
import org.opensearch.action.get.MultiGetRequest;
import org.opensearch.action.index.IndexRequest;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.action.support.GroupedActionListener;
import org.opensearch.action.support.WriteRequest;
import org.opensearch.client.Client;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.core.xcontent.XContentParserUtils;
import org.opensearch.index.IndexNotFoundException;
import org.opensearch.search.SearchHit;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.SecurityAnalyticsPlugin;
import org.opensearch.securityanalytics.model.threatintel.IocMatch;
import org.opensearch.securityanalytics.model.threatintel.IocFinding;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
import org.opensearch.threadpool.ThreadPool;

import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -35,36 +45,39 @@
/**
* Data layer to perform CRUD operations for threat intel ioc match : store in system index.
*/
public class IocMatchService {
public class IocFindingService {
//TODO manage index rollover
public static final String INDEX_NAME = ".opensearch-sap-iocmatch";
private static final Logger log = LogManager.getLogger(IocMatchService.class);
private static final Logger log = LogManager.getLogger(IocFindingService.class);
private final Client client;
private final ClusterService clusterService;

public IocMatchService(final Client client, final ClusterService clusterService) {
private final NamedXContentRegistry xContentRegistry;

public IocFindingService(final Client client, final ClusterService clusterService, final NamedXContentRegistry xContentRegistry) {
this.client = client;
this.clusterService = clusterService;
this.xContentRegistry = xContentRegistry;
}

public void indexIocMatches(List<IocMatch> iocMatches,
public void indexIocMatches(List<IocFinding> iocFindings,
final ActionListener<Void> actionListener) {
try {
Integer batchSize = this.clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE);
createIndexIfNotExists(ActionListener.wrap(
r -> {
List<BulkRequest> bulkRequestList = new ArrayList<>();
BulkRequest bulkRequest = new BulkRequest(INDEX_NAME);
for (int i = 0; i < iocMatches.size(); i++) {
IocMatch iocMatch = iocMatches.get(i);
for (int i = 0; i < iocFindings.size(); i++) {
IocFinding iocFinding = iocFindings.get(i);
try {
IndexRequest indexRequest = new IndexRequest(INDEX_NAME)
.source(iocMatch.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
.source(iocFinding.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
.opType(DocWriteRequest.OpType.CREATE);
bulkRequest.add(indexRequest);
if (
bulkRequest.requests().size() == batchSize
&& i != iocMatches.size() - 1 // final bulk request will be added outside for loop with refresh policy none
&& i != iocFindings.size() - 1 // final bulk request will be added outside for loop with refresh policy none
) {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
bulkRequestList.add(bulkRequest);
Expand Down Expand Up @@ -107,7 +120,7 @@ public void indexIocMatches(List<IocMatch> iocMatches,

private String getIndexMapping() {
try {
try (InputStream is = IocMatchService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) {
try (InputStream is = IocFindingService.class.getResourceAsStream("/mappings/ioc_match_mapping.json")) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
return reader.lines().map(String::trim).collect(Collectors.joining());
}
Expand Down Expand Up @@ -152,4 +165,40 @@ public void createIndexIfNotExists(final ActionListener<Void> listener) {
listener.onFailure(e);
}
}

public void searchIocMatches(SearchSourceBuilder searchSourceBuilder, final ActionListener<GetIocFindingsResponse> actionListener) {
SearchRequest searchRequest = new SearchRequest()
.source(searchSourceBuilder)
.indices(INDEX_NAME);

client.search(searchRequest, new ActionListener<>() {
@Override
public void onResponse(SearchResponse searchResponse) {
try {
long totalIocFindingsCount = searchResponse.getHits().getTotalHits().value;
List<IocFinding> iocFindings = new ArrayList<>();

for (SearchHit hit: searchResponse.getHits()) {
XContentParser xcp = XContentType.JSON.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, hit.getSourceAsString());
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp);
IocFinding iocFinding = IocFinding.parse(xcp);
iocFindings.add(iocFinding);
}
actionListener.onResponse(new GetIocFindingsResponse((int) totalIocFindingsCount, iocFindings));
} catch (Exception ex) {
this.onFailure(ex);
}
}

@Override
public void onFailure(Exception e) {
if (e instanceof IndexNotFoundException) {
actionListener.onResponse(new GetIocFindingsResponse(0, List.of()));
return;
}
actionListener.onFailure(e);
}
});
}
}
Loading

0 comments on commit 3015ee1

Please sign in to comment.