Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate threat intel feeds #669

Merged
merged 39 commits into from
Oct 25, 2023
Merged
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
12dd633
add mapping for indices storing threat intel feed data
eirsep Oct 2, 2023
a0fd6bd
fix feed indices mapping
eirsep Oct 2, 2023
d7ee565
add threat intel feed data dao
eirsep Oct 3, 2023
57c0faa
add threatIntelEnabled field in detector.
eirsep Oct 3, 2023
f0a8bed
add threat intel feed service and searching feeds
eirsep Oct 3, 2023
4594926
ti feed data to doc level query convertor logic added
eirsep Oct 3, 2023
f7ff940
plug threat intel feed into detector creation
eirsep Oct 4, 2023
c193773
Preliminary framework for jobscheduler and datasource (#626)
jowg-amazon Oct 5, 2023
2e52a02
create doc level query from threat intel feed data index docs"
eirsep Oct 7, 2023
1830ec9
handle threat intel enabled check during detector updation
eirsep Oct 9, 2023
dababa8
add tests for testing threat intel feed integration with detectors
eirsep Oct 10, 2023
5c733ed
Threat intel feeds job runner and unit tests (#654)
jowg-amazon Oct 10, 2023
4512308
converge job scheduler code with threat intel feed integration in det…
eirsep Oct 11, 2023
d1d7ca0
converge job scheduler and detector threat intel code
eirsep Oct 11, 2023
d53085b
add feed metadata config files in src and test
eirsep Oct 12, 2023
98bbd42
adds ioc fields list in log type config files and ioc fields object i…
eirsep Oct 12, 2023
decee51
fix compilation issues in tests
eirsep Oct 16, 2023
a79b8ac
test udpate detector disabling threat intel
eirsep Oct 17, 2023
2b59191
add tests for detector creation and updation with threat intel
eirsep Oct 17, 2023
f0f8270
Threat intel test (#673)
eirsep Oct 17, 2023
0bdd58b
fix threat intel integ tests and add update detector logic
eirsep Oct 17, 2023
0e89286
JS for Threat intel feeds - changed extension (#675)
jowg-amazon Oct 19, 2023
730b458
TIF Job Runner Cleanup (#676)
jowg-amazon Oct 20, 2023
7a24bd0
fix TIFJobParameter class
eirsep Oct 20, 2023
74a7440
test detector updation when feed updation job runs
eirsep Oct 20, 2023
4dd4190
removed delete job scheduler code and cleaned up (#678)
jowg-amazon Oct 20, 2023
a5136bf
working integ test (#680)
jowg-amazon Oct 20, 2023
24a7e51
fix timeout of tif job creation
eirsep Oct 21, 2023
31eebcb
remove unncessary thread forking in put tif job action
eirsep Oct 22, 2023
ae084e7
refactoring code to address review comments
eirsep Oct 22, 2023
28f3ba8
detector trigger detection types
eirsep Oct 20, 2023
2481466
pull out threat intel rest tests into separate test class
eirsep Oct 22, 2023
f2068f1
add detection types testing in detector trigger for rules and threat …
eirsep Oct 22, 2023
7225ee6
add license header
eirsep Oct 23, 2023
ae666de
add threat intel field aliases in mapping view response
eirsep Oct 23, 2023
040556c
fix threat intel feed parser
eirsep Oct 25, 2023
e850248
fix workflow failing test
eirsep Oct 25, 2023
2e88825
spotless check failures fixed
eirsep Oct 25, 2023
bed73ff
remove dockerfile (#689)
jowg-amazon Oct 25, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add feed metadata config files in src and test
Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
eirsep committed Oct 16, 2023
commit d53085b0cc34ee20e27cff9a55ca9a4b3bdbbd0c
Original file line number Diff line number Diff line change
@@ -54,6 +54,7 @@
import org.opensearch.securityanalytics.threatIntel.action.*;
import org.opensearch.securityanalytics.threatIntel.common.TIFExecutor;
import org.opensearch.securityanalytics.threatIntel.common.TIFLockService;
import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService;
@@ -141,6 +142,7 @@ public Collection<Object> createComponents(Client client,
Supplier<RepositoriesService> repositoriesServiceSupplier) {

builtinLogTypeLoader = new BuiltinLogTypeLoader();
BuiltInTIFMetadataLoader builtInTIFMetadataLoader = new BuiltInTIFMetadataLoader();
logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader);
detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
ruleTopicIndices = new RuleTopicIndices(client, clusterService, logTypeService);
@@ -153,7 +155,7 @@ public Collection<Object> createComponents(Client client,
ThreatIntelFeedDataService threatIntelFeedDataService = new ThreatIntelFeedDataService(clusterService, client, indexNameExpressionResolver, xContentRegistry);
DetectorThreatIntelService detectorThreatIntelService = new DetectorThreatIntelService(threatIntelFeedDataService);
TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService);
TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService);
TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader);
TIFExecutor threatIntelExecutor = new TIFExecutor(threadPool);
TIFLockService threatIntelLockService = new TIFLockService(clusterService, client);

@@ -163,9 +165,8 @@ public Collection<Object> createComponents(Client client,

return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices,
mapperService, indexTemplateManager, builtinLogTypeLoader, threatIntelFeedDataService, detectorThreatIntelService,
tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService
);
mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService,
tifJobUpdateService, tifJobParameterService, threatIntelExecutor, threatIntelLockService);
}

@Override
Original file line number Diff line number Diff line change
@@ -15,7 +15,6 @@
import org.opensearch.action.support.WriteRequest;
import org.opensearch.action.support.master.AcknowledgedResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.settings.ClusterSettings;
@@ -26,8 +25,6 @@
import org.opensearch.core.xcontent.NamedXContentRegistry;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.findings.FindingsService;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest;
@@ -112,7 +109,7 @@ public void getThreatIntelFeedData(
);

SearchRequest searchRequest = new SearchRequest(tifdIndex);
searchRequest.source().size(9999); //TODO: convert to scroll
searchRequest.source().size(1000); //TODO: convert to scroll
client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(ThreatIntelFeedDataUtils.getTifdList(r, xContentRegistry)), e -> {
log.error(String.format(
"Failed to fetch threat intel feed data from system index %s", tifdIndex), e);
@@ -191,7 +188,7 @@ public void parseAndSaveThreatIntelFeedDataCSV(
List<ThreatIntelFeedData> tifdList = new ArrayList<>();
while (iterator.hasNext()) {
CSVRecord record = iterator.next();
String iocType = tifMetadata.getContainedIocs().get(0); //todo make generic in upcoming versions
String iocType = tifMetadata.getIocType(); //todo make generic in upcoming versions
Integer colNum = tifMetadata.getIocCol();
String iocValue = record.values()[colNum].split(" ")[0];
String feedId = tifMetadata.getFeedId();
@@ -217,7 +214,10 @@ public void parseAndSaveThreatIntelFeedDataCSV(

public void saveTifds(BulkRequest bulkRequest, TimeValue timeout) {

BulkResponse response = StashedThreadContext.run(client, () -> client.bulk(bulkRequest).actionGet(timeout));
try {
BulkResponse response = StashedThreadContext.run(client, () -> {
return client.bulk(bulkRequest).actionGet(timeout);
});
if (response.hasFailures()) {
throw new OpenSearchException(
"error occurred while ingesting threat intel feed data in {} with an error {}",
@@ -226,6 +226,9 @@ public void saveTifds(BulkRequest bulkRequest, TimeValue timeout) {
);
}
bulkRequest.requests().clear();
} catch (OpenSearchException e) {
log.error("failed to save threat intel feed data", e);
}

}

Loading