diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4de766ed5..0b6f0f962 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,6 @@ jobs: DD_API_KEY: "11111111111111111111111111111111" DD_ADDITIONAL_TARGET_LAMBDAS: "ironmaiden,megadeth" run: | - pip install boto3 mock + pip install boto3 mock approvaltests python -m unittest discover ./aws/logs_monitoring/ python -m unittest discover ./aws/rds_enhanced_monitoring/ diff --git a/aws/logs_monitoring/__init__.py b/aws/logs_monitoring/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aws/logs_monitoring/tests/__init__.py b/aws/logs_monitoring/tests/__init__.py index e69de29bb..bc84ea5ec 100644 --- a/aws/logs_monitoring/tests/__init__.py +++ b/aws/logs_monitoring/tests/__init__.py @@ -0,0 +1,13 @@ +from approvaltests.approvals import set_default_reporter +from approvaltests.reporters import GenericDiffReporter, GenericDiffReporterConfig + + +set_default_reporter( + GenericDiffReporter( + GenericDiffReporterConfig( + "VSCODE", + "/Applications/Visual Studio Code.app/contents/Resources/app/bin/code", + ["-d"], + ) + ) +) diff --git a/aws/logs_monitoring/tests/approvaltests_config.json b/aws/logs_monitoring/tests/approvaltests_config.json new file mode 100644 index 000000000..d62c54b49 --- /dev/null +++ b/aws/logs_monitoring/tests/approvaltests_config.json @@ -0,0 +1,3 @@ +{ + "subdirectory": "approved_files" +} diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_rds_postgresql.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_rds_postgresql.approved.json new file mode 100644 index 000000000..9e90266f0 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_rds_postgresql.approved.json @@ -0,0 +1,14 @@ +[ + { + "aws": { + "awslogs": { + "logGroup": "/aws/rds/instance/datadog/postgresql", + "logStream": "datadog.0", + "owner": "123456789012" + } + }, + "id": "31953106606966983378809025079804211143289615424298221568", + "message": "2021-01-02 03:04:05 UTC::@:[5306]:LOG: database system is ready to accept connections", + "timestamp": 1609556645000 + } +] diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_rds_postgresql.metadata.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_rds_postgresql.metadata.approved.json new file mode 100644 index 000000000..74bb23e02 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_rds_postgresql.metadata.approved.json @@ -0,0 +1,6 @@ +{ + "ddsource": "postgresql", + "ddtags": "env:dev,test_tag_key:test_tag_value,logname:postgresql", + "host": "datadog", + "service": "postgresql" +} diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json new file mode 100644 index 000000000..3084e87e3 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json @@ -0,0 +1,14 @@ +[ + { + "aws": { + "awslogs": { + "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", + "owner": "425362996713" + } + }, + "id": "37199773595581154154810589279545129148442535997644275712", + "message": "{\"id\":\"1\",\"type\":\"ExecutionStarted\",\"details\":{\"input\":\"{\"Comment\": \"Insert your JSON here\"}\",\"inputDetails\":{\"truncated\":false},\"roleArn\":\"arn:aws:iam::425362996713:role/service-role/StepFunctions-logs-to-traces-sequential-role-ccd69c03\"},\",previous_event_id\":\"0\",\"event_timestamp\":\"1668095539607\",\"execution_arn\":\"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:d0dbefd8-a0f6-b402-da4c-f4863def7456:7fa0cfbe-be28-4a20-9875-73c37f5dc39e\"}", + "timestamp": 1668095539607 + } +] diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json new file mode 100644 index 000000000..cc1a1de5a --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json @@ -0,0 +1,6 @@ +{ + "ddsource": "stepfunction", + "ddtags": "env:dev,test_tag_key:test_tag_value", + "host": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", + "service": "stepfunction" +} diff --git a/aws/logs_monitoring/tests/approved_files/TestLambdaFunctionEndToEnd.test_datadog_forwarder.approved.json b/aws/logs_monitoring/tests/approved_files/TestLambdaFunctionEndToEnd.test_datadog_forwarder.approved.json new file mode 100644 index 000000000..bb8b2d686 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestLambdaFunctionEndToEnd.test_datadog_forwarder.approved.json @@ -0,0 +1,420 @@ +[ + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576111948622874033876462979853992919938886093242368", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.113Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Patched console output with trace context\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836114 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576111948622874033876462979853992919938886093242369", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.114Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"autoPatchHTTP\":true,\"tracerInitialized\":true,\"status\":\"debug\",\"message\":\"datadog:Not patching HTTP libraries\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836114 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576111948622874033876462979853992919938886093242370", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.114Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Reading trace context from env var Root=1-5e61292c-cc1229a4dfbeae1043928548;Parent=c657b77d9514f70c;Sampled=1\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836114 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576111948622874033876462979853992919938886093242371", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.114Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"parentID\":\"14292093692483532556\",\"sampleMode\":2,\"source\":\"xray\",\"traceID\":\"6899143064054564168\",\"status\":\"debug\",\"message\":\"datadog:read trace context from environment\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836114 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576111948622874033876462979853992919938886093242372", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "{\"e\":1583425836.114,\"m\":\"aws.lambda.enhanced.invocations\",\"t\":[\"region:us-east-1\",\"account_id:601427279990\",\"functionname:hello-dog-node-dev-hello12x\",\"cold_start:false\",\"memorysize:128\",\"runtime:nodejs12.x\"],\"v\":1}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836114 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576112305434797210366433244425485282312670188929029", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.130Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=6899143064054564168 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:set trace context from xray with parent 14292093692483532556 from segment\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836130 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113197464605151591358905854216188247130428145670", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.131Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=6899143064054564168 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:set trace context from xray with parent 14292093692483532556 from segment\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836170 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113197464605151591358905854216188247130428145671", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.131Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=6899143064054564168 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:set trace context from xray with parent 14292093692483532556 from segment\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836170 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113197464605151591358905854216188247130428145672", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.131Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=6899143064054564168 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:set trace context from xray with parent 14292093692483532556 from segment\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836170 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113197464605151591358905854216188247130428145673", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.131Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=6899143064054564168 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Attempting to find parent for datadog trace trace\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836170 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113197464605151591358905854216188247130428145674", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.131Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=6899143064054564168 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Applying lambda context to datadog traces\"}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836170 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113643479509122203821736568581641214360547753995", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.172Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tINFO\t[dd.trace_id=6899143064054564168 dd.span_id=2927617725217152879] Request Headers undefined\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836190 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113643479509122203821736568581641214360547753996", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.172Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tINFO\t[dd.trace_id=6899143064054564168 dd.span_id=2927617725217152879] Root=1-5e61292c-cc1229a4dfbeae1043928548;Parent=c657b77d9514f70c;Sampled=1\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836190 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576113643479509122203821736568581641214360547753997", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "{\"e\":1583425836.172,\"m\":\"hello.js10x.dog-2\",\"t\":[\"dd_lambda_layer:datadog-nodev12.14.1\"],\"v\":10}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836190 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576123455807396475678004012284621606493423179137038", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.592Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tINFO\t[dd.trace_id=6899143064054564168 dd.span_id=3694123456155101779] 8103.047457805628\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836630 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576123478108141674208627153820339879141784685117455", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "2020-03-05T16:30:36.631Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tINFO\t[dd.trace_id=6899143064054564168 dd.span_id=2927617725217152879] Finishing Span\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836631 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576125685881916328740318165856448871329573777178640", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "{\"traces\": [[{\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"0000000000000000\", \"span_id\": \"A812B5E71D1C5417\", \"service\": \"ialbefmodl.execute-api.sa-east-1.amazonaws.com\", \"resource\": \"ialbefmodl.execute-api.sa-east-1.amazonaws.com/\", \"name\": \"aws.apigateway\", \"error\": 0, \"start\": 1636820292450000128, \"duration\": 149992638, \"meta\": {\"runtime-id\": \"810d8797397b4a8c94ca00582b397222\", \"_dd.origin\": \"lambda\", \"operation_name\": \"aws.apigateway.rest\", \"http.url\": \"ialbefmodl.execute-api.sa-east-1.amazonaws.com/\", \"endpoint\": \"/\", \"http.method\": \"GET\", \"resource_names\": \"ialbefmodl.execute-api.sa-east-1.amazonaws.com/\", \"request_id\": \"9f09f496-83c7-441b-bc59-9741107b0683\", \"inferred_span.inherit_lambda\": \"False\", \"inferred_span.is_async\": \"False\", \"http.status_code\": \"200\"}, \"metrics\": {\"_dd.agent_psr\": 1, \"system.pid\": 9, \"_sampling_priority_v1\": 1}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"A812B5E71D1C5417\", \"span_id\": \"080EE818C637C434\", \"service\": \"aws.lambda\", \"resource\": \"inferred-spans-python-dev-initSender\", \"name\": \"aws.lambda\", \"error\": 0, \"start\": 1636820292466458058, \"duration\": 133507715, \"meta\": {\"_dd.origin\": \"lambda\", \"cold_start\": \"false\", \"function_arn\": \"arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender\", \"function_version\": \"$LATEST\", \"request_id\": \"9f09f496-83c7-441b-bc59-9741107b0683\", \"resource_names\": \"inferred-spans-python-dev-initSender\", \"functionname\": \"inferred-spans-python-dev-initsender\", \"datadog_lambda\": \"3.49.0\", \"dd_trace\": \"0.50.4\", \"span.name\": \"aws.lambda\", \"function_trigger.event_source\": \"api-gateway\", \"function_trigger.event_source_arn\": \"arn:aws:apigateway:sa-east-1::/restapis/ialbefmodl/stages/dev\", \"http.url\": \"ialbefmodl.execute-api.sa-east-1.amazonaws.com\", \"http.url_details.path\": \"/dev/\", \"http.method\": \"GET\", \"http.status_code\": \"200\"}, \"type\": \"serverless\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"1E1CBE25CBABD2AD\", \"service\": \"aws.sqs\", \"resource\": \"sqs.sendmessage\", \"name\": \"sqs.command\", \"error\": 0, \"start\": 1636820292466887097, \"duration\": 19825652, \"meta\": {\"_dd.origin\": \"lambda\", \"params.MessageAttributes._datadog.StringValue\": \"{\\\"x-datadog-trace-id\\\": \\\"11303031032863116131\\\", \\\"x-datadog-parent-id\\\": \\\"2169818190025839277\\\", \\\"x-datadog-sampling-priority\\\": \\\"1\\\"}\", \"params.MessageAttributes._datadog.DataType\": \"String\", \"params.QueueUrl\": \"https://sqs.sa-east-1.amazonaws.com/601427279990/serverlessTracingQueuePy\", \"aws.agent\": \"botocore\", \"aws.operation\": \"SendMessage\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"adc84b9d-0bc0-5ad9-82e0-1194ec44018a\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"2084449425493209\", \"service\": \"aws.sns\", \"resource\": \"sns.publish\", \"name\": \"sns.command\", \"error\": 0, \"start\": 1636820292487211990, \"duration\": 21565856, \"meta\": {\"_dd.origin\": \"lambda\", \"params.MessageAttributes._datadog.StringValue\": \"{\\\"x-datadog-trace-id\\\": \\\"11303031032863116131\\\", \\\"x-datadog-parent-id\\\": \\\"580656595079775284\\\", \\\"x-datadog-sampling-priority\\\": \\\"1\\\"}\", \"params.MessageAttributes._datadog.DataType\": \"String\", \"params.Message\": \"Asynchronously invoking a Lambda function with SNS.\", \"params.TopicArn\": \"arn:aws:sns:sa-east-1:601427279990:serverlessTracingTopicPy\", \"aws.agent\": \"botocore\", \"aws.operation\": \"Publish\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"37ec8df5-bc7c-583e-835c-d7fc5bf87b1b\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"640267352BAD0C2B\", \"service\": \"aws.dynamodb\", \"resource\": \"dynamodb.putitem\", \"name\": \"dynamodb.command\", \"error\": 0, \"start\": 1636820292508904508, \"duration\": 6481144, \"meta\": {\"_dd.origin\": \"lambda\", \"params.Item.email.S\": \"e560387c-da16-4aae-8479-2c9232234ed6\", \"params.TableName\": \"usersTable\", \"aws.agent\": \"botocore\", \"aws.operation\": \"PutItem\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"QHMDJ5RCOUB1LNCAB0VVKEDCHVVV4KQNSO5AEMVJF66Q9ASUAAJG\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"D50A24D92FF59FD4\", \"service\": \"aws.kinesis\", \"resource\": \"kinesis.putrecord\", \"name\": \"kinesis.command\", \"error\": 0, \"start\": 1636820292515486978, \"duration\": 7799031, \"meta\": {\"_dd.origin\": \"lambda\", \"params.PartitionKey\": \"partitionkey\", \"params.Data\": \"{\\\"foo\\\": \\\"bar\\\"}\", \"params.StreamName\": \"kinesisStream\", \"aws.agent\": \"botocore\", \"aws.operation\": \"PutRecord\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"dda3348f-e941-9fe5-87aa-d30b19df6a3e\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"0E4800A960384F8F\", \"service\": \"aws.events\", \"resource\": \"events.putevents\", \"name\": \"events.command\", \"error\": 0, \"start\": 1636820292523387895, \"duration\": 13733007, \"meta\": {\"_dd.origin\": \"lambda\", \"params.Entries\": \"[{'Source': 'eventbridge.custom.event.sender', 'DetailType': 'testdetail', 'Detail': '{\\\"foo\\\": \\\"bar\\\"}', 'EventBusName': 'inferredBus'}]\", \"aws.agent\": \"botocore\", \"aws.operation\": \"PutEvents\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"d7545887-c739-4e35-ab65-c6504e714d07\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"154151ED08C6AD19\", \"service\": \"aws.s3\", \"resource\": \"s3.putobject\", \"name\": \"s3.command\", \"error\": 0, \"start\": 1636820292538263155, \"duration\": 38689743, \"meta\": {\"_dd.origin\": \"lambda\", \"params.Key\": \"76909630-1599-4fcc-ab10-5425637e1bee\", \"params.Bucket\": \"inferred-spans-python-bucket\", \"aws.agent\": \"botocore\", \"aws.operation\": \"PutObject\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"0Z7JMRYXA4WXJP0M\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}, {\"trace_id\": \"9CDC6ED69F05CB63\", \"parent_id\": \"080EE818C637C434\", \"span_id\": \"70BB81DD6895FD93\", \"service\": \"aws.lambda\", \"resource\": \"lambda.invoke\", \"name\": \"lambda.command\", \"error\": 0, \"start\": 1636820292579315022, \"duration\": 20111883, \"meta\": {\"_dd.origin\": \"lambda\", \"params.ClientContext\": \"eyJjdXN0b20iOiB7Il9kYXRhZG9nIjogeyJ4LWRhdGFkb2ctdHJhY2UtaWQiOiAiMTEzMDMwMzEwMzI4NjMxMTYxMzEiLCAieC1kYXRhZG9nLXBhcmVudC1pZCI6ICI4MTIzMjI5MTQwODM1MjM3MjY3IiwgIngtZGF0YWRvZy1zYW1wbGluZy1wcmlvcml0eSI6ICIxIn19fQ==\", \"params.FunctionName\": \"inferred-spans-python-dev-directInvokeReceiver\", \"aws.agent\": \"botocore\", \"aws.operation\": \"Invoke\", \"aws.region\": \"sa-east-1\", \"http.status_code\": \"200\", \"aws.requestid\": \"0e6344ef-befc-4066-b755-f2c902fc4f7f\"}, \"metrics\": {\"_dd.measured\": 1, \"retry_attempts\": 0}, \"type\": \"http\"}]]}\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836730 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576126131896820299352780996570814324296803896786961", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "END RequestId: f08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836750 + }, + { + "aws": { + "awslogs": { + "logGroup": "/aws/lambda/hello-dog-node-dev-hello12x", + "logStream": "2020/03/05/[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0", + "owner": "601427279990" + }, + "function_version": 0, + "invoked_function_arn": "arn:aws:lambda:sa-east-1:601427279990:function:inferred-spans-python-dev-initsender" + }, + "ddsource": "lambda", + "ddsourcecategory": "aws", + "ddtags": "forwardername:inferred-spans-python-dev-initsender,forwarder_memorysize:10,forwarder_version:3.95.0,env:none,account_id:601427279990,aws_account:601427279990,functionname:hello-dog-node-dev-hello12x,region:sa-east-1,service:hello-dog-node-dev-hello12x", + "host": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x", + "id": "35311576126131896820299352780996570814324296803896786962", + "lambda": { + "arn": "arn:aws:lambda:sa-east-1:601427279990:function:hello-dog-node-dev-hello12x" + }, + "message": "REPORT RequestId: f08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDuration: 619.31 ms\tBilled Duration: 700 ms\tMemory Size: 128 MB\tMax Memory Used: 118 MB\t\nXRAY TraceId: 1-5e61292c-cc1229a4dfbeae1043928548\tSegmentId: 07a85e713f6302b2\tSampled: true\t\n", + "service": "hello-dog-node-dev-hello12x", + "timestamp": 1583425836750 + } +] diff --git a/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_headers.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_headers.approved.json new file mode 100644 index 000000000..8a34e8231 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_headers.approved.json @@ -0,0 +1,11 @@ +{ + "ddsource": "waf", + "message": { + "httpRequest": { + "headers": { + "header1": "value1", + "header2": "value2" + } + } + } +} diff --git a/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_non_terminating_matching_rules.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_non_terminating_matching_rules.approved.json new file mode 100644 index 000000000..a1d88d897 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_non_terminating_matching_rules.approved.json @@ -0,0 +1,13 @@ +{ + "ddsource": "waf", + "message": { + "nonTerminatingMatchingRules": { + "nonterminating1": { + "action": "COUNT" + }, + "nonterminating2": { + "action": "COUNT" + } + } + } +} diff --git a/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rate_based_rules.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rate_based_rules.approved.json new file mode 100644 index 000000000..ae5094c51 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rate_based_rules.approved.json @@ -0,0 +1,19 @@ +{ + "ddsource": "waf", + "message": { + "rateBasedRuleList": { + "no-rate-limit": { + "limitKey": "IP", + "limitValue": "195.154.122.189", + "maxRateAllowed": 300, + "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889" + }, + "tf-rate-limit-5-min": { + "limitKey": "IP", + "limitValue": "195.154.122.189", + "maxRateAllowed": 300, + "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889" + } + } + } +} diff --git a/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_three_rules_two_group_ids.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_three_rules_two_group_ids.approved.json new file mode 100644 index 000000000..f037eaaec --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_three_rules_two_group_ids.approved.json @@ -0,0 +1,24 @@ +{ + "ddsource": "waf", + "message": { + "ruleGroupList": { + "AWS#AWSManagedRulesSQLiRuleSet": { + "terminatingRule": { + "SQLi_QUERYARGUMENTS": { + "action": "BLOCK" + }, + "secondRULE": { + "action": "BLOCK" + } + } + }, + "A_DIFFERENT_ID": { + "terminatingRule": { + "thirdRULE": { + "action": "BLOCK" + } + } + } + } + } +} diff --git a/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_two_rules_same_group_id.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_two_rules_same_group_id.approved.json new file mode 100644 index 000000000..651ad74e8 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_two_rules_same_group_id.approved.json @@ -0,0 +1,17 @@ +{ + "ddsource": "waf", + "message": { + "ruleGroupList": { + "AWS#AWSManagedRulesSQLiRuleSet": { + "terminatingRule": { + "SQLi_QUERYARGUMENTS": { + "action": "BLOCK" + }, + "secondRULE": { + "action": "BLOCK" + } + } + } + } + } +} diff --git a/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_with_excluded_and_nonterminating_rules.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_with_excluded_and_nonterminating_rules.approved.json new file mode 100644 index 000000000..58e3e8cc4 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseAwsWafLogs.test_waf_rule_group_with_excluded_and_nonterminating_rules.approved.json @@ -0,0 +1,30 @@ +{ + "ddsource": "waf", + "message": { + "ruleGroupList": { + "AWS#AWSManagedRulesSQLiRuleSet": { + "excludedRules": { + "GenericRFI_BODY": { + "exclusionType": "EXCLUDED_AS_COUNT" + }, + "second_exclude": { + "exclusionType": "EXCLUDED_AS_COUNT" + } + }, + "nonTerminatingMatchingRules": { + "first_nonterminating": { + "exclusionType": "REGULAR" + }, + "second_nonterminating": { + "exclusionType": "REGULAR" + } + }, + "terminatingRule": { + "SQLi_QUERYARGUMENTS": { + "action": "BLOCK" + } + } + } + } + } +} diff --git a/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_multiple_findings_multiple_resources.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_multiple_findings_multiple_resources.approved.json new file mode 100644 index 000000000..3cccb36f3 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_multiple_findings_multiple_resources.approved.json @@ -0,0 +1,50 @@ +[ + { + "ddsource": "securityhub", + "detail": { + "finding": { + "myattribute": "somevalue", + "resources": { + "AwsEc2SecurityGroup": { + "Region": "us-east-1" + } + } + } + } + }, + { + "ddsource": "securityhub", + "detail": { + "finding": { + "myattribute": "somevalue", + "resources": { + "AwsEc2SecurityGroup": { + "Region": "us-east-1" + }, + "AwsOtherSecurityGroup": { + "Region": "us-east-1" + } + } + } + } + }, + { + "ddsource": "securityhub", + "detail": { + "finding": { + "myattribute": "somevalue", + "resources": { + "AwsAnotherSecurityGroup": { + "Region": "us-east-1" + }, + "AwsEc2SecurityGroup": { + "Region": "us-east-1" + }, + "AwsOtherSecurityGroup": { + "Region": "us-east-1" + } + } + } + } + } +] diff --git a/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_one_finding_no_resources.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_one_finding_no_resources.approved.json new file mode 100644 index 000000000..ab76d34f7 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_one_finding_no_resources.approved.json @@ -0,0 +1,11 @@ +[ + { + "ddsource": "securityhub", + "detail": { + "finding": { + "myattribute": "somevalue", + "resources": {} + } + } + } +] diff --git a/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_two_findings_one_resource_each.approved.json b/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_two_findings_one_resource_each.approved.json new file mode 100644 index 000000000..792bd0fb1 --- /dev/null +++ b/aws/logs_monitoring/tests/approved_files/TestParseSecurityHubEvents.test_security_hub_two_findings_one_resource_each.approved.json @@ -0,0 +1,28 @@ +[ + { + "ddsource": "securityhub", + "detail": { + "finding": { + "myattribute": "somevalue", + "resources": { + "AwsEc2SecurityGroup": { + "Region": "us-east-1" + } + } + } + } + }, + { + "ddsource": "securityhub", + "detail": { + "finding": { + "myattribute": "somevalue", + "resources": { + "AwsEc2SecurityGroup": { + "Region": "us-east-1" + } + } + } + } + } +] diff --git a/aws/logs_monitoring/tests/test_lambda_function.py b/aws/logs_monitoring/tests/test_lambda_function.py index d863bea77..086df3299 100644 --- a/aws/logs_monitoring/tests/test_lambda_function.py +++ b/aws/logs_monitoring/tests/test_lambda_function.py @@ -7,6 +7,7 @@ import base64 from time import time from botocore.exceptions import ClientError +from approvaltests.approvals import verify_as_json sys.modules["trace_forwarder.connection"] = MagicMock() sys.modules["datadog_lambda.wrapper"] = MagicMock() @@ -167,6 +168,8 @@ def test_datadog_forwarder( enriched_events = enrich(normalized_events) transformed_events = transform(enriched_events) + verify_as_json(transformed_events) + metrics, logs, trace_payloads = split(transformed_events) self.assertEqual(len(trace_payloads), 1) diff --git a/aws/logs_monitoring/tests/test_parsing.py b/aws/logs_monitoring/tests/test_parsing.py index cf8bef0db..84f85d159 100644 --- a/aws/logs_monitoring/tests/test_parsing.py +++ b/aws/logs_monitoring/tests/test_parsing.py @@ -5,6 +5,8 @@ import os import sys import unittest +from approvaltests.approvals import verify_as_json +from approvaltests.namer import NamerFactory sys.modules["trace_forwarder.connection"] = MagicMock() sys.modules["datadog_lambda.wrapper"] = MagicMock() @@ -417,17 +419,7 @@ def test_waf_headers(self): } }, } - self.assertEqual( - parse_aws_waf_logs(event), - { - "ddsource": "waf", - "message": { - "httpRequest": { - "headers": {"header1": "value1", "header2": "value2"} - } - }, - }, - ) + verify_as_json(parse_aws_waf_logs(event)) def test_waf_non_terminating_matching_rules(self): event = { @@ -439,18 +431,7 @@ def test_waf_non_terminating_matching_rules(self): ] }, } - self.assertEqual( - parse_aws_waf_logs(event), - { - "ddsource": "waf", - "message": { - "nonTerminatingMatchingRules": { - "nonterminating2": {"action": "COUNT"}, - "nonterminating1": {"action": "COUNT"}, - } - }, - }, - ) + verify_as_json(parse_aws_waf_logs(event)) def test_waf_rate_based_rules(self): event = { @@ -474,28 +455,7 @@ def test_waf_rate_based_rules(self): ] }, } - self.assertEqual( - parse_aws_waf_logs(event), - { - "ddsource": "waf", - "message": { - "rateBasedRuleList": { - "tf-rate-limit-5-min": { - "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889", - "limitValue": "195.154.122.189", - "maxRateAllowed": 300, - "limitKey": "IP", - }, - "no-rate-limit": { - "rateBasedRuleId": "arn:aws:wafv2:ap-southeast-2:068133125972_MANAGED:regional/ipset/0f94bd8b-0fa5-4865-81ce-d11a60051fb4_fef50279-8b9a-4062-b733-88ecd1cfd889_IPV4/fef50279-8b9a-4062-b733-88ecd1cfd889", - "limitValue": "195.154.122.189", - "maxRateAllowed": 300, - "limitKey": "IP", - }, - } - }, - }, - ) + verify_as_json(parse_aws_waf_logs(event)) def test_waf_rule_group_with_excluded_and_nonterminating_rules(self): event = { @@ -532,33 +492,7 @@ def test_waf_rule_group_with_excluded_and_nonterminating_rules(self): ] }, } - self.assertEqual( - parse_aws_waf_logs(event), - { - "ddsource": "waf", - "message": { - "ruleGroupList": { - "AWS#AWSManagedRulesSQLiRuleSet": { - "nonTerminatingMatchingRules": { - "second_nonterminating": {"exclusionType": "REGULAR"}, - "first_nonterminating": {"exclusionType": "REGULAR"}, - }, - "excludedRules": { - "GenericRFI_BODY": { - "exclusionType": "EXCLUDED_AS_COUNT" - }, - "second_exclude": { - "exclusionType": "EXCLUDED_AS_COUNT" - }, - }, - "terminatingRule": { - "SQLi_QUERYARGUMENTS": {"action": "BLOCK"} - }, - } - } - }, - }, - ) + verify_as_json(parse_aws_waf_logs(event)) def test_waf_rule_group_two_rules_same_group_id(self): event = { @@ -579,22 +513,7 @@ def test_waf_rule_group_two_rules_same_group_id(self): ] }, } - self.assertEqual( - parse_aws_waf_logs(event), - { - "ddsource": "waf", - "message": { - "ruleGroupList": { - "AWS#AWSManagedRulesSQLiRuleSet": { - "terminatingRule": { - "SQLi_QUERYARGUMENTS": {"action": "BLOCK"}, - "secondRULE": {"action": "BLOCK"}, - } - } - } - }, - }, - ) + verify_as_json(parse_aws_waf_logs(event)) def test_waf_rule_group_three_rules_two_group_ids(self): event = { @@ -619,25 +538,7 @@ def test_waf_rule_group_three_rules_two_group_ids(self): ] }, } - self.assertEqual( - parse_aws_waf_logs(event), - { - "ddsource": "waf", - "message": { - "ruleGroupList": { - "AWS#AWSManagedRulesSQLiRuleSet": { - "terminatingRule": { - "SQLi_QUERYARGUMENTS": {"action": "BLOCK"}, - "secondRULE": {"action": "BLOCK"}, - } - }, - "A_DIFFERENT_ID": { - "terminatingRule": {"thirdRULE": {"action": "BLOCK"}} - }, - } - }, - }, - ) + verify_as_json(parse_aws_waf_logs(event)) class TestParseSecurityHubEvents(unittest.TestCase): @@ -653,17 +554,7 @@ def test_security_hub_one_finding_no_resources(self): "ddsource": "securityhub", "detail": {"findings": [{"myattribute": "somevalue"}]}, } - self.assertEqual( - separate_security_hub_findings(event), - [ - { - "ddsource": "securityhub", - "detail": { - "finding": {"myattribute": "somevalue", "resources": {}} - }, - } - ], - ) + verify_as_json(separate_security_hub_findings(event)) def test_security_hub_two_findings_one_resource_each(self): event = { @@ -685,33 +576,7 @@ def test_security_hub_two_findings_one_resource_each(self): ] }, } - self.assertEqual( - separate_security_hub_findings(event), - [ - { - "ddsource": "securityhub", - "detail": { - "finding": { - "myattribute": "somevalue", - "resources": { - "AwsEc2SecurityGroup": {"Region": "us-east-1"} - }, - } - }, - }, - { - "ddsource": "securityhub", - "detail": { - "finding": { - "myattribute": "somevalue", - "resources": { - "AwsEc2SecurityGroup": {"Region": "us-east-1"} - }, - } - }, - }, - ], - ) + verify_as_json(separate_security_hub_findings(event)) def test_security_hub_multiple_findings_multiple_resources(self): event = { @@ -742,47 +607,7 @@ def test_security_hub_multiple_findings_multiple_resources(self): ] }, } - self.assertEqual( - separate_security_hub_findings(event), - [ - { - "ddsource": "securityhub", - "detail": { - "finding": { - "myattribute": "somevalue", - "resources": { - "AwsEc2SecurityGroup": {"Region": "us-east-1"} - }, - } - }, - }, - { - "ddsource": "securityhub", - "detail": { - "finding": { - "myattribute": "somevalue", - "resources": { - "AwsEc2SecurityGroup": {"Region": "us-east-1"}, - "AwsOtherSecurityGroup": {"Region": "us-east-1"}, - }, - } - }, - }, - { - "ddsource": "securityhub", - "detail": { - "finding": { - "myattribute": "somevalue", - "resources": { - "AwsEc2SecurityGroup": {"Region": "us-east-1"}, - "AwsOtherSecurityGroup": {"Region": "us-east-1"}, - "AwsAnotherSecurityGroup": {"Region": "us-east-1"}, - }, - } - }, - }, - ], - ) + verify_as_json(separate_security_hub_findings(event)) class TestAWSLogsHandler(unittest.TestCase): @@ -838,33 +663,8 @@ def test_awslogs_handler_rds_postgresql( context = None metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} - self.assertEqual( - [ - { - "aws": { - "awslogs": { - "logGroup": "/aws/rds/instance/datadog/postgresql", - "logStream": "datadog.0", - "owner": "123456789012", - } - }, - "id": "31953106606966983378809025079804211143289615424298221568", - "message": "2021-01-02 03:04:05 UTC::@:[5306]:LOG: database system is ready " - "to accept connections", - "timestamp": 1609556645000, - } - ], - list(awslogs_handler(event, context, metadata)), - ) - self.assertEqual( - { - "ddsource": "postgresql", - "ddtags": "env:dev,test_tag_key:test_tag_value,logname:postgresql", - "host": "datadog", - "service": "postgresql", - }, - metadata, - ) + verify_as_json(list(awslogs_handler(event, context, metadata))) + verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) @patch("parsing.CloudwatchLogGroupTagsCache.get") @patch("parsing.StepFunctionsTagsCache.get") @@ -924,32 +724,8 @@ def test_awslogs_handler_step_functions_tags_added_properly( context = None metadata = {"ddsource": "postgresql", "ddtags": "env:dev"} - self.assertEqual( - [ - { - "aws": { - "awslogs": { - "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", - "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", - "owner": "425362996713", - } - }, - "id": "37199773595581154154810589279545129148442535997644275712", - "message": '{"id":"1","type":"ExecutionStarted","details":{"input":"{"Comment": "Insert your JSON here"}","inputDetails":{"truncated":false},"roleArn":"arn:aws:iam::425362996713:role/service-role/StepFunctions-logs-to-traces-sequential-role-ccd69c03"},",previous_event_id":"0","event_timestamp":"1668095539607","execution_arn":"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential:d0dbefd8-a0f6-b402-da4c-f4863def7456:7fa0cfbe-be28-4a20-9875-73c37f5dc39e"}', - "timestamp": 1668095539607, - } - ], - list(awslogs_handler(event, context, metadata)), - ) - self.assertEqual( - { - "ddsource": "stepfunction", - "ddtags": "env:dev,test_tag_key:test_tag_value", - "host": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", - "service": "stepfunction", - }, - metadata, - ) + verify_as_json(list(awslogs_handler(event, context, metadata))) + verify_as_json(metadata, options=NamerFactory.with_parameters("metadata")) class TestGetServiceFromTags(unittest.TestCase):