[WIP][AWS logs forwarder] Support Py 3.12

DataDog · Dec 18, 2024 · 60300eb · 60300eb
1 parent a1b9285
commit 60300eb
Show file tree

Hide file tree

Showing 22 changed files with 388 additions and 490 deletions.
diff --git a/aws/logs_monitoring/requirements.txt b/aws/logs_monitoring/requirements.txt
@@ -3,10 +3,10 @@ bytecode
 cattrs
 certifi
 charset-normalizer
-datadog-lambda==5.87.0
+datadog-lambda==6.104.0
 datadog==0.48.0
-ddsketch==2.0.4
-ddtrace==2.6.5
+ddsketch==3.0.1
+ddtrace==2.17.3
 deprecated
 envier
 exceptiongroup

diff --git a/aws/logs_monitoring/setup.py b/aws/logs_monitoring/setup.py
@@ -12,11 +12,11 @@
     author="Datadog, Inc.",
     author_email="[email protected]",
     classifiers=[
-        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     keywords="datadog aws lambda layer",
-    python_requires=">=3.10, <3.12",
+    python_requires=">=3.11, <3.13",
     extras_require={
         "dev": ["nose2==0.9.1", "flake8==3.7.9", "requests==2.22.0", "boto3==1.10.33"]
     },

diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml
@@ -396,7 +396,7 @@ Resources:
             - {DdForwarderVersion: !FindInMap [Constants, DdForwarder, Version]}
         - ZipFile: " "
       MemorySize: !Ref MemorySize
-      Runtime: python3.11
+      Runtime: python3.12
       Timeout: !Ref Timeout
       Tags:
         - Key: "dd_forwarder_version"
@@ -724,7 +724,7 @@ Resources:
         - !Ref AWS::NoValue
       BucketEncryption:
         ServerSideEncryptionConfiguration:
-          - BucketKeyEnabled: true 
+          - BucketKeyEnabled: true
             ServerSideEncryptionByDefault:
               SSEAlgorithm: "aws:kms"
       PublicAccessBlockConfiguration:
@@ -788,7 +788,7 @@ Resources:
     Properties:
       Description: Copies Datadog Forwarder zip to the destination S3 bucket
       Handler: index.handler
-      Runtime: python3.11
+      Runtime: python3.12
       Timeout: 600
       Code:
         ZipFile: |

diff --git a/aws/logs_monitoring/tools/build_bundle.sh b/aws/logs_monitoring/tools/build_bundle.sh
@@ -42,7 +42,7 @@ else
     VERSION=$1
 fi
 
-PYTHON_VERSION="${PYTHON_VERSION:-3.11}"
+PYTHON_VERSION="${PYTHON_VERSION:-3.12}"
 FORWARDER_PREFIX="aws-dd-forwarder"
 FORWARDER_DIR="../.forwarder"
 

diff --git a/aws/logs_monitoring/tools/integration_tests/docker-compose.yml b/aws/logs_monitoring/tools/integration_tests/docker-compose.yml
@@ -26,6 +26,7 @@ services:
       AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
       AWS_SECURITY_TOKEN: "${AWS_SECURITY_TOKEN}"
       AWS_SESSION_TOKEN: "${AWS_SESSION_TOKEN}"
+      AWS_DEFAULT_REGION: us-east-1
       DOCKER_LAMBDA_STAY_OPEN: 1
       DD_LOG_LEVEL: ${LOG_LEVEL:-info}
       DD_API_KEY: abcdefghijklmnopqrstuvwxyz012345 # Must be 32 characters exactly
@@ -46,22 +47,31 @@ services:
       DD_FETCH_STEP_FUNCTIONS_TAGS: "false"  # intentionally set false to allow integration test for step function logs to run without hitting aws
       DD_STORE_FAILED_EVENTS: "false"
     expose:
-      - 9001
+      - 8080
     depends_on:
       recorder:
         condition: service_healthy
+    healthcheck:
+        test: ["CMD", "curl", "-f", "http://localhost:8080/2015-03-31/functions/function/invocations"]
+        interval: 10s
+        timeout: 5s
+        retries: 3
+
 
   tester:
     image: ${PYTHON_BASE}
-    command: /bin/sh -c 'pip install "deepdiff<6" && python -m unittest discover'
+    command: /bin/sh -c 'pip install "deepdiff<6" && pip install "urllib3" && python -m unittest discover'
     volumes:
       - ./tester:/tester
       - ${SNAPSHOTS_DIR_NAME}:/snapshots
     working_dir: /tester
     environment:
       RECORDER_URL: http://recorder:8080/recording
-      FORWARDER_URL: http://forwarder:9001/2015-03-31/functions/myfunction/invocations
+      FORWARDER_URL: http://forwarder:8080/2015-03-31/functions/function/invocations
       UPDATE_SNAPSHOTS: ${UPDATE_SNAPSHOTS:-false}
       SNAPSHOTS_DIR_NAME: ${SNAPSHOTS_DIR_NAME}
     depends_on:
-      - forwarder
+        forwarder:
+            condition: service_healthy
+        recorder:
+            condition: service_healthy
diff --git a/aws/logs_monitoring/tools/integration_tests/forwarder/Dockerfile b/aws/logs_monitoring/tools/integration_tests/forwarder/Dockerfile
@@ -5,3 +5,5 @@ ARG forwarder
 
 # Add the code into /var/task (will unzip files)
 ADD $forwarder /var/task/
+
+CMD ["lambda_function.lambda_handler"]
diff --git a/aws/logs_monitoring/tools/integration_tests/integration_tests.sh b/aws/logs_monitoring/tools/integration_tests/integration_tests.sh
@@ -7,8 +7,9 @@
 
 set -e
 
-PYTHON_VERSION="python3.11"
-PYTHON_IMAGE="python:3.11"
+PYTHON_VERSION="python3.12"
+PYTHON_VERSION_TAG="3.12"
+PYTHON_IMAGE="python:3.12"
 SKIP_FORWARDER_BUILD=false
 UPDATE_SNAPSHOTS=false
 LOG_LEVEL=info
@@ -38,10 +39,11 @@ for arg in "$@"; do
 
         # -v or --python-version
         # The version of the Python Lambda runtime to use
-        # Must be 3.9 or 3.10
+        # Must be 3.11 or 3.12
         -v=* | --python-version=*)
                 PYTHON_VERSION="python${arg#*=}"
                 PYTHON_IMAGE="python:${arg#*=}"
+                PYTHON_VERSION_TAG="${arg#*=}"
                 shift
                 ;;
 
@@ -78,8 +80,8 @@ for arg in "$@"; do
         esac
 done
 
-if [ $PYTHON_VERSION != "python3.10" ] && [ $PYTHON_VERSION != "python3.11" ]; then
-        echo "Must use either Python 3.10 or 3.11"
+if [ $PYTHON_VERSION != "python3.11" ] && [ $PYTHON_VERSION != "python3.12" ]; then
+        echo "Must use either Python 3.11 or 3.12"
         exit 1
 fi
 
@@ -152,7 +154,7 @@ cd $INTEGRATION_TESTS_DIR
 echo "Building Docker Image for Forwarder with tag datadog-log-forwarder:$PYTHON_VERSION"
 docker buildx build --platform linux/amd64 --file "${INTEGRATION_TESTS_DIR}/forwarder/Dockerfile" -t "datadog-log-forwarder:$PYTHON_VERSION" ../../.forwarder --no-cache \
         --build-arg forwarder='aws-dd-forwarder-0.0.0' \
-        --build-arg image="mlupin/docker-lambda:${PYTHON_VERSION}-x86_64"
+        --build-arg image="public.ecr.aws/lambda/python:${PYTHON_VERSION_TAG}-x86_64"
 
 echo "Running integration tests for ${PYTHON_VERSION}"
 LOG_LEVEL=${LOG_LEVEL} \

diff --git a/...tegration_tests/snapshots/cloudwatch_customized_log_group_lambda_invocation.json~snapshot b/...tegration_tests/snapshots/cloudwatch_customized_log_group_lambda_invocation.json~snapshot
@@ -10,15 +10,15 @@
               "owner": "601427279990"
             },
             "function_version": "$LATEST",
-            "invoked_function_arn": "arn:aws:lambda:us-east-1:0000000000:function:test"
+            "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function"
           },
           "ddsource": "lambda",
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0000000000,aws_account:0000000000,functionname:test-customized-loggroup,region:us-east-1,service:test-customized-loggroup",
-          "host": "arn:aws:lambda:us-east-1:0000000000:function:test-customized-loggroup",
+          "ddtags": "forwardername:test_function,forwarder_memorysize:3008,forwarder_version:<redacted from snapshot>,env:none,account_id:012345678912,aws_account:012345678912,functionname:test-customized-loggroup,region:us-east-1,service:test-customized-loggroup",
+          "host": "arn:aws:lambda:us-east-1:012345678912:function:test-customized-loggroup",
           "id": "35311576111948622874033876462979853992919938886093242368",
           "lambda": {
-            "arn": "arn:aws:lambda:us-east-1:0000000000:function:test-customized-loggroup"
+            "arn": "arn:aws:lambda:us-east-1:012345678912:function:test-customized-loggroup"
           },
           "message": "2020-03-05T16:30:36.113Z\t<redacted from snapshot>\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Patched console output with trace context\"}\n",
           "service": "test-customized-loggroup",
@@ -32,15 +32,15 @@
               "owner": "601427279990"
             },
             "function_version": "$LATEST",
-            "invoked_function_arn": "arn:aws:lambda:us-east-1:0000000000:function:test"
+            "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function"
           },
           "ddsource": "lambda",
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0000000000,aws_account:0000000000,functionname:test-customized-loggroup,region:us-east-1,service:test-customized-loggroup",
-          "host": "arn:aws:lambda:us-east-1:0000000000:function:test-customized-loggroup",
+          "ddtags": "forwardername:test_function,forwarder_memorysize:3008,forwarder_version:<redacted from snapshot>,env:none,account_id:012345678912,aws_account:012345678912,functionname:test-customized-loggroup,region:us-east-1,service:test-customized-loggroup",
+          "host": "arn:aws:lambda:us-east-1:012345678912:function:test-customized-loggroup",
           "id": "35311576111948622874033876462979853992919938886093242369",
           "lambda": {
-            "arn": "arn:aws:lambda:us-east-1:0000000000:function:test-customized-loggroup"
+            "arn": "arn:aws:lambda:us-east-1:012345678912:function:test-customized-loggroup"
           },
           "message": "2020-03-05T16:30:36.114Z\t<redacted from snapshot>\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"autoPatchHTTP\":true,\"tracerInitialized\":true,\"status\":\"debug\",\"message\":\"datadog:Not patching HTTP libraries\"}\n",
           "service": "test-customized-loggroup",
@@ -54,15 +54,15 @@
               "owner": "601427279990"
             },
             "function_version": "$LATEST",
-            "invoked_function_arn": "arn:aws:lambda:us-east-1:0000000000:function:test"
+            "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function"
           },
           "ddsource": "lambda",
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0000000000,aws_account:0000000000,functionname:test-customized-loggroup,region:us-east-1,service:test-customized-loggroup",
-          "host": "arn:aws:lambda:us-east-1:0000000000:function:test-customized-loggroup",
+          "ddtags": "forwardername:test_function,forwarder_memorysize:3008,forwarder_version:<redacted from snapshot>,env:none,account_id:012345678912,aws_account:012345678912,functionname:test-customized-loggroup,region:us-east-1,service:test-customized-loggroup",
+          "host": "arn:aws:lambda:us-east-1:012345678912:function:test-customized-loggroup",
           "id": "35311576111948622874033876462979853992919938886093242370",
           "lambda": {
-            "arn": "arn:aws:lambda:us-east-1:0000000000:function:test-customized-loggroup"
+            "arn": "arn:aws:lambda:us-east-1:012345678912:function:test-customized-loggroup"
           },
           "message": "2020-03-05T16:30:36.114Z\t<redacted from snapshot>\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Reading trace context from env var Root=1-5e61292c-cc1229a4dfbeae1043928548;Parent=c657b77d9514f70c;Sampled=1\"}\n",
           "service": "test-customized-loggroup",
@@ -79,12 +79,7 @@
         "DD-EVP-ORIGIN": "aws_forwarder",
         "DD-EVP-ORIGIN-VERSION": "<redacted from snapshot>",
         "Host": "recorder:8080",
-        "User-Agent": "<redacted from snapshot>",
-        "traceparent": "00-0000000000000000433534434534ef0d-91ed514f1e5c03b2-01",
-        "tracestate": "dd=s:2",
-        "x-datadog-parent-id": "<redacted from snapshot>",
-        "x-datadog-sampling-priority": "2",
-        "x-datadog-trace-id": "4842834437835386637"
+        "User-Agent": "<redacted from snapshot>"
       },
       "path": "/api/v2/logs",
       "verb": "POST"
@@ -99,8 +94,8 @@
             "metric": "aws.dd_forwarder.loggroup_cache_fetch_failure",
             "points": "<redacted from snapshot>",
             "tags": [
-              "forwardername:test",
-              "forwarder_memorysize:1536",
+              "forwardername:test_function",
+              "forwarder_memorysize:3008",
               "forwarder_version:<redacted from snapshot>",
               "event_type:awslogs"
             ],
@@ -113,8 +108,8 @@
             "metric": "aws.dd_forwarder.list_tags_log_group_api_call",
             "points": "<redacted from snapshot>",
             "tags": [
-              "forwardername:test",
-              "forwarder_memorysize:1536",
+              "forwardername:test_function",
+              "forwarder_memorysize:3008",
               "forwarder_version:<redacted from snapshot>",
               "event_type:awslogs"
             ],
@@ -127,8 +122,8 @@
             "metric": "aws.dd_forwarder.loggroup_cache_write_failure",
             "points": "<redacted from snapshot>",
             "tags": [
-              "forwardername:test",
-              "forwarder_memorysize:1536",
+              "forwardername:test_function",
+              "forwarder_memorysize:3008",
               "forwarder_version:<redacted from snapshot>",
               "event_type:awslogs"
             ],
@@ -141,8 +136,8 @@
             "metric": "aws.dd_forwarder.incoming_events",
             "points": "<redacted from snapshot>",
             "tags": [
-              "forwardername:test",
-              "forwarder_memorysize:1536",
+              "forwardername:test_function",
+              "forwarder_memorysize:3008",
               "forwarder_version:<redacted from snapshot>",
               "event_type:awslogs"
             ],
@@ -155,8 +150,8 @@
             "metric": "aws.dd_forwarder.logs_forwarded",
             "points": "<redacted from snapshot>",
             "tags": [
-              "forwardername:test",
-              "forwarder_memorysize:1536",
+              "forwardername:test_function",
+              "forwarder_memorysize:3008",
               "forwarder_version:<redacted from snapshot>",
               "event_type:awslogs"
             ],
@@ -169,8 +164,8 @@
             "metric": "aws.dd_forwarder.metrics_forwarded",
             "points": "<redacted from snapshot>",
             "tags": [
-              "forwardername:test",
-              "forwarder_memorysize:1536",
+              "forwardername:test_function",
+              "forwarder_memorysize:3008",
               "forwarder_version:<redacted from snapshot>",
               "event_type:awslogs"
             ],
@@ -187,12 +182,7 @@
         "Content-Type": "application/json",
         "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
-        "User-Agent": "<redacted from snapshot>",
-        "traceparent": "00-0000000000000000433534434534ef0d-91ed514f1e5c03b2-01",
-        "tracestate": "dd=s:2",
-        "x-datadog-parent-id": "<redacted from snapshot>",
-        "x-datadog-sampling-priority": "2",
-        "x-datadog-trace-id": "4842834437835386637"
+        "User-Agent": "<redacted from snapshot>"
       },
       "path": "/api/v1/distribution_points",
       "verb": "POST"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -5,3 +5,5 @@ ARG forwarder

		# Add the code into /var/task (will unzip files)
		ADD $forwarder /var/task/

		CMD ["lambda_function.lambda_handler"]