[AWS logs forwarder] Support Python 3.12 (#878)

* [WIP][AWS logs forwarder] Support Py 3.12

* Update github workflow

* add missing package ujson

.github/workflows/integration_test.yml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.11", "3.12"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.10', '3.11']
+        python-version: ['3.11', '3.12']
     steps:
       - name: Checkout source
         uses: actions/checkout@v3

aws/logs_monitoring/requirements.txt

@@ -3,10 +3,10 @@ bytecode
 cattrs
 certifi
 charset-normalizer
-datadog-lambda==5.87.0
-datadog==0.48.0
-ddsketch==2.0.4
-ddtrace==2.6.5
+datadog-lambda==6.104.0
+datadog==0.50.2
+ddsketch==3.0.1
+ddtrace==2.17.3
 deprecated
 envier
 exceptiongroup
@@ -22,3 +22,4 @@ urllib3>=1.26.19,<3.0
 wrapt==1.14.0
 xmltodict
 zipp
+ujson

aws/logs_monitoring/setup.py

@@ -12,11 +12,11 @@
     author="Datadog, Inc.",
     author_email="[email protected]",
     classifiers=[
-        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     keywords="datadog aws lambda layer",
-    python_requires=">=3.10, <3.12",
+    python_requires=">=3.11, <3.13",
     extras_require={
         "dev": ["nose2==0.9.1", "flake8==3.7.9", "requests==2.22.0", "boto3==1.10.33"]
     },

aws/logs_monitoring/template.yaml

@@ -259,6 +259,13 @@ Parameters:
       - WARN
       - ERROR
       - CRITICAL
+  DdTraceEnabled:
+    Type: String
+    Default: "true"
+    AllowedValues:
+      - "true"
+      - "false"
+    Description: Set to false to disable trace forwarding.
 Conditions:
   IsAWSChina: !Equals [!Ref 'AWS::Partition', aws-cn]
   IsGovCloud: !Equals [!Ref 'AWS::Partition', aws-us-gov]
@@ -348,6 +355,7 @@ Conditions:
     - !Equals [!Join ["", !Ref VPCSubnetIds], ""]
   SetDdLogLevel: !Not
     - !Equals [!Ref DdLogLevel, ""]
+  SetDdTraceEnabled: !Equals [!Ref DdTraceEnabled, false]
 Rules:
   MustSetDdApiKey:
     Assertions:
@@ -396,7 +404,7 @@ Resources:
             - {DdForwarderVersion: !FindInMap [Constants, DdForwarder, Version]}
         - ZipFile: " "
       MemorySize: !Ref MemorySize
-      Runtime: python3.11
+      Runtime: python3.12
       Timeout: !Ref Timeout
       Tags:
         - Key: "dd_forwarder_version"
@@ -540,6 +548,10 @@ Resources:
             - SetDdLogLevel
             - !Ref DdLogLevel
             - !Ref AWS::NoValue
+          DD_TRACE_ENABLED: !If
+            - SetDdTraceEnabled
+            - !Ref DdTraceEnabled
+            - "true"
       ReservedConcurrentExecutions: !If
         - SetReservedConcurrentExecutions
         - !Ref ReservedConcurrency
@@ -724,7 +736,7 @@ Resources:
         - !Ref AWS::NoValue
       BucketEncryption:
         ServerSideEncryptionConfiguration:
-          - BucketKeyEnabled: true 
+          - BucketKeyEnabled: true
             ServerSideEncryptionByDefault:
               SSEAlgorithm: "aws:kms"
       PublicAccessBlockConfiguration:
@@ -788,7 +800,7 @@ Resources:
     Properties:
       Description: Copies Datadog Forwarder zip to the destination S3 bucket
       Handler: index.handler
-      Runtime: python3.11
+      Runtime: python3.12
       Timeout: 600
       Code:
         ZipFile: |

aws/logs_monitoring/tools/build_bundle.sh

@@ -42,7 +42,7 @@ else
     VERSION=$1
 fi
 
-PYTHON_VERSION="${PYTHON_VERSION:-3.11}"
+PYTHON_VERSION="${PYTHON_VERSION:-3.12}"
 FORWARDER_PREFIX="aws-dd-forwarder"
 FORWARDER_DIR="../.forwarder"
 

aws/logs_monitoring/tools/integration_tests/docker-compose.yml

@@ -26,7 +26,7 @@ services:
       AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}"
       AWS_SECURITY_TOKEN: "${AWS_SECURITY_TOKEN}"
       AWS_SESSION_TOKEN: "${AWS_SESSION_TOKEN}"
-      DOCKER_LAMBDA_STAY_OPEN: 1
+      AWS_DEFAULT_REGION: us-east-1
       DD_LOG_LEVEL: ${LOG_LEVEL:-info}
       DD_API_KEY: abcdefghijklmnopqrstuvwxyz012345 # Must be 32 characters exactly
       DD_URL: recorder # Used for logs intake
@@ -45,11 +45,18 @@ services:
       DD_FETCH_LOG_GROUP_TAGS: "true"
       DD_FETCH_STEP_FUNCTIONS_TAGS: "false"  # intentionally set false to allow integration test for step function logs to run without hitting aws
       DD_STORE_FAILED_EVENTS: "false"
+      DD_TRACE_ENABLED: "true"
     expose:
-      - 9001
+      - 8080
     depends_on:
       recorder:
         condition: service_healthy
+    healthcheck:
+        test: ["CMD", "curl", "-f", "http://localhost:8080/2015-03-31/functions/function/invocations"]
+        interval: 10s
+        timeout: 5s
+        retries: 3
+
 
   tester:
     image: ${PYTHON_BASE}
@@ -60,8 +67,11 @@ services:
     working_dir: /tester
     environment:
       RECORDER_URL: http://recorder:8080/recording
-      FORWARDER_URL: http://forwarder:9001/2015-03-31/functions/myfunction/invocations
+      FORWARDER_URL: http://forwarder:8080/2015-03-31/functions/function/invocations
       UPDATE_SNAPSHOTS: ${UPDATE_SNAPSHOTS:-false}
       SNAPSHOTS_DIR_NAME: ${SNAPSHOTS_DIR_NAME}
     depends_on:
-      - forwarder
+        forwarder:
+            condition: service_healthy
+        recorder:
+            condition: service_healthy

aws/logs_monitoring/tools/integration_tests/integration_tests.sh

@@ -7,8 +7,9 @@
 
 set -e
 
-PYTHON_VERSION="python3.11"
-PYTHON_IMAGE="python:3.11"
+PYTHON_VERSION="python3.12"
+PYTHON_VERSION_TAG="3.12"
+PYTHON_IMAGE="python:3.12"
 SKIP_FORWARDER_BUILD=false
 UPDATE_SNAPSHOTS=false
 LOG_LEVEL=info
@@ -38,10 +39,11 @@ for arg in "$@"; do
 
         # -v or --python-version
         # The version of the Python Lambda runtime to use
-        # Must be 3.9 or 3.10
+        # Must be 3.11 or 3.12
         -v=* | --python-version=*)
                 PYTHON_VERSION="python${arg#*=}"
                 PYTHON_IMAGE="python:${arg#*=}"
+                PYTHON_VERSION_TAG="${arg#*=}"
                 shift
                 ;;
 
@@ -78,8 +80,8 @@ for arg in "$@"; do
         esac
 done
 
-if [ $PYTHON_VERSION != "python3.10" ] && [ $PYTHON_VERSION != "python3.11" ]; then
-        echo "Must use either Python 3.10 or 3.11"
+if [ $PYTHON_VERSION != "python3.11" ] && [ $PYTHON_VERSION != "python3.12" ]; then
+        echo "Must use either Python 3.11 or 3.12"
         exit 1
 fi
 
@@ -152,7 +154,7 @@ cd $INTEGRATION_TESTS_DIR
 echo "Building Docker Image for Forwarder with tag datadog-log-forwarder:$PYTHON_VERSION"
 docker buildx build --platform linux/amd64 --file "${INTEGRATION_TESTS_DIR}/forwarder/Dockerfile" -t "datadog-log-forwarder:$PYTHON_VERSION" ../../.forwarder --no-cache \
         --build-arg forwarder='aws-dd-forwarder-0.0.0' \
-        --build-arg image="mlupin/docker-lambda:${PYTHON_VERSION}-x86_64"
+        --build-arg image="public.ecr.aws/lambda/python:${PYTHON_VERSION_TAG}-x86_64"
 
 echo "Running integration tests for ${PYTHON_VERSION}"
 LOG_LEVEL=${LOG_LEVEL} \