[AWSX-537] Support python version 3.10

.github/workflows/integration_test.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ['3.9', '3.10']
     steps:
       - name: Checkout source
         uses: actions/checkout@v3

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v3
         with:
-          python-version: 3.9
+          python-version: '3.10'
 
       - name: Install pip
         run: |

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ['3.9', '3.10']
     steps:
       - name: Checkout source
         uses: actions/checkout@v3

aws/logs_monitoring/README.md

@@ -114,7 +114,7 @@ resource "aws_cloudformation_stack" "datadog_forwarder" {
 
 If you can't install the Forwarder using the provided CloudFormation template, you can install the Forwarder manually following the steps below. Feel free to open an issue or pull request to let us know if there is anything we can improve to make the template work for you.
 
-1. Create a Python 3.9 Lambda function using `aws-dd-forwarder-<VERSION>.zip` from the latest [releases][101].
+1. Create a Python 3.10 Lambda function using `aws-dd-forwarder-<VERSION>.zip` from the latest [releases][101].
 2. Save your [Datadog API key][102] in AWS Secrets Manager, set environment variable `DD_API_KEY_SECRET_ARN` with the secret ARN on the Lambda function, and add the `secretsmanager:GetSecretValue` permission to the Lambda execution role.
 3. If you need to forward logs from S3 buckets, add the `s3:GetObject` permission to the Lambda execution role.
 4. Set the environment variable `DD_ENHANCED_METRICS` to `false` on the Forwarder. This stops the Forwarder from generating enhanced metrics itself, but it will still forward custom metrics from other lambdas.
@@ -138,6 +138,10 @@ If you can't install the Forwarder using the provided CloudFormation template, y
 
 If you encounter issues upgrading to the latest version, check the Troubleshooting section.
 
+### Upgrade an older version to +3.98.0
+
+Since version 3.98.0 the Lambda function has been updated to require **Python 3.10**. If upgrading an older forwarder installation to 3.98.0 or above, ensure the AWS Lambda function is configured to use Python 3.10
+
 ### Upgrade an older version to +3.74.0
 
 Since version 3.74.0 the Lambda function has been updated to require **Python 3.9**. If upgrading an older forwarder installation to 3.74.0 or above, ensure the AWS Lambda function is configured to use Python 3.9

aws/logs_monitoring/setup.py

@@ -12,12 +12,15 @@
     author="Datadog, Inc.",
     author_email="[email protected]",
     classifiers=[
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
     keywords="datadog aws lambda layer",
-    python_requires=">=3.7, <3.10",
-    install_requires=["datadog-lambda==3.39.0", "requests-futures==1.0.0"],
+    python_requires=">=3.9, <3.11",
+    install_requires=[
+        "datadog-lambda==4.77.0",
+        "requests-futures==1.0.0",
+    ],
     extras_require={
         "dev": ["nose2==0.9.1", "flake8==3.7.9", "requests==2.22.0", "boto3==1.10.33"]
     },

aws/logs_monitoring/template.yaml

@@ -478,7 +478,7 @@ Resources:
 
       MemorySize:
         Ref: MemorySize
-      Runtime: python3.9
+      Runtime: python3.10
       Timeout:
         Ref: Timeout
       Tags:
@@ -876,7 +876,7 @@ Resources:
     Properties:
       Description: Copies Datadog Forwarder zip to the destination S3 bucket
       Handler: index.handler
-      Runtime: python3.9
+      Runtime: python3.10
       Timeout: 300
       Code:
         ZipFile: |

aws/logs_monitoring/tools/Dockerfile_bundle

@@ -19,4 +19,3 @@ RUN rm -rf ./botocore*
 # like `os.execl`, which cause security scans to fail for certain customers.
 # These files are not directly used by the Forwarder.
 RUN rm ./ddtrace/commands/ddtrace_run.py
-RUN rm ./decorator.py

aws/logs_monitoring/tools/add_new_region.sh

@@ -12,7 +12,7 @@ set -e
 
 OLD_REGION='us-east-1'
 
-PYTHON_VERSIONS_FOR_AWS_CLI=("python3.9")
+PYTHON_VERSIONS_FOR_AWS_CLI=("python3.10")
 LAYER_NAMES=("Datadog-Forwarder")
 NEW_REGION=$1
 
@@ -82,4 +82,4 @@ for layer_name in "${LAYER_NAMES[@]}"; do
     done
 
     j=$(expr $j + 1)
-done
+done

aws/logs_monitoring/tools/build_bundle.sh

@@ -22,7 +22,7 @@ else
     VERSION=$1
 fi
 
-PYTHON_VERSION="${PYTHON_VERSION:-3.9}"
+PYTHON_VERSION="${PYTHON_VERSION:-3.10}"
 FORWARDER_PREFIX="aws-dd-forwarder"
 FORWARDER_DIR="../.forwarder"
 
@@ -42,7 +42,7 @@ function docker_build_zip {
     temp_dir=$(mktemp -d)
 
     docker buildx build --platform linux/amd64 --file "${DIR}/Dockerfile_bundle" -t "datadog-bundle:$1" .. --no-cache \
-        --build-arg runtime=$1
+        --build-arg runtime=$PYTHON_VERSION
 
     # Run the image by runtime tag, tar its generated `python` directory to sdout,
     # then extract it to a temp directory.

aws/logs_monitoring/tools/integration_tests/cache_test_lambda/serverless.yml

@@ -1,7 +1,7 @@
 service: integration-tests
 provider:
   name: aws
-  runtime: python3.9
+  runtime: python3.10
 
 functions:
   cache_test_lambda:

aws/logs_monitoring/tools/integration_tests/docker-compose.yml

@@ -1,6 +1,22 @@
-version: "3.9"
+version: "3.8"
 
 services:
+  recorder:
+    image: ${PYTHON_BASE}
+    command: /bin/sh -c 'pip install protobuf && ./recorder.py'
+    volumes:
+      - ./recorder:/recorder
+    working_dir: /recorder
+    environment:
+      SERVER_PORT: 8080
+    expose:
+      - 8080
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
   forwarder:
     image: datadog-log-forwarder:${PYTHON_RUNTIME}
     command: lambda_function.lambda_handler
@@ -30,26 +46,21 @@ services:
       DD_FETCH_STEP_FUNCTIONS_TAGS: "false"  # intentionally set false to allow integration test for step function logs to run without hitting aws
     expose:
       - 9001
-
-  recorder:
-    build: recorder
-    command: ./recorder.py
-    environment:
-      SERVER_PORT: 8080
-    expose:
-      - 8080
+    depends_on:
+      recorder:
+        condition: service_healthy
 
   tester:
-    build: tester
-    command: python -m unittest discover
+    image: ${PYTHON_BASE}
+    command: /bin/sh -c 'pip install "deepdiff<6" && python -m unittest discover'
     volumes:
+      - ./tester:/tester
       - ${SNAPSHOTS_DIR_NAME}:/snapshots
+    working_dir: /tester
     environment:
       RECORDER_URL: http://recorder:8080/recording
       FORWARDER_URL: http://forwarder:9001/2015-03-31/functions/myfunction/invocations
       UPDATE_SNAPSHOTS: ${UPDATE_SNAPSHOTS:-false}
       SNAPSHOTS_DIR_NAME: ${SNAPSHOTS_DIR_NAME}
-
     depends_on:
       - forwarder
-      - recorder

aws/logs_monitoring/tools/integration_tests/external_lambda/serverless.yml

@@ -1,7 +1,7 @@
 service: forwarder-tests-external-lambda
 provider:
   name: aws
-  runtime: python2.7
+  runtime: python3.10
 
 functions:
   ironmaiden:

aws/logs_monitoring/tools/integration_tests/integration_tests.sh

@@ -7,7 +7,8 @@
 
 set -e
 
-PYTHON_VERSION="python3.9"
+PYTHON_VERSION="python3.10"
+PYTHON_IMAGE="python:3.10"
 SKIP_FORWARDER_BUILD=false
 UPDATE_SNAPSHOTS=false
 LOG_LEVEL=info
@@ -39,9 +40,10 @@ do
 
 		# -v or --python-version
 		# The version of the Python Lambda runtime to use
-		# Must be 3.8 or 3.9
+		# Must be 3.9 or 3.10
 		-v=*|--python-version=*)
 		PYTHON_VERSION="python${arg#*=}"
+		PYTHON_IMAGE="python:${arg#*=}"
 		shift
 		;;
 
@@ -78,8 +80,8 @@ do
 	esac
 done
 
-if [ $PYTHON_VERSION != "python3.8" ] && [ $PYTHON_VERSION != "python3.9" ]; then
-    echo "Must use either Python 3.8 or 3.9"
+if [ $PYTHON_VERSION != "python3.9" ] && [ $PYTHON_VERSION != "python3.10" ]; then
+    echo "Must use either Python 3.9 or 3.10"
     exit 1
 fi
 
@@ -148,28 +150,18 @@ fi
 
 cd $INTEGRATION_TESTS_DIR
 
-if [ $PYTHON_VERSION == "python3.8" ]; then
-# Build Docker image of Forwarder for tests 3.8 compatibility
-	echo "Building Docker Image for Forwarder with tag datadog-log-forwarder:$PYTHON_VERSION"
-	docker buildx build --platform linux/amd64 --file "${INTEGRATION_TESTS_DIR}/forwarder/Dockerfile" -t "datadog-log-forwarder:$PYTHON_VERSION" ../../.forwarder --no-cache \
-    --build-arg forwarder='aws-dd-forwarder-0.0.0' \
-    --build-arg image="lambci/lambda:${PYTHON_VERSION}"
-fi
 
-# Build Docker image of Forwarder for tests >= 3.9 compatibility
-# See https://github.com/lambci/lambci/issues/138, previous image not supported anymore
-# and we need the DOCKER_LAMBDA_STAY_OPEN=1 option,so relying on a fork
-if [ $PYTHON_VERSION == "python3.9" ]; then
-	echo "Building Docker Image for Forwarder with tag datadog-log-forwarder:$PYTHON_VERSION"
-	docker buildx build --platform linux/amd64 --file "${INTEGRATION_TESTS_DIR}/forwarder/Dockerfile" -t "datadog-log-forwarder:$PYTHON_VERSION" ../../.forwarder --no-cache \
-    --build-arg forwarder='aws-dd-forwarder-0.0.0' \
-    --build-arg image="mlupin/docker-lambda:${PYTHON_VERSION}-x86_64"
-fi
+# Build Docker image of Forwarder for tests
+echo "Building Docker Image for Forwarder with tag datadog-log-forwarder:$PYTHON_VERSION"
+docker buildx build --platform linux/amd64 --file "${INTEGRATION_TESTS_DIR}/forwarder/Dockerfile" -t "datadog-log-forwarder:$PYTHON_VERSION" ../../.forwarder --no-cache \
+--build-arg forwarder='aws-dd-forwarder-0.0.0' \
+--build-arg image="mlupin/docker-lambda:${PYTHON_VERSION}-x86_64"
 
 echo "Running integration tests for ${PYTHON_VERSION}"
 LOG_LEVEL=${LOG_LEVEL} \
 UPDATE_SNAPSHOTS=${UPDATE_SNAPSHOTS} \
 PYTHON_RUNTIME=${PYTHON_VERSION} \
+PYTHON_BASE=${PYTHON_IMAGE} \
 EXTERNAL_LAMBDAS=${EXTERNAL_LAMBDAS} \
 DD_S3_BUCKET_NAME=${DD_S3_BUCKET_NAME} \
 AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_customized_log_group_lambda_invocation.json~snapshot

@@ -155,13 +155,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log.json~snapshot

@@ -127,13 +127,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_apigateway.json~snapshot

@@ -54,13 +54,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_cloudtrail.json~snapshot

@@ -165,13 +165,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_coldstart.json~snapshot

@@ -82,13 +82,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_custom_tags.json~snapshot

@@ -94,13 +94,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]

aws/logs_monitoring/tools/integration_tests/snapshots/cloudwatch_log_fsx_windows.json~snapshot

@@ -108,13 +108,14 @@
         "Content-Encoding": "deflate",
         "Content-Length": "<redacted from snapshot>",
         "Content-Type": "application/json",
+        "DD-API-KEY": "abcdefghijklmnopqrstuvwxyz012345",
         "Host": "recorder:8080",
         "User-Agent": "<redacted from snapshot>",
         "x-datadog-parent-id": "<redacted from snapshot>",
         "x-datadog-sampling-priority": "2",
         "x-datadog-trace-id": "4842834437835386637"
       },
-      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "path": "/api/v1/distribution_points",
       "verb": "POST"
     }
   ]