ci(release): Fixing the release script for prod

Signed-off-by: Vincent Boutour <[email protected]>
DataDog · Jul 26, 2024 · b867f92 · b867f92
1 parent 3f3007f
commit b867f92
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 112 deletions.
diff --git a/aws/logs_monitoring/.editorconfig b/aws/logs_monitoring/.editorconfig
@@ -5,3 +5,6 @@ indent_size = 4
 indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
+
+[*.yaml]
+indent_size = 2
diff --git a/aws/logs_monitoring/release.sh b/aws/logs_monitoring/release.sh
@@ -178,14 +178,12 @@ prod_release() {
     log_info "You are about to\n\t- bump the version from ${CURRENT_VERSION} to ${FORWARDER_VERSION}\n\t- create lambda layer version ${LAYER_VERSION}\n\t- create a release of aws-dd-forwarder-${FORWARDER_VERSION} on GitHub\n\t- upload the template.yaml to s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml\n"
 
     # Confirm to proceed
-    if user_confirm "Continue"; then
+    if ! user_confirm "Continue"; then
         log_error "Aborting..."
     fi
 
     # Get the latest code
     git pull origin master
-    GIT_COMMIT="$(git rev-parse --short HEAD)"
-    log_info "Using ${GIT_COMMIT} commit as the release target..."
 
     log_info "Bumping the version number to ${FORWARDER_VERSION}..."
     perl -pi -e "s/DD_FORWARDER_VERSION = \"[0-9\.]+/DD_FORWARDER_VERSION = \"${FORWARDER_VERSION}/g" "settings.py"
@@ -194,8 +192,16 @@ prod_release() {
     yq --inplace ".Mappings.Constants.DdForwarder.Version |= \"${FORWARDER_VERSION}\"" "template.yaml"
     yq --inplace ".Mappings.Constants.DdForwarder.LayerVersion |= \"${LAYER_VERSION}\"" "template.yaml"
 
-    # Confirm to proceed
-    if [[ ! -e ${BUNDLE_PATH} ]] || user_confirm "Bundle already exists. Do you want to use it" "true"; then
+    if git diff --quiet; then
+        log_info "Committing version number change..."
+        git add "settings.py" "template.yaml"
+        git commit --signoff --message "ci(release): Update version from ${CURRENT_VERSION} to ${FORWARDER_VERSION}"
+    fi
+
+    GIT_COMMIT="$(git rev-parse --short HEAD)"
+    log_info "Using ${GIT_COMMIT} commit as the release target..."
+
+    if [[ ! -e ${BUNDLE_PATH} ]] || ! user_confirm "Bundle already exists. Do you want to use it" "true"; then
         log_info "Building the Forwarder bundle..."
         ./tools/build_bundle.sh "${FORWARDER_VERSION}"
 

diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml
@@ -46,9 +46,7 @@ Parameters:
     ConstraintDescription: ReservedConcurrency must be an integer number
     Default: ""
     Description: >-
-      Reserved concurrency for the Datadog Forwarder Lambda function. If empty, use unreserved account concurrency.
-      We recommend using at least 10 reserved concurrency, but default to 0 as you may need to increase your limits for this.
-      If using unreserved account concurrency you may limit other lambda functions in your environment.
+      Reserved concurrency for the Datadog Forwarder Lambda function. If empty, use unreserved account concurrency. We recommend using at least 10 reserved concurrency, but default to 0 as you may need to increase your limits for this. If using unreserved account concurrency you may limit other lambda functions in your environment.
   LogRetentionInDays:
     Type: Number
     Default: 90
@@ -232,7 +230,7 @@ Parameters:
   DdForwarderBucketsAccessLogsTarget:
     Type: String
     Default: ""
-    Description: (Optional) The name of the S3 bucket to store access logs. Leave empty if access logging is not needed.    
+    Description: (Optional) The name of the S3 bucket to store access logs. Leave empty if access logging is not needed.
   DdStoreFailedEvents:
     Type: String
     Default: false
@@ -353,12 +351,12 @@ Conditions:
   CreateS3Bucket:
     Fn::And:
       - Fn::Or:
-        - Fn::Equals:
-            - Ref: DdFetchLogGroupTags
-            - true
-        - Fn::Equals:
-            - Ref: DdFetchLambdaTags
-            - true
+          - Fn::Equals:
+              - Ref: DdFetchLogGroupTags
+              - true
+          - Fn::Equals:
+              - Ref: DdFetchLambdaTags
+              - true
       - Fn::Equals:
           - Ref: DdForwarderExistingBucketName
           - ""
@@ -437,7 +435,7 @@ Conditions:
       - Fn::Equals:
           - Ref: ReservedConcurrency
           - ""
-  ShouldUseAccessLogBucket: !Not [!Equals [!Ref DdForwarderBucketsAccessLogsTarget, ""]]          
+  ShouldUseAccessLogBucket: !Not [!Equals [!Ref DdForwarderBucketsAccessLogsTarget, ""]]
   SetForwarderBucket:
     Fn::Or:
       - Condition: CreateS3Bucket
@@ -489,7 +487,6 @@ Resources:
                       - layer
                       - Datadog-Forwarder
                       - Fn::FindInMap: [Constants, DdForwarder, LayerVersion]
-
       Code:
         Fn::If:
           - UseZipCopier
@@ -501,12 +498,8 @@ Resources:
             S3Key:
               Fn::Sub:
                 - "aws-dd-forwarder-${DdForwarderVersion}.zip"
-                - {
-                    DdForwarderVersion:
-                      !FindInMap [Constants, DdForwarder, Version],
-                  }
+                - {DdForwarderVersion: !FindInMap [Constants, DdForwarder, Version]}
           - ZipFile: " "
-
       MemorySize:
         Ref: MemorySize
       Runtime: python3.11
@@ -690,7 +683,6 @@ Resources:
           - SecurityGroupIds: !Ref VPCSecurityGroupIds
             SubnetIds: !Ref VPCSubnetIds
           - Ref: AWS::NoValue
-
   ForwarderRole:
     Type: AWS::IAM::Role
     Properties:
@@ -725,12 +717,12 @@ Resources:
                       - s3:DeleteObject
                     Resource:
                       - Fn::If:
-                        - CreateS3Bucket
-                        - Fn::Join:
-                            - "/"
-                            - - Fn::GetAtt: ForwarderBucket.Arn
-                              - "*"
-                        - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}/*"
+                          - CreateS3Bucket
+                          - Fn::Join:
+                              - "/"
+                              - - Fn::GetAtt: ForwarderBucket.Arn
+                                - "*"
+                          - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}/*"
                     Effect: Allow
                   - Ref: AWS::NoValue
               # Get the actual log content from the s3 bucket based on the received s3 event.
@@ -741,9 +733,9 @@ Resources:
                       - s3:ListBucket
                     Resource:
                       - Fn::If:
-                        - CreateS3Bucket
-                        - Fn::GetAtt: ForwarderBucket.Arn
-                        - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}"
+                          - CreateS3Bucket
+                          - Fn::GetAtt: ForwarderBucket.Arn
+                          - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}"
                     Condition:
                       StringLike:
                         s3:prefix:
@@ -820,7 +812,6 @@ Resources:
               - DdForwarder
               - Version
           Key: dd_forwarder_version
-
   CloudWatchLogsPermission:
     Type: AWS::Lambda::Permission
     Properties:
@@ -890,20 +881,18 @@ Resources:
         BlockPublicPolicy: true
         IgnorePublicAcls: true
         RestrictPublicBuckets: true
-      LoggingConfiguration:
-        !If
-          - ShouldUseAccessLogBucket
-          -
-            DestinationBucketName: !Ref DdForwarderBucketsAccessLogsTarget
-            LogFilePrefix: "datadog-forwarder/"
-          - !Ref "AWS::NoValue"        
+      LoggingConfiguration: !If
+        - ShouldUseAccessLogBucket
+        - DestinationBucketName: !Ref DdForwarderBucketsAccessLogsTarget
+          LogFilePrefix: "datadog-forwarder/"
+        - !Ref "AWS::NoValue"
       LifecycleConfiguration:
-       Rules:
-       - Id: delete-incomplete-mpu-7days
-         Prefix: ''
-         AbortIncompleteMultipartUpload:
-           DaysAfterInitiation: 7
-         Status: Enabled
+        Rules:
+          - Id: delete-incomplete-mpu-7days
+            Prefix: ""
+            AbortIncompleteMultipartUpload:
+              DaysAfterInitiation: 7
+            Status: Enabled
   ForwarderBucketPolicy:
     Type: "AWS::S3::BucketPolicy"
     Condition: CreateS3Bucket
@@ -939,10 +928,7 @@ Resources:
           - !Ref SourceZipUrl
           - Fn::Sub:
               - "https://github.com/DataDog/datadog-serverless-functions/releases/download/aws-dd-forwarder-${DdForwarderVersion}/aws-dd-forwarder-${DdForwarderVersion}.zip"
-              - {
-                  DdForwarderVersion:
-                    !FindInMap [Constants, DdForwarder, Version],
-                }
+              - {DdForwarderVersion: !FindInMap [Constants, DdForwarder, Version]}
   # The Forwarder's source code is too big to fit the inline code size limit for CloudFormation. In most of AWS
   # partitions and regions, the Forwarder is able to load its source code from a Lambda layer attached to it.
   # In places where Datadog can't/doesn't yet publish Lambda layers, use another Lambda to copy the source code
@@ -1059,20 +1045,20 @@ Resources:
                   - s3:DeleteObject
                 Resource:
                   - Fn::If:
-                    - CreateS3Bucket
-                    - Fn::Join:
-                        - "/"
-                        - - Fn::GetAtt: "ForwarderBucket.Arn"
-                          - "*"
-                    - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}/*"
+                      - CreateS3Bucket
+                      - Fn::Join:
+                          - "/"
+                          - - Fn::GetAtt: "ForwarderBucket.Arn"
+                            - "*"
+                      - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}/*"
               - Effect: Allow
                 Action:
                   - s3:ListBucket
                 Resource:
                   - Fn::If:
-                    - CreateS3Bucket
-                    - Fn::GetAtt: "ForwarderBucket.Arn"
-                    - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}"
+                      - CreateS3Bucket
+                      - Fn::GetAtt: "ForwarderBucket.Arn"
+                      - Fn::Sub: "arn:aws:s3:::${DdForwarderExistingBucketName}"
               - !If
                 - SetS3SourceZip
                 - Effect: Allow

diff --git a/aws/logs_monitoring/tools/publish_prod.sh b/aws/logs_monitoring/tools/publish_prod.sh
@@ -2,47 +2,53 @@
 
 # Use with `./publish_prod.sh <DESIRED_NEW_VERSION>
 
-set -e
+set -o nounset -o pipefail -o errexit
 
-# Ensure on main, and pull the latest
-BRANCH=$(git rev-parse --abbrev-ref HEAD)
-if [ $BRANCH != "master" ]; then
-    echo "Not on master, aborting"
+log_info() {
+    local BLUE='\033[0;34m'
+    local RESET='\033[0m'
+
+    printf -- "%b%b%b\n" "${BLUE}" "${*}" "${RESET}" 1>&2
+}
+
+log_error() {
+    local RED='\033[0;31m'
+    local RESET='\033[0m'
+
+    printf -- "%b%b%b\n" "${RED}" "${*}" "${RESET}" 1>&2
     exit 1
-else
-    echo "Updating master"
-    git pull origin master
+}
+
+# Ensure on main, and pull the latest
+if [[ $(git rev-parse --abbrev-ref HEAD) != "master" ]]; then
+    log_error "Not on master, aborting"
 fi
 
 # Ensure no uncommitted changes
-if [ -n "$(git status --porcelain)" ]; then
-    echo "Detected uncommitted changes, aborting"
-    exit 1
+if ! git diff --quiet; then
+    log_error "Detected uncommitted changes, aborting"
 fi
 
 # Read the new version
-if [ -z "$1" ]; then
-    echo "Must specify a layer version"
-    exit 1
-else
-    LAYER_VERSION=$1
+if [[ -z ${1:-} ]]; then
+    log_error "Must specify a layer version"
 fi
 
+LAYER_VERSION=$1
+
 # Read the new version
-if [ -z "$2" ]; then
-    echo "Must specify a forwarder version"
-    exit 1
-else
-    FORWARDER_VERSION=$2
+if [[ -z ${2:-} ]]; then
+    log_error "Must specify a forwarder version"
 fi
 
+FORWARDER_VERSION=$2
+
 # Ensure AWS access before proceeding
 aws-vault exec sso-govcloud-us1-fed-engineering -- aws sts get-caller-identity
 aws-vault exec sso-prod-engineering -- aws sts get-caller-identity
 
-echo "Publishing layers to GovCloud AWS regions"
+log_info "Publishing layers to GovCloud AWS regions"
 LAYER_VERSION="${LAYER_VERSION}" FORWARDER_VERSION"=${FORWARDER_VERSION}" aws-vault exec sso-govcloud-us1-fed-engineering -- ./tools/publish_layers.sh
 
-echo
-echo "Publishing layers to commercial AWS regions"
+log_info "Publishing layers to commercial AWS regions"
 LAYER_VERSION="${LAYER_VERSION}" FORWARDER_VERSION"=${FORWARDER_VERSION}" aws-vault exec sso-prod-engineering -- ./tools/publish_layers.sh
diff --git a/aws/logs_monitoring/trace_forwarder/README.md b/aws/logs_monitoring/trace_forwarder/README.md
@@ -1,11 +1,12 @@
 # Datadog Trace Forwarder
 
-Shared libary for submitting traces to trace intake. 
+Shared libary for submitting traces to trace intake.
 Features include:
- * Tools for building AWS Lambda Layer from library
- * Trace obfuscation, (using logic from datadog-agent)
- * Submits Stats/Transcations for traces
- * Python bindings
+
+-   Tools for building AWS Lambda Layer from library
+-   Trace obfuscation, (using logic from datadog-agent)
+-   Submits Stats/Transcations for traces
+-   Python bindings
 
 ```python
 from trace_forwarder.connection import TraceConnection
@@ -42,9 +43,9 @@ conn.send("""
 
 ## Requirements
 
-* dep
-* go 1.12 or higher
-* docker
+-   dep
+-   go 1.12 or higher
+-   docker
 
 ## Building Go Binary
 
@@ -58,24 +59,3 @@ Output is saved to bin, and the shared library will be compatible with your loca
 ```bash
 ./scripts/build_linux_go_bin.sh
 ```
-
-## Lambda Layer
-### Building Lambda Layer
-
-You can build the lambda layer with the following command
-
-```bash
-./scripts/build_layers.sh
-```
-
-### Publishing to staging
-
-```bash
-./scripts/publish_staging.sh
-```
-
-### Publishing to prod
-
-```bash
-./scripts/publish_prod.sh
-```