diff --git a/aws/logs_monitoring/.editorconfig b/aws/logs_monitoring/.editorconfig new file mode 100644 index 00000000..6b92b381 --- /dev/null +++ b/aws/logs_monitoring/.editorconfig @@ -0,0 +1,7 @@ +[*] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/aws/logs_monitoring/release.sh b/aws/logs_monitoring/release.sh index 1aeb5fad..580c112a 100755 --- a/aws/logs_monitoring/release.sh +++ b/aws/logs_monitoring/release.sh @@ -1,197 +1,263 @@ -#!/bin/bash +#!/usr/bin/env bash -# Usage: ./release.sh +set -o nounset -o pipefail -o errexit -set -e +log_info() { + local BLUE='\033[0;34m' + local RESET='\033[0m' -LAYER_NAME="Datadog-Forwarder" + printf -- "%b%b%b\n" "${BLUE}" "${*}" "${RESET}" 1>&2 +} -# Read the current version -CURRENT_VERSION=$(grep -E -o 'Version: [0-9]+\.[0-9]+\.[0-9]+' template.yaml | cut -d' ' -f2) +log_success() { + local GREEN='\033[0;32m' + local RESET='\033[0m' -# Read the desired version -if [ -z "$1" ]; then - echo "ERROR: You must specify a desired version number" - exit 1 -elif [[ ! $1 =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then - echo "ERROR: You must use a semantic version (e.g. 3.1.4)" + printf -- "%b%b%b\n" "${GREEN}" "${*}" "${RESET}" 1>&2 +} + +log_error() { + local RED='\033[0;31m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${RED}" "${*}" "${RESET}" 1>&2 exit 1 -else - FORWARDER_VERSION=$1 +} + +user_read_input() { + if [[ ${#} -lt 2 ]]; then + var_red "Usage: var_read_input VAR_PROMPT VAR_NAME" + return 1 + fi + + local VAR_PROMPT="${1}" + shift + + local VAR_NAME="${1}" + shift + + if [[ -n ${BASH_VERSION} ]]; then + read -r -p "${VAR_PROMPT}" "${@}" "${VAR_NAME}" /dev/null 2>&1; then + log_error "jq not found, please install it following instructions here https://github.com/jqlang/jq#installation" +fi -# Check account parameter -VALID_ACCOUNTS=("sandbox" "prod") -if [ -z "$2" ]; then - echo "ERROR: You must pass an account parameter. Please choose sandbox or prod." - exit 1 +if ! command -v yq >/dev/null 2>&1; then + log_error "yq not found, please install it following instructions here https://github.com/mikefarah/yq#install" fi -if [[ ! "${VALID_ACCOUNTS[@]}" =~ $2 ]]; then - echo "ERROR: The account parameter was invalid. Please choose sandbox or prod." - exit 1 + +if ! command -v hub >/dev/null 2>&1; then + log_error "hub not found, please install it following instructions here https://github.com/github/hub#installation" +fi + +# Read the desired version +if [[ ! ${1} =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then + log_error "You must use a semantic version (e.g. 3.1.4)" +else + FORWARDER_VERSION="${1}" fi +# Check account parameter +VALID_ACCOUNTS=("sandbox" "prod") +if [[ ! ${VALID_ACCOUNTS[@]} =~ ${2} ]]; then + log_error "The account parameter was invalid. Please choose sandbox or prod." +fi ACCOUNT="${2}" -if [ "$ACCOUNT" = "sandbox" ]; then - if [ "$DEPLOY_TO_SERVERLESS_SANDBOX" = "true" ] ; then + +if [[ ${ACCOUNT} == "prod" ]]; then + BUCKET="datadog-cloudformation-template" +elif [[ ${ACCOUNT} == "sandbox" ]]; then + if [[ ${DEPLOY_TO_SERVERLESS_SANDBOX:-} == "true" ]]; then BUCKET="datadog-cloudformation-template-serverless-sandbox" else BUCKET="datadog-cloudformation-template-sandbox" fi fi -if [ "$ACCOUNT" = "prod" ]; then - BUCKET="datadog-cloudformation-template" -fi -function aws-login() { - cfg=( "$@" ) +# Read the current version +CURRENT_VERSION=$(yq '.Mappings.Constants.DdForwarder.Version' "template.yaml") + +LAYER_NAME="Datadog-Forwarder" +BUNDLE_PATH=".forwarder/aws-dd-forwarder-${FORWARDER_VERSION}.zip" + +aws_login() { + cfg=("$@") shift - if [ "$ACCOUNT" = "prod" ] ; then - aws-vault exec sso-prod-engineering -- ${cfg[@]} + + if [[ ${ACCOUNT} == "prod" ]]; then + aws-vault exec sso-prod-engineering -- ${cfg[@]} else - if [ "$DEPLOY_TO_SERVERLESS_SANDBOX" = "true" ] ; then - aws-vault exec sso-serverless-sandbox-account-admin -- ${cfg[@]} + if [[ ${DEPLOY_TO_SERVERLESS_SANDBOX:-} == "true" ]]; then + aws-vault exec sso-serverless-sandbox-account-admin -- ${cfg[@]} else - aws-vault exec sso-sandbox-account-admin -- ${cfg[@]} + aws-vault exec sso-sandbox-account-admin -- ${cfg[@]} fi fi } get_max_layer_version() { - last_layer_version=$(aws-login aws lambda list-layer-versions --layer-name $LAYER_NAME --region us-west-2 | jq -r ".LayerVersions | .[0] | .Version") + last_layer_version=$(aws_login aws lambda list-layer-versions --layer-name "${LAYER_NAME}" --region us-west-2 | jq -r ".LayerVersions | .[0] | .Version") if [ "$last_layer_version" == "null" ]; then echo 0 else - echo $last_layer_version + echo "${last_layer_version}" fi } -# Validate identity -aws-login aws sts get-caller-identity - -CURRENT_ACCOUNT="$(aws-login aws sts get-caller-identity --query Account --output text)" -CURRENT_LAYER_VERSION=$(get_max_layer_version) -LAYER_VERSION=$(($CURRENT_LAYER_VERSION +1)) +sandbox_release() { + if [[ ! -e ${BUNDLE_PATH} ]] || ! user_confirm "Bundle already exists. Do you want to use it" "true"; then + log_info "Building the Forwarder bundle..." + ./tools/build_bundle.sh "${FORWARDER_VERSION}" -echo -echo "Current layer version is $CURRENT_LAYER_VERSION, next layer version is $LAYER_VERSION" + log_info "Signing the Forwarder bundle..." + aws_login "./tools/sign_bundle.sh" "${BUNDLE_PATH}" "${ACCOUNT}" + fi -# Validate the template -echo -echo "Validating template.yaml..." -aws-login aws cloudformation validate-template --template-body file://template.yaml + # Upload the bundle to S3 instead of GitHub for a sandbox release + log_info "Uploading non-public sandbox version of Forwarder to S3..." + aws_login aws s3 cp "${BUNDLE_PATH}" "s3://${BUCKET}/aws/forwarder-staging-zip/aws-dd-forwarder-${FORWARDER_VERSION}.zip" + # Upload the sandbox layers + log_info "Uploading layers" + ./tools/publish_sandbox.sh "${LAYER_VERSION}" "${FORWARDER_VERSION}" -if [ "$ACCOUNT" = "prod" ] ; then + # Set vars for use in the installation test + TEMPLATE_URL="https://${BUCKET}.s3.amazonaws.com/aws/forwarder-staging/latest.yaml" + FORWARDER_SOURCE_URL="s3://${BUCKET}/aws/forwarder-staging-zip/aws-dd-forwarder-${FORWARDER_VERSION}.zip" +} - if [[ ! $(./tools/semver.sh "$FORWARDER_VERSION" "$CURRENT_VERSION") > 0 ]]; then - echo "Must use a version greater than the current ($CURRENT_VERSION)" - exit 1 +prod_release() { + if [[ ! $(./tools/semver.sh "${FORWARDER_VERSION}" "${CURRENT_VERSION}") > 0 ]]; then + log_error "Must use a version greater than the current ($CURRENT_VERSION)" fi # Make sure we are on the master branch - BRANCH=$(git rev-parse --abbrev-ref HEAD) - if [ $BRANCH != "master" ]; then - echo "ERROR: Not on the master branch, aborting." - exit 1 + if [[ $(git rev-parse --abbrev-ref HEAD) != "master" ]]; then + log_error "Not on the master branch, aborting." fi + log_info "You are about to\n\t- bump the version from ${CURRENT_VERSION} to ${FORWARDER_VERSION}\n\t- create lambda layer version ${LAYER_VERSION}\n\t- create a release of aws-dd-forwarder-${FORWARDER_VERSION} on GitHub\n\t- upload the template.yaml to s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml\n" + # Confirm to proceed - echo - read -p "About to bump the version from ${CURRENT_VERSION} to ${FORWARDER_VERSION}, create a release of aws-dd-forwarder-${FORWARDER_VERSION} on GitHub, upload the template.yaml to s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml and create lambda layer version ${LAYER_VERSION}. Continue (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Exiting..." - exit 1 + if user_confirm "Continue"; then + log_error "Aborting..." fi # Get the latest code git pull origin master + GIT_COMMIT="$(git rev-parse --short HEAD)" + log_info "Using ${GIT_COMMIT} commit as the release target..." - # Bump version number in settings.py and template.yml - echo "Bumping the version number to ${FORWARDER_VERSION}..." - perl -pi -e "s/DD_FORWARDER_VERSION = \"[0-9\.]+/DD_FORWARDER_VERSION = \"${FORWARDER_VERSION}/g" settings.py - perl -pi -e "s/Version: [0-9\.]+/Version: ${FORWARDER_VERSION}/g" template.yaml - perl -pi -e "s/LayerVersion: [0-9\.]+/LayerVersion: ${LAYER_VERSION}/g" template.yaml + log_info "Bumping the version number to ${FORWARDER_VERSION}..." + perl -pi -e "s/DD_FORWARDER_VERSION = \"[0-9\.]+/DD_FORWARDER_VERSION = \"${FORWARDER_VERSION}/g" "settings.py" - # Commit version number changes to git - echo "Committing version number change..." - git add settings.py template.yaml - git commit -m "Bump version from ${CURRENT_VERSION} to ${FORWARDER_VERSION}" - git push origin master + # Update template.yaml + yq --inplace ".Mappings.Constants.DdForwarder.Version |= \"${FORWARDER_VERSION}\"" "template.yaml" + yq --inplace ".Mappings.Constants.DdForwarder.LayerVersion |= \"${LAYER_VERSION}\"" "template.yaml" - # Build the bundle - echo - echo "Building the Forwarder bundle..." - ./tools/build_bundle.sh "${FORWARDER_VERSION}" + # Confirm to proceed + if [[ ! -e ${BUNDLE_PATH} ]] || user_confirm "Bundle already exists. Do you want to use it" "true"; then + log_info "Building the Forwarder bundle..." + ./tools/build_bundle.sh "${FORWARDER_VERSION}" + + log_info "Signing the Forwarder bundle..." + aws_login ./tools/sign_bundle.sh "${BUNDLE_PATH}" "${ACCOUNT}" + fi - # Sign the bundle - echo - echo "Signing the Forwarder bundle..." - aws-login ./tools/sign_bundle.sh $BUNDLE_PATH $ACCOUNT + log_info "Uploading layers..." + ./tools/publish_prod.sh "${LAYER_VERSION}" "${FORWARDER_VERSION}" + + log_info "Committing version number change..." + git add "settings.py" "template.yaml" + git commit --signoff --message "ci(release): Update version from ${CURRENT_VERSION} to ${FORWARDER_VERSION}" + git push origin master # Create a GitHub release - echo - echo "Releasing aws-dd-forwarder-${FORWARDER_VERSION} to GitHub..." - go get github.com/github/hub - hub release create -a $BUNDLE_PATH -m "aws-dd-forwarder-${FORWARDER_VERSION}" aws-dd-forwarder-${FORWARDER_VERSION} - - # Upload the prod layers - echo - echo "Uploading layers" - ./tools/publish_prod.sh $LAYER_VERSION $FORWARDER_VERSION + log_info "Releasing aws-dd-forwarder-${FORWARDER_VERSION} to GitHub..." + hub release create -a "${BUNDLE_PATH}" -m "aws-dd-forwarder-${FORWARDER_VERSION}" -t "${GIT_COMMIT}" "aws-dd-forwarder-${FORWARDER_VERSION}" # Set vars for use in the installation test TEMPLATE_URL="https://${BUCKET}.s3.amazonaws.com/aws/forwarder/latest.yaml" FORWARDER_SOURCE_URL="https://github.com/DataDog/datadog-serverless-functions/releases/download/aws-dd-forwarder-${FORWARDER_VERSION}/aws-dd-forwarder-${FORWARDER_VERSION}.zip'" -else - # Build the bundle - echo - echo "Building the Forwarder bundle..." - ./tools/build_bundle.sh $FORWARDER_VERSION +} - # Sign the bundle - echo - echo "Signing the Forwarder bundle..." - aws-login ./tools/sign_bundle.sh $BUNDLE_PATH $ACCOUNT +# Validate identity +aws_login aws sts get-caller-identity - # Upload the bundle to S3 instead of GitHub for a sandbox release - echo - echo "Uploading non-public sandbox version of Forwarder to S3..." - aws-login aws s3 cp $BUNDLE_PATH s3://${BUCKET}/aws/forwarder-staging-zip/aws-dd-forwarder-${FORWARDER_VERSION}.zip - - # Upload the sandbox layers - echo - echo "Uploading layers" - ./tools/publish_sandbox.sh $LAYER_VERSION $FORWARDER_VERSION +CURRENT_ACCOUNT="$(aws_login aws sts get-caller-identity --query Account --output text)" +CURRENT_LAYER_VERSION=$(get_max_layer_version) +LAYER_VERSION=$((CURRENT_LAYER_VERSION + 1)) - # Set vars for use in the installation test - TEMPLATE_URL="https://${BUCKET}.s3.amazonaws.com/aws/forwarder-staging/latest.yaml" - FORWARDER_SOURCE_URL="s3://${BUCKET}/aws/forwarder-staging-zip/aws-dd-forwarder-${FORWARDER_VERSION}.zip" -fi +log_info "Current layer version is ${CURRENT_LAYER_VERSION}, next layer version will be ${LAYER_VERSION}" -# Upload the CloudFormation template to the S3 bucket -echo -echo "Uploading template.yaml to s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml" +log_info "Validating template.yaml..." +aws_login aws cloudformation validate-template --template-body "file://template.yaml" -if [ "$ACCOUNT" = "prod" ] ; then - aws-login aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml \ - --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers - aws-login aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder/latest.yaml \ - --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers +if [[ ${ACCOUNT} == "prod" ]]; then + prod_release else - aws-login aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder-staging/${FORWARDER_VERSION}.yaml - aws-login aws s3 cp template.yaml s3://${BUCKET}/aws/forwarder-staging/latest.yaml + sandbox_release fi -echo "Done uploading the CloudFormation template!" -echo -echo "Here is the CloudFormation quick launch URL:" -echo "https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=datadog-serverless&templateURL=${TEMPLATE_URL}" -echo -echo "Forwarder release process complete!" -if [ "$ACCOUNT" = "prod" ] ; then - echo "Don't forget to add release notes in GitHub!" +log_info "Uploading template.yaml to s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml" + +if [[ ${ACCOUNT} == "prod" ]]; then + aws_login aws s3 cp template.yaml "s3://${BUCKET}/aws/forwarder/${FORWARDER_VERSION}.yaml" \ + --grants "read=uri=http://acs.amazonaws.com/groups/global/AllUsers" + aws_login aws s3 cp template.yaml "s3://${BUCKET}/aws/forwarder/latest.yaml" \ + --grants "read=uri=http://acs.amazonaws.com/groups/global/AllUsers" +else + aws_login aws s3 cp template.yaml "s3://${BUCKET}/aws/forwarder-staging/${FORWARDER_VERSION}.yaml" + aws_login aws s3 cp template.yaml "s3://${BUCKET}/aws/forwarder-staging/latest.yaml" fi +log_success "Done uploading the CloudFormation template!" + +log_info "Here is the CloudFormation quick launch URL:" +log_info "https://console.aws.amazon.com/cloudformation/home#/stacks/new?stackName=datadog-serverless&templateURL=${TEMPLATE_URL}" + +log_success "" +log_success "Forwarder release process complete!" + +if [[ ${ACCOUNT} == "prod" ]]; then + log_info "Don't forget to add release notes in GitHub!" +fi diff --git a/aws/logs_monitoring/tools/build_bundle.sh b/aws/logs_monitoring/tools/build_bundle.sh index 172b4af8..434d0ac8 100755 --- a/aws/logs_monitoring/tools/build_bundle.sh +++ b/aws/logs_monitoring/tools/build_bundle.sh @@ -5,7 +5,29 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc -set -e +set -o nounset -o pipefail -o errexit + +log_info() { + local BLUE='\033[0;34m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${BLUE}" "${*}" "${RESET}" 1>&2 +} + +log_success() { + local GREEN='\033[0;32m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${GREEN}" "${*}" "${RESET}" 1>&2 +} + +log_error() { + local RED='\033[0;31m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${RED}" "${*}" "${RESET}" 1>&2 + exit 1 +} # Move into the tools directory DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" @@ -13,67 +35,64 @@ cd $DIR # Read the desired version if [ -z "$1" ]; then - echo "Must specify a desired version number" - exit 1 + log_error "Must specify a desired version number" elif [[ ! $1 =~ [0-9]+\.[0-9]+\.[0-9]+ ]]; then - echo "Must use a semantic version, e.g., 3.1.4" - exit 1 + log_error "Must use a semantic version, e.g., 3.1.4" else - VERSION=$1 + VERSION=$1 fi PYTHON_VERSION="${PYTHON_VERSION:-3.11}" FORWARDER_PREFIX="aws-dd-forwarder" FORWARDER_DIR="../.forwarder" -function make_path_absolute { - echo "$( - cd "$(dirname "$1")" - pwd - )/$(basename "$1")" +make_path_absolute() { + echo "$( + cd "$(dirname "$1")" + pwd + )/$(basename "$1")" } ../trace_forwarder/scripts/build_linux_go_bin.sh -function docker_build_zip { - # Args: [python version] [zip destination] - zip_destination=$(make_path_absolute $2) - layer_destination=$(make_path_absolute $3) +docker_build_zip() { + # Args: [python version] [zip destination] + zip_destination=$(make_path_absolute "${2}") + layer_destination=$(make_path_absolute "${3}") - # Install datadogpy in a docker container to avoid the mess from switching - # between different python runtimes. - temp_dir=$(mktemp -d) + # Install datadogpy in a docker container to avoid the mess from switching + # between different python runtimes. + temp_dir=$(mktemp -d) - docker buildx build --platform linux/amd64 --file "${DIR}/Dockerfile_bundle" -t "datadog-bundle:$1" .. --no-cache \ - --build-arg runtime=$PYTHON_VERSION + docker buildx build --platform linux/amd64 --file "${DIR}/Dockerfile_bundle" -t "datadog-bundle:$1" .. --no-cache --build-arg "runtime=${PYTHON_VERSION}" - # Run the image by runtime tag, tar its generated `python` directory to sdout, - # then extract it to a temp directory. - docker run --platform linux/amd64 datadog-bundle:$1 tar cf - . | tar -xf - -C $temp_dir + # Run the image by runtime tag, tar its generated `python` directory to sdout, + # then extract it to a temp directory. + docker run --platform linux/amd64 "datadog-bundle:${1}" tar cf - . | tar -xf - -C "${temp_dir}" - # Zip to destination, and keep directory structure as based in $temp_dir - (cd $temp_dir && zip -q -r $zip_destination ./) + # Zip to destination, and keep directory structure as based in $temp_dir + (cd "${temp_dir}" && zip -q -r "${zip_destination}" ./) - rm -rf $temp_dir - echo "Done creating forwarder zip archive $zip_destination" + rm -rf "${temp_dir}" + log_success "Done creating forwarder zip archive $zip_destination" - temp_dir=$(mktemp -d) - SUB_DIRECTORY=python - mkdir $temp_dir/$SUB_DIRECTORY + temp_dir=$(mktemp -d) + SUB_DIRECTORY=python + mkdir "${temp_dir}/${SUB_DIRECTORY}" - # Run the image by runtime tag, tar its generated `python` directory to sdout, - # then extract it to a temp directory. - docker run --platform linux/amd64 datadog-bundle:$1 tar cf - . | tar -xf - -C $temp_dir/$SUB_DIRECTORY + # Run the image by runtime tag, tar its generated `python` directory to sdout, + # then extract it to a temp directory. + docker run --platform linux/amd64 datadog-bundle:$1 tar cf - . | tar -xf - -C $temp_dir/$SUB_DIRECTORY - # Zip to destination, and keep directory structure as based in $temp_dir - (cd $temp_dir && zip -q -r $layer_destination ./) - echo "Done creating layer zip archive $layer_destination" + # Zip to destination, and keep directory structure as based in $temp_dir + (cd "${temp_dir}" && zip -q -r "${layer_destination}" ./) + log_success "Done creating layer zip archive $layer_destination" } -rm -rf $FORWARDER_DIR -mkdir $FORWARDER_DIR +rm -rf "${FORWARDER_DIR}" +mkdir "${FORWARDER_DIR}" -docker_build_zip ${PYTHON_VERSION} ${FORWARDER_DIR}/${FORWARDER_PREFIX}-${VERSION}.zip ${FORWARDER_DIR}/${FORWARDER_PREFIX}-${VERSION}-layer.zip +docker_build_zip "${PYTHON_VERSION}" "${FORWARDER_DIR}/${FORWARDER_PREFIX}-${VERSION}.zip" "${FORWARDER_DIR}/${FORWARDER_PREFIX}-${VERSION}-layer.zip" -echo "Successfully created Forwarder bundle!" -ls $FORWARDER_DIR | xargs -I _ echo "${FORWARDER_DIR}/_" +log_success "Successfully created Forwarder bundle!" +ls "${FORWARDER_DIR}" | xargs -I _ echo "${FORWARDER_DIR}/_" diff --git a/aws/logs_monitoring/tools/installation_test.sh b/aws/logs_monitoring/tools/installation_test.sh index 60024490..9b9cc122 100755 --- a/aws/logs_monitoring/tools/installation_test.sh +++ b/aws/logs_monitoring/tools/installation_test.sh @@ -12,9 +12,9 @@ set -e # Deploy the stack to a less commonly used region to avoid any problems with limits if [ "$DEPLOY_TO_SERVERLESS_SANDBOX" = "true" ]; then - AWS_REGION="sa-east-1" + AWS_REGION="sa-east-1" else - AWS_REGION="us-west-2" + AWS_REGION="us-west-2" fi # Limits any layer publishing to the test region @@ -34,49 +34,49 @@ DD_API_KEY=RUN_ID CURRENT_VERSION="$(grep -E -o 'Version: [0-9]+\.[0-9]+\.[0-9]+' template.yaml | cut -d' ' -f2)" function aws-login() { - cfg=("$@") - shift - if [ "$ACCOUNT" = "prod" ]; then - aws-vault exec sso-prod-engineering -- ${cfg[@]} + cfg=("$@") + shift + if [ "$ACCOUNT" = "prod" ]; then + aws-vault exec sso-prod-engineering -- ${cfg[@]} + else + if [ "$DEPLOY_TO_SERVERLESS_SANDBOX" = "true" ]; then + aws-vault exec sso-serverless-sandbox-account-admin -- ${cfg[@]} else - if [ "$DEPLOY_TO_SERVERLESS_SANDBOX" = "true" ]; then - aws-vault exec sso-serverless-sandbox-account-admin -- ${cfg[@]} - else - aws-vault exec sso-sandbox-account-admin -- ${cfg[@]} - fi + aws-vault exec sso-sandbox-account-admin -- ${cfg[@]} fi + fi } # Run script in this process. This gives us TEMPLATE_URL, NEXT_LAYER_VERSION and FORWARDER_SOURCE_URL env vars . release.sh $CURRENT_VERSION sandbox function param { - KEY=$1 - VALUE=$2 - echo "{\"ParameterKey\":\"${KEY}\",\"ParameterValue\":${VALUE}}" + KEY=$1 + VALUE=$2 + echo "{\"ParameterKey\":\"${KEY}\",\"ParameterValue\":${VALUE}}" } echo $FORWARDER_SOURCE_URL publish_test() { - ADDED_PARAMS=$1 + ADDED_PARAMS=$1 - PARAM_LIST=[$(param DdApiKey \"${DD_API_KEY}\"),$(param DdSite \"datadoghq.com\"),$(param ReservedConcurrency \"1\"),$ADDED_PARAMS] - echo "Setting params ${PARAM_LIST}" + PARAM_LIST=[$(param DdApiKey \"${DD_API_KEY}\"),$(param DdSite \"datadoghq.com\"),$(param ReservedConcurrency \"1\"),$ADDED_PARAMS] + echo "Setting params ${PARAM_LIST}" - # Create an instance of the stack - STACK_NAME="datadog-forwarder-integration-stack-${RUN_ID}" + # Create an instance of the stack + STACK_NAME="datadog-forwarder-integration-stack-${RUN_ID}" - echo "Creating stack using Zip Copier Flow ${STACK_NAME}" - aws-login aws cloudformation create-stack --stack-name $STACK_NAME --template-url $TEMPLATE_URL --capabilities "CAPABILITY_AUTO_EXPAND" "CAPABILITY_IAM" --on-failure "DELETE" \ - --parameters=$PARAM_LIST --region $AWS_REGION + echo "Creating stack using Zip Copier Flow ${STACK_NAME}" + aws-login aws cloudformation create-stack --stack-name $STACK_NAME --template-url $TEMPLATE_URL --capabilities "CAPABILITY_AUTO_EXPAND" "CAPABILITY_IAM" --on-failure "DELETE" \ + --parameters=$PARAM_LIST --region $AWS_REGION - echo "Waiting for stack to complete creation ${STACK_NAME}" - aws-login aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $AWS_REGION + echo "Waiting for stack to complete creation ${STACK_NAME}" + aws-login aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $AWS_REGION - echo "Completed stack creation" + echo "Completed stack creation" - echo "Cleaning up stack" - aws-login aws cloudformation delete-stack --stack-name $STACK_NAME --region $AWS_REGION + echo "Cleaning up stack" + aws-login aws cloudformation delete-stack --stack-name $STACK_NAME --region $AWS_REGION } echo diff --git a/aws/logs_monitoring/tools/publish_layers.sh b/aws/logs_monitoring/tools/publish_layers.sh index 264e7540..b5858fcf 100755 --- a/aws/logs_monitoring/tools/publish_layers.sh +++ b/aws/logs_monitoring/tools/publish_layers.sh @@ -8,7 +8,86 @@ # Publish the datadog forwarder layer across regions, using the AWS CLI # Usage: VERSION=5 REGIONS=us-east-1 LAYERS=Datadog-Python27 publish_layers.sh # VERSION is required. -set -e + +set -o nounset -o pipefail -o errexit + +log_info() { + local BLUE='\033[0;34m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${BLUE}" "${*}" "${RESET}" 1>&2 +} + +log_success() { + local GREEN='\033[0;32m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${GREEN}" "${*}" "${RESET}" 1>&2 +} + +log_warning() { + local YELLOW='\033[0;33m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${YELLOW}" "${*}" "${RESET}" 1>&2 +} + +log_error() { + local RED='\033[0;31m' + local RESET='\033[0m' + + printf -- "%b%b%b\n" "${RED}" "${*}" "${RESET}" 1>&2 + exit 1 +} + +user_read_input() { + if [[ ${#} -lt 2 ]]; then + var_red "Usage: var_read_input VAR_PROMPT VAR_NAME" + return 1 + fi + + local VAR_PROMPT="${1}" + shift + + local VAR_NAME="${1}" + shift + + if [[ -n ${BASH_VERSION} ]]; then + read -r -p "${VAR_PROMPT}" "${@}" "${VAR_NAME}" /dev/null + + echo "${version_nbr}" } for region in $REGIONS; do - echo "Starting publishing layer for region $region..." - # Publish the layers for each version of python - for layer_name in "${LAYERS[@]}"; do - latest_version=$(aws lambda list-layer-versions --region $region --layer-name $layer_name --query 'LayerVersions[0].Version || `0`') - if [ $latest_version -ge $LAYER_VERSION ]; then - echo "Layer $layer_name version $LAYER_VERSION already exists in region $region, skipping..." - continue - elif [ $latest_version -lt $((LAYER_VERSION - 1)) ]; then - read -p "WARNING: The latest version of layer $layer_name in region $region is $latest_version, publish all the missing versions including $LAYER_VERSION or EXIT the script (y/n)?" CONT - if [ "$CONT" != "y" ]; then - echo "Exiting" - exit 1 - fi - fi - - index=$(index_of_layer $layer_name) - aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}" - layer_path="${LAYER_PATHS[$index]}" - - while [ $latest_version -lt $LAYER_VERSION ]; do - latest_version=$(publish_layer $region $layer_name $aws_version_key $layer_path) - echo "Published version $latest_version for layer $layer_name in region $region" - - # This shouldn't happen unless someone manually deleted the latest version, say 28 - # and then try to republish it again. The published version is actually be 29, because - # Lambda layers are immutable and AWS will skip deleted version and use the next number. - if [ $latest_version -gt $LAYER_VERSION ]; then - echo "ERROR: Published version $latest_version is greater than the desired version $LAYER_VERSION!" - echo "Exiting" - exit 1 - fi - done + log_info "Starting publishing layer for region $region..." + + # Publish the layers for each version of python + for layer_name in "${LAYERS[@]}"; do + latest_version=$(aws lambda list-layer-versions --region "${region}" --layer-name "${layer_name}" --query 'LayerVersions[0].Version || `0`') + if [[ ${latest_version} -ge ${LAYER_VERSION} ]]; then + log_warning "Layer $layer_name version $LAYER_VERSION already exists in region $region, skipping..." + continue + elif [[ ${latest_version} -lt $((LAYER_VERSION - 1)) ]]; then + log_warning "The latest version of layer ${layer_name} in region ${region} is ${latest_version}" + + if ! user_confirm "Publish all the missing versions including ${LAYER_VERSION}" "true"; then + log_error "Exiting" + fi + fi + + index=$(index_of_layer "${layer_name}") + aws_version_key="${PYTHON_VERSIONS_FOR_AWS_CLI[$index]}" + layer_path="${LAYER_PATHS[$index]}" + + while [[ ${latest_version} -lt ${LAYER_VERSION} ]]; do + latest_version=$(publish_layer "${region}" "${layer_name}" "${aws_version_key}" "${layer_path}") + log_success "Published version $latest_version for layer $layer_name in region $region" + + # This shouldn't happen unless someone manually deleted the latest version, say 28 + # and then try to republish it again. The published version is actually be 29, because + # Lambda layers are immutable and AWS will skip deleted version and use the next number. + if [[ ${latest_version} -gt ${LAYER_VERSION} ]]; then + log_error "Published version ${latest_version} is greater than the desired version ${LAYER_VERSION}!" + fi done + done done echo "Done !" diff --git a/aws/logs_monitoring/tools/publish_prod.sh b/aws/logs_monitoring/tools/publish_prod.sh index f6ad7219..428cf42d 100755 --- a/aws/logs_monitoring/tools/publish_prod.sh +++ b/aws/logs_monitoring/tools/publish_prod.sh @@ -7,33 +7,33 @@ set -e # Ensure on main, and pull the latest BRANCH=$(git rev-parse --abbrev-ref HEAD) if [ $BRANCH != "master" ]; then - echo "Not on master, aborting" - exit 1 + echo "Not on master, aborting" + exit 1 else - echo "Updating master" - git pull origin master + echo "Updating master" + git pull origin master fi # Ensure no uncommitted changes if [ -n "$(git status --porcelain)" ]; then - echo "Detected uncommitted changes, aborting" - exit 1 + echo "Detected uncommitted changes, aborting" + exit 1 fi # Read the new version if [ -z "$1" ]; then - echo "Must specify a layer version" - exit 1 + echo "Must specify a layer version" + exit 1 else - LAYER_VERSION=$1 + LAYER_VERSION=$1 fi # Read the new version if [ -z "$2" ]; then - echo "Must specify a forwarder version" - exit 1 + echo "Must specify a forwarder version" + exit 1 else - FORWARDER_VERSION=$2 + FORWARDER_VERSION=$2 fi # Ensure AWS access before proceeding @@ -41,8 +41,8 @@ aws-vault exec sso-govcloud-us1-fed-engineering -- aws sts get-caller-identity aws-vault exec sso-prod-engineering -- aws sts get-caller-identity echo "Publishing layers to GovCloud AWS regions" -LAYER_VERSION=$LAYER_VERSION FORWARDER_VERSION=$FORWARDER_VERSION aws-vault exec sso-govcloud-us1-fed-engineering -- ./tools/publish_layers.sh +LAYER_VERSION="${LAYER_VERSION}" FORWARDER_VERSION"=${FORWARDER_VERSION}" aws-vault exec sso-govcloud-us1-fed-engineering -- ./tools/publish_layers.sh echo echo "Publishing layers to commercial AWS regions" -LAYER_VERSION=$LAYER_VERSION FORWARDER_VERSION=$FORWARDER_VERSION aws-vault exec sso-prod-engineering -- ./tools/publish_layers.sh +LAYER_VERSION="${LAYER_VERSION}" FORWARDER_VERSION"=${FORWARDER_VERSION}" aws-vault exec sso-prod-engineering -- ./tools/publish_layers.sh diff --git a/aws/logs_monitoring/tools/publish_sandbox.sh b/aws/logs_monitoring/tools/publish_sandbox.sh index 099e5648..8198b82c 100755 --- a/aws/logs_monitoring/tools/publish_sandbox.sh +++ b/aws/logs_monitoring/tools/publish_sandbox.sh @@ -3,18 +3,18 @@ set -e # Read the new version if [ -z "$1" ]; then - echo "Must specify a desired version number" - exit 1 + echo "Must specify a desired version number" + exit 1 else - LAYER_VERSION=$1 + LAYER_VERSION=$1 fi # Read the new version if [ -z "$2" ]; then - echo "Must specify a forwarder version" - exit 1 + echo "Must specify a forwarder version" + exit 1 else - FORWARDER_VERSION=$2 + FORWARDER_VERSION=$2 fi echo "FORWARDER_VERSION=$FORWARDER_VERSION" diff --git a/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh b/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh index 946e7af1..1761f428 100755 --- a/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh +++ b/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh @@ -17,10 +17,10 @@ rm -rf ./bin # between different python runtimes. if [[ $(docker image ls | grep -c datadog-go-layer) -lt 1 ]]; then - docker buildx build --platform linux/amd64 -t datadog-go-layer . --no-cache --build-arg runtime=python:3.7 + docker buildx build --platform linux/amd64 -t datadog-go-layer . --no-cache --build-arg "runtime=python:3.7" fi id=$(docker create --platform linux/amd64 datadog-go-layer) -docker cp $id:/go/src/github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring/trace_forwarder/bin . -docker rm -v $id +docker cp "${id}:/go/src/github.com/DataDog/datadog-serverless-functions/aws/logs_monitoring/trace_forwarder/bin" . +docker rm -v "${id}" echo "Done creating archive bin"