Merge pull request #2 from better/feature/support-additional-vars

.buildkite/pipeline.yml

@@ -7,3 +7,9 @@ steps:
     plugins:
       - docker-compose#v3.3.0:
           run: tests
+  - label: ":shell::heavy_check_mark: Shellcheck"
+    plugins:
+      - shellcheck#v1.1.2:
+          files:
+            - 'lib/*.sh'
+            - 'hooks/*'

README.md

@@ -21,6 +21,22 @@ steps:
 The host where this agent can reach a running instance of dogstatsd for
 reporting metrics to.
 
+### `dogstatsd_port` (Optional, string)
+
+The port where this agent can reach dogstatsd on wherever is specified
+in `dogstatsd_host`. This defaults to the default port for dogstatsd
+which is `8125`.
+
+### `additional_tags` (Optional, array)
+
+This is an array of additional tags you want to send and where to find
+them in the environment variables. Each entry is an object with
+properties
+
+* `tag` which is the tag name to send to datadog
+* `env_var` which is the environment variable to pull the tag's value from
+* `value` which is a static value to send for that tag
+
 ## Developing
 
 To run the tests:

hooks/environment

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "--- :datadog: Recording step start time"
+
+BUILDKITE_PLUGIN_DATADOG_STATS_STEP_START_TIME_MS=$(date +%s%3N)
+
+export BUILDKITE_PLUGIN_DATADOG_STATS_STEP_START_TIME_MS
+
+echo "Step started at ${BUILDKITE_PLUGIN_DATADOG_STATS_STEP_START_TIME_MS}"

hooks/post-checkout

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+basedir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+
+# shellcheck source=lib/get-tags.sh
+. "$basedir/lib/get-tags.sh"
+# shellcheck source=lib/report-to-datadog.sh
+. "$basedir/lib/report-to-datadog.sh"
+# shellcheck source=lib/time-diff.sh
+. "$basedir/lib/time-diff.sh"
+# shellcheck source=lib/get-metric-name.sh
+. "$basedir/lib/get-metric-name.sh"
+
+NOW=$(date +%s%3N)
+
+echo "--- :datadog: Recording checkout run time"
+
+METRIC_NAME=$(getMetricName "checkout.duration")
+METRIC_VALUE=$(timeDiff "$BUILDKITE_PLUGIN_DATADOG_STATS_CHECKOUT_START_TIME_MS")
+TAGS=$(getTags)
+
+reportToDatadog "${METRIC_NAME}" "${METRIC_VALUE}" d "${TAGS}"

hooks/post-command

@@ -1,21 +1,24 @@
 #!/bin/bash
+
 set -euo pipefail
 
 basedir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
 
+# shellcheck source=lib/get-tags.sh
 . "$basedir/lib/get-tags.sh"
+# shellcheck source=lib/report-to-datadog.sh
 . "$basedir/lib/report-to-datadog.sh"
+# shellcheck source=lib/time-diff.sh
+. "$basedir/lib/time-diff.sh"
+# shellcheck source=lib/get-metric-name.sh
+. "$basedir/lib/get-metric-name.sh"
 
 NOW=$(date +%s%3N)
-START_TIME=$BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS
-BASE_METRIC_PATH=${BUILDKITE_PLUGIN_DATADOG_STATS_METRIC_PREFIX:-buildkite.steps}
 
 echo "--- :datadog: Recording command run time"
 
-METRIC_NAME=${BASE_METRIC_PATH}.duration
-METRIC_VALUE=$(($NOW-$START_TIME))
+METRIC_NAME=$(getMetricName "command.duration")
+METRIC_VALUE=$(timeDiff "$BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS")
 TAGS=$(getTags)
 
-echo "Reporting ${METRIC_NAME} as ${METRIC_VALUE} with tags ${TAGS} to datadog"
-
-reportToDatadog $METRIC_NAME $METRIC_VALUE d "${TAGS}"
+reportToDatadog "${METRIC_NAME}" "${METRIC_VALUE}" d "${TAGS}"

hooks/pre-checkout

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "--- :datadog: Recording checkout start time"
+
+BUILDKITE_PLUGIN_DATADOG_STATS_CHECKOUT_START_TIME_MS=$(date +%s%3N)
+
+export BUILDKITE_PLUGIN_DATADOG_STATS_CHECKOUT_START_TIME_MS
+
+echo "Checkout started at ${BUILDKITE_PLUGIN_DATADOG_STATS_CHECKOUT_START_TIME_MS}"

hooks/pre-command

@@ -4,6 +4,8 @@ set -euo pipefail
 
 echo "--- :datadog: Recording command start time"
 
-export BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS=`date +%s%3N`
+BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS=$(date +%s%3N)
+
+export BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS
 
 echo "Command started at ${BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS}"

hooks/pre-exit

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+basedir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
+
+# shellcheck source=lib/get-tags.sh
+. "$basedir/lib/get-tags.sh"
+# shellcheck source=lib/report-to-datadog.sh
+. "$basedir/lib/report-to-datadog.sh"
+# shellcheck source=lib/time-diff.sh
+. "$basedir/lib/time-diff.sh"
+# shellcheck source=lib/get-metric-name.sh
+. "$basedir/lib/get-metric-name.sh"
+
+NOW=$(date +%s%3N)
+
+echo "--- :datadog: Recording step run time"
+
+METRIC_NAME=$(getMetricName "step.duration")
+METRIC_VALUE=$(timeDiff "$BUILDKITE_PLUGIN_DATADOG_STATS_STEP_START_TIME_MS")
+TAGS=$(getTags)
+
+reportToDatadog "${METRIC_NAME}" "${METRIC_VALUE}" d "${TAGS}"

lib/get-metric-name.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+function getMetricName() {
+  local BASE_METRIC_PATH=${BUILDKITE_PLUGIN_DATADOG_STATS_METRIC_PREFIX:-buildkite.steps}
+  echo -n "${BASE_METRIC_PATH}.${1}"
+}

lib/get-tags.sh

@@ -1,3 +1,30 @@
+#!/bin/bash
+
+function readAdditionalTags() {
+  local result=()
+  # Provide a limit to avoid accidental infinite loops
+  for i in {0..100}; do
+    tag_var="BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_${i}_TAG"
+    env_var="BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_${i}_ENV_VAR"
+    value_var="BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_${i}_VALUE"
+
+    if [[ -z ${!tag_var:-} ]]; then
+      break;
+    fi
+
+    if [ -z ${!env_var:-} ]; then
+      # If env_var is not set, then a hard-coded value must be
+      result+=("${!tag_var}:${!value_var}")
+    else
+      local env_var_value=${!env_var}
+      result+=("${!tag_var}:${!env_var_value:-}")
+    fi
+  done
+
+  # shellcheck disable=SC2001
+  echo -n "$(IFS=$','; echo "${result[*]}" | sed 's/,$//')"
+}
+
 function getTags() {
   BK_COMMAND=${BUILDKITE_COMMAND}
   BK_LABEL=${BUILDKITE_LABEL}
@@ -10,6 +37,13 @@ function getTags() {
 
   PIPELINE_SLUG=${BUILDKITE_PIPELINE_SLUG}
   RETRY_COUNT=${BUILDKITE_RETRY_COUNT:-0}
+  ADDITIONAL_TAGS=$(readAdditionalTags)
+
+  RESULT="is_master:${IS_MASTER},pipeline_slug:${PIPELINE_SLUG},step_command:${BK_COMMAND},step_label:${BK_LABEL},retry_count:${RETRY_COUNT}"
+
+  if [ -n "$ADDITIONAL_TAGS" ]; then
+    RESULT+=",${ADDITIONAL_TAGS}"
+  fi
 
-  echo is_master:${IS_MASTER},pipeline_slug:${PIPELINE_SLUG},step_command:${BK_COMMAND},step_label:${BK_LABEL},retry_count:${RETRY_COUNT}
+  echo "${RESULT}"
 }

lib/report-to-datadog.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 function reportToDatadog() {
   local metric_name=$1
   local metric_value=$2
@@ -7,6 +9,6 @@ function reportToDatadog() {
   DD_HOST=${BUILDKITE_PLUGIN_DATADOG_STATS_DOGSTATSD_HOST:-localhost}
   DD_PORT=${BUILDKITE_PLUGIN_DATADOG_STATS_DOGSTATSD_PORT:-8125}
 
-  echo -n "${metric_name}:${metric_value}|${metric_type}|#${tags}"
-  echo -n "${metric_name}:${metric_value}|${metric_type}|#${tags}" | nc -u -w1 $DD_HOST $DD_PORT
+  echo "Reporting ${metric_name} with value=${metric_value}, type=${metric_type}, tags=${tags}"
+  echo -n "${metric_name}:${metric_value}|${metric_type}|#${tags}" | nc -u -w1 "${DD_HOST}" "${DD_PORT}"
 }

lib/time-diff.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+function timeDiff() {
+  START_TIME=$1
+  NOW=$(date +%s%3N)
+
+  echo -n $((NOW-START_TIME))
+}

plugin.yml

@@ -10,6 +10,18 @@ configuration:
       type: string
     dogstatsd_port:
       type: string
+    additional_tags:
+      type: array
+      minimum: 1
+      items:
+        type: object
+        properties:
+          tag:
+            type: string
+          env_var:
+            type: string
+          value:
+            type: string
   required:
     - dogstatsd_host
   additionalProperties: false

tests/environment.bats

@@ -0,0 +1,10 @@
+#!/usr/bin/env bats
+
+load "$BATS_PATH/load.bash"
+
+@test "Records the start time of the step" {
+  run "$PWD/hooks/environment"
+
+  assert_success
+  assert_output --partial "Step started at $(date +%s)"
+}

tests/post-checkout.bats

@@ -0,0 +1,24 @@
+#!/usr/bin/env bats
+
+load "$BATS_PATH/load.bash"
+
+@test "Sets the correct tags and runtime for the master branch" {
+  NOW=$(date +%s%3N)
+  export BUILDKITE_PLUGIN_DATADOG_STATS_CHECKOUT_START_TIME_MS=$((NOW-900))
+  export BUILDKITE_BRANCH=master
+  export BUILDKITE_PIPELINE_SLUG=monorepo
+  export BUILDKITE_COMMAND="cd somewhere && make do-something"
+  export BUILDKITE_LABEL=":shipit: deploy-prod"
+
+  run "$PWD/hooks/post-checkout"
+
+  assert_success
+  assert_output --partial "Reporting buildkite.steps.checkout.duration with value=90"
+  assert_output --partial "tags=is_master:true,pipeline_slug:monorepo,step_command:cd somewhere && make do-something,step_label::shipit: deploy-prod,retry_count:0"
+
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS
+  unset BUILDKITE_BRANCH
+  unset BUILDKITE_PIPELINE_SLUG
+  unset BUILDKITE_COMMAND
+  unset BUILDKITE_LABEL
+}

tests/post-command.bats

@@ -0,0 +1,76 @@
+#!/usr/bin/env bats
+
+load "$BATS_PATH/load.bash"
+
+@test "Sets the correct tags and runtime for the master branch" {
+  NOW=$(date +%s%3N)
+  export BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS=$((NOW-900))
+  export BUILDKITE_BRANCH=master
+  export BUILDKITE_PIPELINE_SLUG=monorepo
+  export BUILDKITE_COMMAND="cd somewhere && make do-something"
+  export BUILDKITE_LABEL=":shipit: deploy-prod"
+
+  run "$PWD/hooks/post-command"
+
+  assert_success
+  assert_output --partial "Reporting buildkite.steps.command.duration with value=90"
+  assert_output --partial "tags=is_master:true,pipeline_slug:monorepo,step_command:cd somewhere && make do-something,step_label::shipit: deploy-prod,retry_count:0"
+
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS
+  unset BUILDKITE_BRANCH
+  unset BUILDKITE_PIPELINE_SLUG
+  unset BUILDKITE_COMMAND
+  unset BUILDKITE_LABEL
+}
+
+@test "Sets the correct tags and runtime for a non-master branch" {
+  NOW=$(date +%s%3N)
+  export BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS=$((NOW-900))
+  export BUILDKITE_BRANCH=some-branch
+  export BUILDKITE_PIPELINE_SLUG=monorepo
+  export BUILDKITE_COMMAND="cd somewhere && make do-something"
+  export BUILDKITE_LABEL=":shipit: deploy-prod"
+
+  run "$PWD/hooks/post-command"
+
+  assert_success
+  assert_output --partial "Reporting buildkite.steps.command.duration with value=90"
+  assert_output --partial "tags=is_master:false,pipeline_slug:monorepo,step_command:cd somewhere && make do-something,step_label::shipit: deploy-prod,retry_count:0"
+
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS
+  unset BUILDKITE_BRANCH
+  unset BUILDKITE_PIPELINE_SLUG
+  unset BUILDKITE_COMMAND
+  unset BUILDKITE_LABEL
+}
+
+@test "It supports specifying additional tags by value and env var" {
+  NOW=$(date +%s%3N)
+  export BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS=$((NOW-900))
+  export BUILDKITE_BRANCH=some-branch
+  export BUILDKITE_PIPELINE_SLUG=monorepo
+  export BUILDKITE_COMMAND="cd somewhere && make do-something"
+  export BUILDKITE_LABEL=":shipit: deploy-prod"
+  export BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_0_TAG="my-tag"
+  export BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_0_ENV_VAR="MY_ENV_VAR"
+  export MY_ENV_VAR="my-tag-value"
+  export BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_1_TAG="my-other-tag"
+  export BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_1_VALUE="my-other-tag-value"
+
+  run "$PWD/hooks/post-command"
+
+  assert_success
+  assert_output --partial "Reporting buildkite.steps.command.duration with value=90"
+  assert_output --partial "tags=is_master:false,pipeline_slug:monorepo,step_command:cd somewhere && make do-something,step_label::shipit: deploy-prod,retry_count:0,my-tag:my-tag-value,my-other-tag:my-other-tag-value"
+
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_COMMAND_START_TIME_MS
+  unset BUILDKITE_BRANCH
+  unset BUILDKITE_PIPELINE_SLUG
+  unset BUILDKITE_COMMAND
+  unset BUILDKITE_LABEL
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_0_TAG
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_0_ENV_VAR
+  unset MY_ENV_VAR
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_1_TAG
+  unset BUILDKITE_PLUGIN_DATADOG_STATS_ADDITIONAL_TAGS_1_VALUE
+}