diff --git a/Dockerfile b/Dockerfile index fb2f0ca0..b6c0df14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,9 @@ FROM debian:jessie MAINTAINER Datadog ENV DOCKER_DD_AGENT=yes \ - AGENT_VERSION=1:5.16.0-1 + AGENT_VERSION=1:5.16.0-1 \ + PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:${PATH}" \ + PYTHONPATH=/opt/datadog-agent/agent # Install the Agent RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/datadog.list \ @@ -15,15 +17,13 @@ RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/d && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Configure the Agent -# 1. Listen to statsd (8125) and traces (8126) from other containers -# 2. Turn syslog off +# 1. Turn syslog off by using the DD_CONF_LOG_TO_SYSLOG env var # 3. Remove dd-agent user from supervisor configuration # 4. Remove dd-agent user from init.d configuration # 5. Fix permission on /etc/init.d/datadog-agent +ENV DD_CONF_LOG_TO_SYSLOG=no \ + NON_LOCAL_TRAFFIC=yes RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" /etc/dd-agent/datadog.conf \ - && sed -i "/user=dd-agent/d" /etc/dd-agent/supervisor.conf \ && sed -i 's/AGENTUSER="dd-agent"/AGENTUSER="root"/g' /etc/init.d/datadog-agent \ && rm /etc/dd-agent/conf.d/network.yaml.default \ || chmod +x /etc/init.d/datadog-agent @@ -45,5 +45,6 @@ HEALTHCHECK --interval=5m --timeout=3s --retries=1 \ -c /etc/dd-agent/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \ -eq 0 || exit 1 +ADD config_builder.py /config_builder.py ENTRYPOINT ["/entrypoint.sh"] CMD ["supervisord", "-n", "-c", "/etc/dd-agent/supervisor.conf"] diff --git a/alpine/Dockerfile b/alpine/Dockerfile index e29fa665..c854106a 100644 --- a/alpine/Dockerfile +++ b/alpine/Dockerfile @@ -7,13 +7,16 @@ ENV DD_HOME=/opt/datadog-agent \ DD_START_AGENT=0 \ DOCKER_DD_AGENT=yes \ PYCURL_SSL_LIBRARY=openssl \ - AGENT_VERSION=5.16.0 + AGENT_VERSION=5.16.0 \ + PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:$PATH" \ + PYTHONPATH=/opt/datadog-agent/agent # Add Docker check COPY conf.d/docker_daemon.yaml "$DD_HOME/agent/conf.d/docker_daemon.yaml" # Add install and config files ADD https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/setup_agent.sh /tmp/setup_agent.sh +ADD https://raw.githubusercontent.com/DataDog/docker-dd-agent/master/config_builder.py /config_builder.py COPY entrypoint.sh /entrypoint.sh # Expose supervisor and DogStatsD port @@ -34,9 +37,9 @@ RUN apk add -qU --no-cache -t .build-deps gcc musl-dev pgcluster-dev linux-heade # 2. Turn syslog off # 3. Remove dd-agent user from supervisor configuration # 4. Remove setup script +ENV DD_CONF_LOG_TO_SYSLOG=no \ + NON_LOCAL_TRAFFIC=yes RUN cp "$DD_HOME/agent/datadog.conf.example" "$DD_HOME/agent/datadog.conf" \ - && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" "$DD_HOME/agent/datadog.conf" \ - && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" "$DD_HOME/agent/datadog.conf" \ && sed -i "/user=dd-agent/d" "$DD_HOME/agent/supervisor.conf" \ && rm "$DD_HOME/agent/conf.d/network.yaml.default" \ || rm /tmp/setup_agent.sh diff --git a/alpine/entrypoint.sh b/alpine/entrypoint.sh index 665a1974..bb874773 100755 --- a/alpine/entrypoint.sh +++ b/alpine/entrypoint.sh @@ -3,47 +3,7 @@ set -x ##### Core config ##### - -if [[ $DD_API_KEY ]]; then - export API_KEY=${DD_API_KEY} -fi - -if [[ $DD_API_KEY_FILE ]]; then - export API_KEY=$(cat $DD_API_KEY_FILE) -fi - -if [[ $API_KEY ]]; then - sed -i -e "s/^.*api_key:.*$/api_key: ${API_KEY}/" /opt/datadog-agent/agent/datadog.conf -else - echo "You must set API_KEY environment variable to run the Datadog Agent container" - exit 1 -fi - -if [[ $DD_HOSTNAME ]]; then - sed -i -r -e "s/^# ?hostname.*$/hostname: ${DD_HOSTNAME}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $DD_TAGS ]]; then - export TAGS=${DD_TAGS} -fi - -if [[ $EC2_TAGS ]]; then - export EC2_TAGS=${EC2_TAGS//\//\\/} # escape forward slashes from tags before invoking sed - sed -i -e "s/^# collect_ec2_tags.*$/collect_ec2_tags: ${EC2_TAGS}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $TAGS ]]; then - export TAGS=${TAGS//\//\\/} # escape forward slashes from tags before invoking sed - sed -i -r -e "s/^# ?tags:.*$/tags: ${TAGS}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $DD_LOG_LEVEL ]]; then - export LOG_LEVEL=$DD_LOG_LEVEL -fi - -if [[ $LOG_LEVEL ]]; then - sed -i -e"s/^.*log_level:.*$/log_level: ${LOG_LEVEL}/" /opt/datadog-agent/agent/datadog.conf -fi +/opt/datadog-agent/venv/bin/activate && python /config_builder.py if [[ $DD_LOGS_STDOUT ]]; then export LOGS_STDOUT=$DD_LOGS_STDOUT @@ -57,75 +17,6 @@ if [[ $LOGS_STDOUT == "yes" ]]; then sed -i -e "/^.*\[program:.*\].*$/a stderr_logfile_maxbytes=0" /opt/datadog-agent/agent/supervisor.conf fi -if [[ $DD_URL ]]; then - sed -i -e 's@^.*dd_url:.*$@dd_url: '${DD_URL}'@' /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $STATSD_METRIC_NAMESPACE ]]; then - sed -i -e "s/^# statsd_metric_namespace:.*$/statsd_metric_namespace: ${STATSD_METRIC_NAMESPACE}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $USE_DOGSTATSD ]]; then - sed -i -e "s/^.*use_dogstatsd:.*$/use_dogstatsd: ${USE_DOGSTATSD}/" /opt/datadog-agent/agent/datadog.conf -fi - - -##### Proxy config ##### - -if [[ $PROXY_HOST ]]; then - sed -i -e "s/^# proxy_host:.*$/proxy_host: ${PROXY_HOST}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $PROXY_PORT ]]; then - sed -i -e "s/^# proxy_port:.*$/proxy_port: ${PROXY_PORT}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $PROXY_USER ]]; then - sed -i -e "s/^# proxy_user:.*$/proxy_user: ${PROXY_USER}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $PROXY_PASSWORD ]]; then - sed -i -e "s/^# proxy_password:.*$/proxy_password: ${PROXY_PASSWORD}/" /opt/datadog-agent/agent/datadog.conf -fi - -##### Service discovery ##### -EC2_HOST_IP=`curl --silent http://169.254.169.254/latest/meta-data/local-ipv4 --max-time 1` - -if [[ $SD_BACKEND ]]; then - sed -i -e "s/^# service_discovery_backend:.*$/service_discovery_backend: ${SD_BACKEND}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $SD_CONFIG_BACKEND ]]; then - sed -i -e "s/^# sd_config_backend:.*$/sd_config_backend: ${SD_CONFIG_BACKEND}/" /opt/datadog-agent/agent/datadog.conf - # If no SD_BACKEND_HOST value is defined AND running in EC2 and host ip is available - if [[ -z $SD_BACKEND_HOST && -n $EC2_HOST_IP ]]; then - export SD_BACKEND_HOST="$EC2_HOST_IP" - fi -fi - -if [[ $SD_BACKEND_HOST ]]; then - sed -i -e "s/^# sd_backend_host:.*$/sd_backend_host: ${SD_BACKEND_HOST}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $SD_BACKEND_PORT ]]; then - sed -i -e "s/^# sd_backend_port:.*$/sd_backend_port: ${SD_BACKEND_PORT}/" /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $SD_TEMPLATE_DIR ]]; then - sed -i -e 's@^# sd_template_dir:.*$@sd_template_dir: '${SD_TEMPLATE_DIR}'@' /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $SD_CONSUL_TOKEN ]]; then - sed -i -e 's@^# consul_token:.*$@consul_token: '${SD_CONSUL_TOKEN}'@' /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $SD_BACKEND_USER ]]; then - sed -i -e 's@^# sd_backend_username:.*$@sd_backend_username: '${SD_BACKEND_USER}'@' /opt/datadog-agent/agent/datadog.conf -fi - -if [[ $SD_BACKEND_PASSWORD ]]; then - sed -i -e 's@^# sd_backend_password:.*$@sd_backend_password: '${SD_BACKEND_PASSWORD}'@' /opt/datadog-agent/agent/datadog.conf -fi ##### Integrations config ##### @@ -184,8 +75,6 @@ find /checks.d -name '*.py' -exec cp {} /opt/datadog-agent/agent/checks.d \; ##### Starting up ##### -export PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:$PATH" - if [[ $DOGSTATSD_ONLY ]]; then source /opt/datadog-agent/venv/bin/activate && python /opt/datadog-agent/agent/dogstatsd.py else diff --git a/config_builder.py b/config_builder.py new file mode 100755 index 00000000..f09cd16f --- /dev/null +++ b/config_builder.py @@ -0,0 +1,191 @@ +#!/opt/datadog-agent/embedded/bin/python +''' +This script is used to generate the configuration of the datadog agent, its +integrations and other moving parts. +''' + +from os import getenv, environ +import logging +from urllib2 import urlopen, URLError, HTTPError +from socket import getdefaulttimeout, setdefaulttimeout +from ConfigParser import ConfigParser + +class ConfBuilder(object): + ''' + This class manages the configuration files + ''' + def __init__(self): + # excludes from the generic variables parsing the ones that have a + # certain logic warpped around them + self.exclude_from_generic = [ + 'DD_API_KEY', 'DD_API_KEY_FILE', 'DD_HOME', + 'DD_START_AGENT', 'DD_LOGS_STDOUT' + ] + self.datadog_conf_file = '/etc/dd-agent/datadog.conf' + self.supervisor_conf_file = '/etc/dd-agent/supervisor.conf' + dd_home = getenv('DD_HOME') + if dd_home is not None: + self.datadog_conf_file = '{}/agent/datadog.conf'.format(dd_home) + self.supervisor_conf_file = '{}/agent/supervisor.conf'.format(dd_home) + # This will store the config parser object that is used in the different functions + self.config = None + + def load_config(self, config_file): + ''' + Loads a config file using ConfigParser + ''' + self.config = ConfigParser() + # import existing config from file + with open(config_file, 'rb') as cfd: + self.config.readfp(cfd) + + def save_config(self, config_file): + ''' + Saves a ConfigParser object (self.config) to the given file + ''' + if self.config is None: + logging.error('config object needs to be created before saving anything') + exit(1) + with open(config_file, 'wb') as cfd: + self.config.write(cfd) + + def build_supervisor_conf(self): + ''' + Builds the supervisor.conf based on the environment variables + ''' + self.load_config(self.supervisor_conf_file) + + _logs_stdout = getenv('DD_LOGS_STDOUT', getenv('LOGS_STDOUT', 'no')) + for _section in self.config.sections(): + if self.config.has_option(_section, 'user'): + self.config.remove_option(_section, 'user') + if _logs_stdout == 'yes': + for _opt in self.config.options(_section): + if _opt.endswith('_logfile'): + self.config.remove_option(_section, _opt) + if _section.startswith('program:'): + self.set_property('stdout_logfile', '/dev/stdout', _section) + self.set_property('stdout_logfile_maxbytes', '0', _section) + self.set_property('stderr_logfile', '/dev/stderr', _section) + self.set_property('stderr_logfile_maxbytes', '0', _section) + + if getenv('KUBERNETES') is not None or getenv('MESOS_MASTER') is not None or getenv('MESOS_SLAVE') is not None: + # expose supervisord as a health check + if not self.config.has_section('inet_http_server'): + self.config.add_section('inet_http_server') + self.set_property('port', '0.0.0.0:9001', 'inet_http_server') + + self.save_config(self.supervisor_conf_file) + + def build_datadog_conf(self): + ''' + Builds the datadog.conf based on the environment variables + ''' + self.load_config(self.datadog_conf_file) + + ##### Core config ##### + self.set_api_key() + self.set_from_env_mapping('DD_HOSTNAME', 'hostname') + self.set_from_env_mapping('EC2_TAGS', 'collect_ec2_tags') + # The TAGS env variable superseeds DD_TAGS + self.set_from_env_mapping('DD_TAGS', 'tags') + self.set_from_env_mapping('TAGS', 'tags') + # The LOG_LEVEL env variable superseeds DD_LOG_LEVEL + self.set_from_env_mapping('DD_LOG_LEVEL', 'log_level') + self.set_from_env_mapping('LOG_LEVEL', 'log_level') + self.set_from_env_mapping('NON_LOCAL_TRAFFIC', 'non_local_traffic', action='store_true') + self.set_from_env_mapping('DD_URL', 'dd_url') + self.set_from_env_mapping('STATSD_METRIC_NAMESPACE', 'statsd_metric_namespace') + self.set_from_env_mapping('USE_DOGSTATSD', 'use_dogstatsd') + ##### Proxy config ##### + self.set_from_env_mapping('PROXY_HOST', 'proxy_host') + self.set_from_env_mapping('PROXY_PORT', 'proxy_port') + self.set_from_env_mapping('PROXY_USER', 'proxy_user') + self.set_from_env_mapping('PROXY_PASSWORD', 'proxy_password') + ##### Service discovery ##### + self.set_from_env_mapping('SD_BACKEND', 'service_discovery_backend') + self.set_sd_backend_host() + self.set_from_env_mapping('SD_BACKEND_PORT', 'sd_backend_port') + self.set_from_env_mapping('SD_TEMPLATE_DIR', 'sd_template_dir') + self.set_from_env_mapping('SD_CONSUL_TOKEN', 'consul_token') + self.set_from_env_mapping('SD_BACKEND_USER', 'sd_backend_username') + self.set_from_env_mapping('SD_BACKEND_PASSWORD', 'sd_backend_password') + # Magic trick to automatically add properties not yet define in the doc + self.set_generics('DD_CONF_') + + self.save_config(self.datadog_conf_file) + + def set_api_key(self): + ''' + Used for building datadog.conf + Gets the API key from the environment or the key file + and sets it in the configuration + ''' + api_key = getenv('DD_API_KEY', getenv('API_KEY', '')) + keyfile = getenv('DD_API_KEY_FILE') + if keyfile is not None: + try: + with open(keyfile, 'r') as kfile: + api_key = kfile.read() + except IOError: + logging.warning('Unable to read the content of they key file specified in DD_API_KEY_FILE') + if len(api_key) <= 0: + logging.error('You must set API_KEY environment variable or include a DD_API_KEY_FILE to run the Datadog Agent container') + exit(1) + self.set_property('api_key', api_key) + + def set_from_env_mapping(self, env_var_name, property_name, section='Main', action=None): + ''' + Sets a property using the corresponding environment variable if it exists + It also returns the value in case you want to play with it + If action is specified to 'store_true', whatever the content of the + env variable is (if exists), the value of the property will be true + ''' + _val = getenv(env_var_name) + if _val is not None: + if action == 'store_true': + _val = 'true' + self.set_property(property_name, _val, section) + return _val + return None + + def set_sd_backend_host(self): + ''' + Used for building datadog.conf + Sets sd_config_backend and sd_backend_host depending on the environment + ''' + _config_backend = getenv('SD_CONFIG_BACKEND', 'sd_config_backend') + if _config_backend is not None: + _backend_host = getenv('SD_BACKEND_HOST', 'sd_backend_host') + if _backend_host is None: + _timeout = getdefaulttimeout() + try: + setdefaulttimeout(1) + _ec2_ip = urlopen('http://169.254.169.254/latest/meta-data/local-ipv4') + self.set_property('sd_backend_host', _ec2_ip.read()) + except (URLError, HTTPError): + pass # silent fail on purpose + setdefaulttimeout(_timeout) + + def set_generics(self, prefix='DD_CONF_'): + ''' + Looks for environment variables starting by the given prefix and consider that the + rest of the variable name is the name of the property to set + ''' + for dd_var in environ: + if dd_var.startswith(prefix) and dd_var.upper() not in self.exclude_from_generic: + if len(dd_var) > 0: + self.set_property(dd_var[len(prefix):].lower(), environ[dd_var]) + + def set_property(self, property_name, property_value, section='Main'): + ''' + Sets the given property to the given value in the configuration + ''' + if self.config is None: + logging.error('config object needs to be created before setting properties') + exit(1) + self.config.set(section, property_name, property_value) + +if __name__ == '__main__': + cfg = ConfBuilder() + cfg.build_datadog_conf() diff --git a/dogstatsd/Dockerfile b/dogstatsd/Dockerfile index 47e47279..731cde24 100644 --- a/dogstatsd/Dockerfile +++ b/dogstatsd/Dockerfile @@ -3,7 +3,9 @@ FROM debian:jessie MAINTAINER Datadog ENV DOCKER_DD_AGENT=yes \ - AGENT_VERSION=1:5.16.0-1 + AGENT_VERSION=1:5.16.0-1 \ + PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:${PATH}" \ + PYTHONPATH=/opt/datadog-agent/agent COPY entrypoint.sh supervisor.conf / @@ -19,9 +21,9 @@ RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/d # 1. Listen to statsd from other containers # 2. Turn syslog off # 3. Use custom supervisor.conf +ENV DD_CONF_LOG_TO_SYSLOG=no \ + NON_LOCAL_TRAFFIC=yes RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" /etc/dd-agent/datadog.conf \ && mv /supervisor.conf /etc/dd-agent/supervisor.conf # Expose supervisor, DogStatsD and trace-agent port @@ -39,6 +41,7 @@ HEALTHCHECK --interval=5m --timeout=3s --retries=1 \ -c /etc/dd-agent/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \ -eq 0 || exit 1 +ADD https://raw.githubusercontent.com/DataDog/docker-dd-agent/master/config_builder.py /config_builder.py ENTRYPOINT ["/entrypoint.sh"] USER 1001 diff --git a/dogstatsd/alpine/Dockerfile b/dogstatsd/alpine/Dockerfile index 6a74c631..4084de8f 100644 --- a/dogstatsd/alpine/Dockerfile +++ b/dogstatsd/alpine/Dockerfile @@ -6,10 +6,13 @@ ENV DD_HOME=/opt/datadog-agent \ DOCKER_DD_AGENT=yes \ # prevent the agent from being started after install DD_START_AGENT=0 \ - AGENT_VERSION=5.16.0 + AGENT_VERSION=5.16.0 \ + PATH="${DD_HOME}/venv/bin:${DD_HOME}/bin:${PATH}" \ + PYTHONPATH=/opt/datadog-agent/agent # Add install and config files ADD https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/setup_agent.sh /tmp/setup_agent.sh +ADD https://raw.githubusercontent.com/DataDog/docker-dd-agent/master/config_builder.py /config_builder.py COPY entrypoint.sh supervisor.conf / # Expose supervisor and DogStatsD port @@ -30,9 +33,9 @@ RUN apk add -qU --no-cache -t .build-deps curl gcc musl-dev pgcluster-dev linux- # 2. Turn syslog off # 3. Use custom supervisor.conf # 4. Clean up the install script +ENV DD_CONF_LOG_TO_SYSLOG=no \ + NON_LOCAL_TRAFFIC=yes RUN mv $DD_HOME/agent/datadog.conf.example $DD_HOME/agent/datadog.conf \ - && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" $DD_HOME/agent/datadog.conf \ - && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" $DD_HOME/agent/datadog.conf \ && mv /supervisor.conf $DD_HOME/agent/supervisor.conf \ && rm /tmp/setup_agent.sh @@ -40,6 +43,7 @@ RUN mv $DD_HOME/agent/datadog.conf.example $DD_HOME/agent/datadog.conf \ RUN chmod -R g+wx $DD_HOME \ && chmod g+x /entrypoint.sh +ADD config_builder.py /config_builder.py ENTRYPOINT ["/entrypoint.sh"] USER 1001 diff --git a/dogstatsd/alpine/entrypoint.sh b/dogstatsd/alpine/entrypoint.sh index 8fd6eb29..3d11359c 100644 --- a/dogstatsd/alpine/entrypoint.sh +++ b/dogstatsd/alpine/entrypoint.sh @@ -1,29 +1,6 @@ #!/bin/sh #set -e -if [[ $DD_API_KEY ]]; then - export API_KEY=${DD_API_KEY} -fi - -if [[ $DD_API_KEY_FILE ]]; then - export API_KEY=$(cat $DD_API_KEY_FILE) -fi - -if [[ $API_KEY ]]; then - sed -i -e "s/^.*api_key:.*$/api_key: ${API_KEY}/" $DD_HOME/agent/datadog.conf -else - echo "You must set API_KEY environment variable to run the DogStatsD container" - exit 1 -fi - -if [[ $DD_URL ]]; then - sed -i -e 's@^.*dd_url:.*$@dd_url: '${DD_URL}'@' $DD_HOME/agent/datadog.conf -fi - -if [[ $DD_HOSTNAME ]]; then - sed -i -r -e "s/^# ?hostname.*$/hostname: ${DD_HOSTNAME}/" $DD_HOME/agent/datadog.conf -fi - -export PATH="$DD_HOME/venv/bin:$DD_HOME/bin:$PATH" +/opt/datadog-agent/embedded/bin/python /config_builder.py exec "$@" diff --git a/dogstatsd/entrypoint.sh b/dogstatsd/entrypoint.sh index fa9a889d..288b8ab1 100755 --- a/dogstatsd/entrypoint.sh +++ b/dogstatsd/entrypoint.sh @@ -1,28 +1,7 @@ #!/bin/bash #set -e -if [[ $DD_API_KEY ]]; then - export API_KEY=${DD_API_KEY} -fi - -if [[ $DD_API_KEY_FILE ]]; then - export API_KEY=$(cat $DD_API_KEY_FILE) -fi - -if [[ $API_KEY ]]; then - sed -i -e "s/^.*api_key:.*$/api_key: ${API_KEY}/" /etc/dd-agent/datadog.conf -else - echo "You must set API_KEY environment variable to run the DogStatsD container" - exit 1 -fi - -if [[ $DD_URL ]]; then - sed -i -e 's@^.*dd_url:.*$@dd_url: '${DD_URL}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $DD_HOSTNAME ]]; then - sed -i -r -e "s/^# ?hostname.*$/hostname: ${DD_HOSTNAME}/" /etc/dd-agent/datadog.conf -fi +/opt/datadog-agent/embedded/bin/python /config_builder.py # ensure that the trace-agent doesn't run unless instructed to if [[ $DD_APM_ENABLED ]]; then @@ -41,6 +20,4 @@ if [[ -z $DD_HOSTNAME && $DD_APM_ENABLED ]]; then export DD_HOSTNAME=`PYTHONPATH=/opt/datadog-agent/agent /opt/datadog-agent/embedded/bin/python -c "from utils.hostname import get_hostname; print get_hostname()"` fi -export PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:$PATH" - exec "$@" diff --git a/entrypoint.sh b/entrypoint.sh index 056dd3ff..97e558a0 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,144 +1,10 @@ #!/bin/bash #set -e - -##### Core config ##### - -if [[ $DD_API_KEY ]]; then - export API_KEY=${DD_API_KEY} -fi - -if [[ $DD_API_KEY_FILE ]]; then - export API_KEY=$(cat $DD_API_KEY_FILE) -fi - -if [[ $API_KEY ]]; then - sed -i -e "s/^.*api_key:.*$/api_key: ${API_KEY}/" /etc/dd-agent/datadog.conf -else - echo "You must set API_KEY environment variable or include a DD_API_KEY_FILE to run the Datadog Agent container" - exit 1 -fi - -if [[ $DD_HOSTNAME ]]; then - sed -i -r -e "s/^# ?hostname.*$/hostname: ${DD_HOSTNAME}/" /etc/dd-agent/datadog.conf -fi - -if [[ $DD_TAGS ]]; then - export TAGS=${DD_TAGS} -fi - -if [[ $EC2_TAGS ]]; then - export EC2_TAGS=${EC2_TAGS//\//\\/} # escape forward slashes from tags before invoking sed - sed -i -e "s/^# collect_ec2_tags.*$/collect_ec2_tags: ${EC2_TAGS}/" /etc/dd-agent/datadog.conf -fi - -if [[ $TAGS ]]; then - export TAGS=${TAGS//\//\\/} # escape forward slashes from tags before invoking sed - sed -i -r -e "s/^# ?tags:.*$/tags: ${TAGS}/" /etc/dd-agent/datadog.conf -fi - -if [[ $DD_LOG_LEVEL ]]; then - export LOG_LEVEL=$DD_LOG_LEVEL -fi - -if [[ $LOG_LEVEL ]]; then - sed -i -e"s/^.*log_level:.*$/log_level: ${LOG_LEVEL}/" /etc/dd-agent/datadog.conf -fi - -if [[ $DD_LOGS_STDOUT ]]; then - export LOGS_STDOUT=$DD_LOGS_STDOUT -fi - -if [[ $LOGS_STDOUT == "yes" ]]; then - sed -i -e "/^.*_logfile.*$/d" /etc/dd-agent/supervisor.conf - sed -i -e "/^.*\[program:.*\].*$/a stdout_logfile=\/dev\/stdout\nstdout_logfile_maxbytes=0\nstderr_logfile=\/dev\/stderr\nstderr_logfile_maxbytes=0" /etc/dd-agent/supervisor.conf -fi - -if [[ $DD_URL ]]; then - sed -i -e 's@^.*dd_url:.*$@dd_url: '${DD_URL}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $NON_LOCAL_TRAFFIC ]]; then - sed -i -e 's/^# non_local_traffic:.*$/non_local_traffic: true/' /etc/dd-agent/datadog.conf -fi - -if [[ $STATSD_METRIC_NAMESPACE ]]; then - sed -i -e "s/^# statsd_metric_namespace:.*$/statsd_metric_namespace: ${STATSD_METRIC_NAMESPACE}/" /etc/dd-agent/datadog.conf -fi - -if [[ $USE_DOGSTATSD ]]; then - sed -i -e "s/^.*use_dogstatsd:.*$/use_dogstatsd: ${USE_DOGSTATSD}/" /etc/dd-agent/datadog.conf -fi - - -##### Proxy config ##### - -if [[ $PROXY_HOST ]]; then - sed -i -e "s/^# proxy_host:.*$/proxy_host: ${PROXY_HOST}/" /etc/dd-agent/datadog.conf -fi - -if [[ $PROXY_PORT ]]; then - sed -i -e "s/^# proxy_port:.*$/proxy_port: ${PROXY_PORT}/" /etc/dd-agent/datadog.conf -fi - -if [[ $PROXY_USER ]]; then - sed -i -e "s/^# proxy_user:.*$/proxy_user: ${PROXY_USER}/" /etc/dd-agent/datadog.conf -fi - -if [[ $PROXY_PASSWORD ]]; then - sed -i -e "s/^# proxy_password:.*$/proxy_password: ${PROXY_PASSWORD}/" /etc/dd-agent/datadog.conf -fi - - -##### Service discovery ##### -EC2_HOST_IP=`/opt/datadog-agent/embedded/bin/curl --silent http://169.254.169.254/latest/meta-data/local-ipv4 --max-time 1` - -if [[ $SD_BACKEND ]]; then - sed -i -e "s/^# service_discovery_backend:.*$/service_discovery_backend: ${SD_BACKEND}/" /etc/dd-agent/datadog.conf -fi - -if [[ $SD_CONFIG_BACKEND ]]; then - sed -i -e "s/^# sd_config_backend:.*$/sd_config_backend: ${SD_CONFIG_BACKEND}/" /etc/dd-agent/datadog.conf - # If no SD_BACKEND_HOST value is defined AND running in EC2 and host ip is available - if [[ -z $SD_BACKEND_HOST && -n $EC2_HOST_IP ]]; then - export SD_BACKEND_HOST="$EC2_HOST_IP" - fi -fi - -if [[ $SD_BACKEND_HOST ]]; then - sed -i -e "s/^# sd_backend_host:.*$/sd_backend_host: ${SD_BACKEND_HOST}/" /etc/dd-agent/datadog.conf -fi - -if [[ $SD_BACKEND_PORT ]]; then - sed -i -e "s/^# sd_backend_port:.*$/sd_backend_port: ${SD_BACKEND_PORT}/" /etc/dd-agent/datadog.conf -fi - -if [[ $SD_TEMPLATE_DIR ]]; then - sed -i -e 's@^# sd_template_dir:.*$@sd_template_dir: '${SD_TEMPLATE_DIR}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $SD_CONSUL_TOKEN ]]; then - sed -i -e 's@^# consul_token:.*$@consul_token: '${SD_CONSUL_TOKEN}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $SD_BACKEND_USER ]]; then - sed -i -e 's@^# sd_backend_username:.*$@sd_backend_username: '${SD_BACKEND_USER}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $SD_BACKEND_PASSWORD ]]; then - sed -i -e 's@^# sd_backend_password:.*$@sd_backend_password: '${SD_BACKEND_PASSWORD}'@' /etc/dd-agent/datadog.conf -fi +/opt/datadog-agent/embedded/bin/python /config_builder.py ##### Integrations config ##### -if [[ $KUBERNETES || $MESOS_MASTER || $MESOS_SLAVE ]]; then - # expose supervisord as a health check - echo " -[inet_http_server] -port = 0.0.0.0:9001 -" >> /etc/dd-agent/supervisor.conf -fi - if [[ $KUBERNETES ]]; then # enable kubernetes check cp /etc/dd-agent/conf.d/kubernetes.yaml.example /etc/dd-agent/conf.d/kubernetes.yaml @@ -187,8 +53,6 @@ find /checks.d -name '*.py' -exec cp {} /etc/dd-agent/checks.d \; ##### Starting up ##### -export PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:$PATH" - if [[ -z $DD_HOSTNAME && $DD_APM_ENABLED ]]; then # When starting up the trace-agent without an explicit hostname # we need to ensure that the trace-agent will report as the same host as the @@ -200,7 +64,7 @@ fi if [[ $DOGSTATSD_ONLY ]]; then echo "[WARNING] This option is deprecated as of agent 5.8.0, it will be removed in the next few versions. Please use the dogstatsd image instead." - PYTHONPATH=/opt/datadog-agent/agent /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/dogstatsd.py + /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/dogstatsd.py else exec "$@" fi diff --git a/jmx/Dockerfile b/jmx/Dockerfile index 0fb44278..dcdf3ee4 100644 --- a/jmx/Dockerfile +++ b/jmx/Dockerfile @@ -3,7 +3,9 @@ FROM debian:jessie MAINTAINER Datadog ENV DOCKER_DD_AGENT=yes \ - AGENT_VERSION=1:5.16.0-1 + AGENT_VERSION=1:5.16.0-1 \ + PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:${PATH}" \ + PYTHONPATH=/opt/datadog-agent/agent # Install the Agent RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/datadog.list \ @@ -15,16 +17,14 @@ RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/d && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Configure the Agent -# 1. Listen to statsd from other containers -# 2. Turn syslog off -# 3. Enable Service Discovery with JMXFetch +# 1. Enable Service Discovery with JMXFetch # 4. Remove dd-agent user from supervisor configuration # 5. Remove dd-agent user from init.d configuration # 6. Fix permission on /etc/init.d/datadog-agent +ENV DD_CONF_SD_JMX_ENABLE=yes \ + DD_CONF_LOG_TO_SYSLOG=no \ + NON_LOCAL_TRAFFIC=yes RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" /etc/dd-agent/datadog.conf \ - && sed -i -e"s/^.*sd_jmx_enable:.*$/sd_jmx_enable: yes/" /etc/dd-agent/datadog.conf \ && sed -i "/user=dd-agent/d" /etc/dd-agent/supervisor.conf \ && sed -i 's/AGENTUSER="dd-agent"/AGENTUSER="root"/g' /etc/init.d/datadog-agent \ && rm /etc/dd-agent/conf.d/network.yaml.default \ @@ -47,5 +47,6 @@ HEALTHCHECK --interval=5m --timeout=3s --retries=1 \ -c /etc/dd-agent/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \ -eq 0 || exit 1 +ADD https://raw.githubusercontent.com/DataDog/docker-dd-agent/master/config_builder.py /config_builder.py ENTRYPOINT ["/entrypoint.sh"] CMD ["supervisord", "-n", "-c", "/etc/dd-agent/supervisor.conf"] diff --git a/jmx/entrypoint.sh b/jmx/entrypoint.sh index 149a7f99..8723c5ca 100755 --- a/jmx/entrypoint.sh +++ b/jmx/entrypoint.sh @@ -3,47 +3,7 @@ ##### Core config ##### - -if [[ $DD_API_KEY ]]; then - export API_KEY=${DD_API_KEY} -fi - -if [[ $DD_API_KEY_FILE ]]; then - export API_KEY=$(cat $DD_API_KEY_FILE) -fi - -if [[ $API_KEY ]]; then - sed -i -e "s/^.*api_key:.*$/api_key: ${API_KEY}/" /etc/dd-agent/datadog.conf -else - echo "You must set API_KEY environment variable to run the Datadog Agent container" - exit 1 -fi - -if [[ $DD_HOSTNAME ]]; then - sed -i -r -e "s/^# ?hostname.*$/hostname: ${DD_HOSTNAME}/" /etc/dd-agent/datadog.conf -fi - -if [[ $DD_TAGS ]]; then - export TAGS=${DD_TAGS} -fi - -if [[ $EC2_TAGS ]]; then - export EC2_TAGS=${EC2_TAGS//\//\\/} # escape forward slashes from tags before invoking sed - sed -i -e "s/^# collect_ec2_tags.*$/collect_ec2_tags: ${EC2_TAGS}/" /etc/dd-agent/datadog.conf -fi - -if [[ $TAGS ]]; then - export TAGS=${TAGS//\//\\/} # escape forward slashes from tags before invoking sed - sed -i -r -e "s/^# ?tags:.*$/tags: ${TAGS}/" /etc/dd-agent/datadog.conf -fi - -if [[ $DD_LOG_LEVEL ]]; then - export LOG_LEVEL=$DD_LOG_LEVEL -fi - -if [[ $LOG_LEVEL ]]; then - sed -i -e"s/^.*log_level:.*$/log_level: ${LOG_LEVEL}/" /etc/dd-agent/datadog.conf -fi +/opt/datadog-agent/embedded/bin/python /config_builder.py if [[ $DD_LOGS_STDOUT ]]; then export LOGS_STDOUT=$DD_LOGS_STDOUT @@ -57,80 +17,6 @@ if [[ $LOGS_STDOUT == "yes" ]]; then sed -i -e "/^.*\[program:.*\].*$/a stderr_logfile_maxbytes=0" /etc/dd-agent/supervisor.conf fi -if [[ $DD_URL ]]; then - sed -i -e 's@^.*dd_url:.*$@dd_url: '${DD_URL}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $NON_LOCAL_TRAFFIC ]]; then - sed -i -e 's/^# non_local_traffic:.*$/non_local_traffic: true/' /etc/dd-agent/datadog.conf -fi - -if [[ $STATSD_METRIC_NAMESPACE ]]; then - sed -i -e "s/^# statsd_metric_namespace:.*$/statsd_metric_namespace: ${STATSD_METRIC_NAMESPACE}/" /etc/dd-agent/datadog.conf -fi - -if [[ $USE_DOGSTATSD ]]; then - sed -i -e "s/^.*use_dogstatsd:.*$/use_dogstatsd: ${USE_DOGSTATSD}/" /etc/dd-agent/datadog.conf -fi - - -##### Proxy config ##### - -if [[ $PROXY_HOST ]]; then - sed -i -e "s/^# proxy_host:.*$/proxy_host: ${PROXY_HOST}/" /etc/dd-agent/datadog.conf -fi - -if [[ $PROXY_PORT ]]; then - sed -i -e "s/^# proxy_port:.*$/proxy_port: ${PROXY_PORT}/" /etc/dd-agent/datadog.conf -fi - -if [[ $PROXY_USER ]]; then - sed -i -e "s/^# proxy_user:.*$/proxy_user: ${PROXY_USER}/" /etc/dd-agent/datadog.conf -fi - -if [[ $PROXY_PASSWORD ]]; then - sed -i -e "s/^# proxy_password:.*$/proxy_password: ${PROXY_PASSWORD}/" /etc/dd-agent/datadog.conf -fi - - -##### Service discovery ##### -EC2_HOST_IP=`/opt/datadog-agent/embedded/bin/curl --silent http://169.254.169.254/latest/meta-data/local-ipv4 --max-time 1` - -if [[ $SD_BACKEND ]]; then - sed -i -e "s/^# service_discovery_backend:.*$/service_discovery_backend: ${SD_BACKEND}/" /etc/dd-agent/datadog.conf -fi - -if [[ $SD_CONFIG_BACKEND ]]; then - sed -i -e "s/^# sd_config_backend:.*$/sd_config_backend: ${SD_CONFIG_BACKEND}/" /etc/dd-agent/datadog.conf - # If no SD_BACKEND_HOST value is defined AND running in EC2 and host ip is available - if [[ -z $SD_BACKEND_HOST && -n $EC2_HOST_IP ]]; then - export SD_BACKEND_HOST="$EC2_HOST_IP" - fi -fi - -if [[ $SD_BACKEND_HOST ]]; then - sed -i -e "s/^# sd_backend_host:.*$/sd_backend_host: ${SD_BACKEND_HOST}/" /etc/dd-agent/datadog.conf -fi - -if [[ $SD_BACKEND_PORT ]]; then - sed -i -e "s/^# sd_backend_port:.*$/sd_backend_port: ${SD_BACKEND_PORT}/" /etc/dd-agent/datadog.conf -fi - -if [[ $SD_TEMPLATE_DIR ]]; then - sed -i -e 's@^# sd_template_dir:.*$@sd_template_dir: '${SD_TEMPLATE_DIR}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $SD_CONSUL_TOKEN ]]; then - sed -i -e 's@^# consul_token:.*$@consul_token: '${SD_CONSUL_TOKEN}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $SD_BACKEND_USER ]]; then - sed -i -e 's@^# sd_backend_username:.*$@sd_backend_username: '${SD_BACKEND_USER}'@' /etc/dd-agent/datadog.conf -fi - -if [[ $SD_BACKEND_PASSWORD ]]; then - sed -i -e 's@^# sd_backend_password:.*$@sd_backend_password: '${SD_BACKEND_PASSWORD}'@' /etc/dd-agent/datadog.conf -fi ##### Integrations config ##### @@ -189,8 +75,6 @@ find /checks.d -name '*.py' -exec cp {} /etc/dd-agent/checks.d \; ##### Starting up ##### -export PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:$PATH" - if [[ $DOGSTATSD_ONLY ]]; then echo "[WARNING] This option is deprecated as of agent 5.8.0, it will be removed in the next few versions. Please use the dogstatsd image instead." PYTHONPATH=/opt/datadog-agent/agent /opt/datadog-agent/embedded/bin/python /opt/datadog-agent/agent/dogstatsd.py