Merge pull request #198 from sportsru/fix_handlers_and_systemd_unit

Fix handlers and systemd unit, speed up tests

.gitignore

@@ -3,3 +3,4 @@ tests/*.retry
 \#*\#
 .\#*
 /.python-version
+*.pyc

.travis.yml

@@ -4,7 +4,7 @@ dist: trusty
 sudo: required
 language: python
 python:
-  - "2.7"
+  - "2.7.15"
 env:
   - >
     DISTRIBUTION=ubuntu
@@ -46,17 +46,17 @@ env:
     DOCKER_PRIVILEGED=true
   - >
     DISTRIBUTION=ubuntu-upstart
-    DIST_VERSION=14.04
+    DIST_VERSION=14_04-builded
     MONGODB_VERSION=4.0
     DOCKER_PRIVILEGED=true
   - >
     DISTRIBUTION=ubuntu-upstart
-    DIST_VERSION=14.04
+    DIST_VERSION=14_04-builded
     MONGODB_VERSION=3.6
     DOCKER_PRIVILEGED=true
   - >
     DISTRIBUTION=ubuntu-upstart
-    DIST_VERSION=14.04
+    DIST_VERSION=14_04-builded
     MONGODB_VERSION=3.4
     DOCKER_PRIVILEGED=true
   - >

defaults/main.yml

@@ -16,6 +16,8 @@ mongodb_pymongo_pip_version: 3.7.1
 mongodb_user_update_password: "on_create"        # MongoDB user password update default policy
 mongodb_manage_service: true
 mongodb_manage_systemd_unit: true
+mongodb_systemd_unit_limit_nofile: 64000
+mongodb_systemd_unit_limit_nproc: 64000
 
 mongodb_disable_transparent_hugepages: false
 

handlers/main.yml

@@ -3,23 +3,13 @@
 - name: reload systemd
   systemd:
     daemon_reload: yes
-  when:
-    - ansible_service_mgr == "systemd"
-    - mongodb_manage_service | bool
 
 - name: run disable-transparent-hugepages
   systemd:
     name: disable-transparent-hugepages
     enabled: true
     state: started
     daemon_reload: yes
-  when: ansible_service_mgr == "systemd" and mongodb_disable_transparent_hugepages | bool
-
-- name: mongodb reload
-  service:
-    name: "{{ mongodb_daemon_name }}"
-    state: reloaded
-  when: mongodb_manage_service | bool
 
 - name: mongodb restart
   service:
@@ -38,11 +28,6 @@
     name: sysfsutils
     state: restarted
 
-- name: service started
-  service:
-    name: "{{ mongodb_daemon_name }}"
-    state: started
-
 - name: wait when mongodb is started
   wait_for:
     host: "{{ item }}"
@@ -56,3 +41,4 @@
     port: "{{ mongodb_net_port }}"
     delay: 5
     timeout: 120
+

tasks/configure.yml

@@ -81,7 +81,6 @@
     mode: 0644
   notify:
     - mongodb restart
-    - service started
     - wait when mongodb is started
 
 - name: Flush all handlers at this point

tasks/disable_transparent_hugepages.yml

@@ -10,4 +10,7 @@
     - "disable-transparent-hugepages.{{ ansible_distribution | lower }}.service.j2"
     - "disable-transparent-hugepages.{{ ansible_os_family | lower }}.service.j2"
   notify: "run disable-transparent-hugepages"
+  when:
+    - ansible_service_mgr == "systemd"
+    - mongodb_disable_transparent_hugepages | bool
 

tasks/install.debian.yml

@@ -59,8 +59,8 @@
     state: present
 
 - name: Add systemd configuration if present
-  copy:
-    src: mongodb.service
+  template:
+    src: mongodb.service.j2
     dest: "/lib/systemd/system/{{mongodb_daemon_name}}.service"
     owner: root
     group: root

tasks/main.yml

@@ -124,6 +124,12 @@
   no_log: false
   tags: [mongodb]
 
+- name: service started
+  service:
+    name: "{{ mongodb_daemon_name }}"
+    state: started
+    enabled: yes
+
 - name: Include MMS Agent configuration
   include: mms-agent.yml
   when: mongodb_mms_api_key | length > 0

tasks/mms-agent.yml

@@ -19,9 +19,14 @@
     mode: 0755
 
 - name: Configure the MMS agent pt. 2
-  template: src=monitoring-agent.config.j2 dest=/etc/mongodb-mms/monitoring-agent.config
+  template:
+   src: monitoring-agent.config.j2
+   dest: /etc/mongodb-mms/monitoring-agent.config
   notify: mongodb-mms-monitoring-agent restart
 
 - name: Ensure that the MMS agent is started
-  service: name=mongodb-mms-monitoring-agent state=started enabled=yes
-  when: mongodb_manage_service
+  service:
+    name: mongodb-mms-monitoring-agent
+    state: started
+    enabled: yes
+

tasks/replication.yml

@@ -19,7 +19,7 @@
   with_items:
     - "{{ mongodb_replication_params|default([]) }}"
 
-- name: Replication configuration without auth
+- name: Replication configuration without auth on master
   mongodb_replication:
     build_indexes: "{{ item.build_indexes|default(omit) }}"
     login_host: "{{ mongodb_login_host|default('localhost') }}"
@@ -32,6 +32,26 @@
     priority: "{{ item.priority|default(1.0) }}"
     votes: "{{ item.votes|default(omit) }}"
     ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}"
-  when: mongodb_security_authorization == 'disabled'
+  when: ( mongodb_security_authorization == 'disabled'
+        and mongodb_master is defined and mongodb_master )
   with_items:
     - "{{ mongodb_replication_params|default([]) }}"
+
+- name: Replication configuration without auth on replicas and arbiters
+  mongodb_replication:
+    build_indexes: "{{ item.build_indexes|default(omit) }}"
+    login_host: "{{ mongodb_login_host|default('localhost') }}"
+    login_port: "{{ mongodb_login_port|default(27017) }}"
+    replica_set: "{{ mongodb_replication_replset }}"
+    host_name: "{{ item.host_name }}"
+    host_port: "{{ item.host_port|default(27017) }}"
+    host_type: "{{ item.host_type|default('replica') }}"
+    hidden: "{{ item.hidden|default(false) }}"
+    priority: "{{ item.priority|default(1.0) }}"
+    votes: "{{ item.votes|default(omit) }}"
+    ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}"
+  when: ( mongodb_security_authorization == 'disabled'
+        and ( mongodb_master is not defined or not mongodb_master) )
+  with_items:
+    - "{{ mongodb_replication_params|default([]) }}"
+

files/mongodb.service → templates/mongodb.service.j2

@@ -1,9 +1,10 @@
+# {{ ansible_managed }}
 [Unit]
 Description=An object/document-oriented database
 Documentation=man:mongod(1)
 
 [Service]
-User=mongodb
+User={{ mongodb_user }}
 ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod --config /etc/mongod.conf
 # file size
 LimitFSIZE=infinity
@@ -12,9 +13,9 @@ LimitCPU=infinity
 # virtual memory size
 LimitAS=infinity
 # open files
-LimitNOFILE=64000
+LimitNOFILE={{ mongodb_systemd_unit_limit_nofile }}
 # processes/threads
-LimitNPROC=64000
+LimitNPROC={{ mongodb_systemd_unit_limit_nproc }}
 # locked memory
 LimitMEMLOCK=infinity
 # total threads (user+kernel)

tests/Dockerfile.amazonlinux_2-builded

@@ -1,14 +1,12 @@
 FROM amazonlinux:2
 
 # This is needed so that ansible managed to read "ansible_default_ipv4"
-RUN yum install iproute -y
+# This step is needed since standard CentOS docker image does not come with i
+# This package seems to be required for Mongo 3.2 and downwards
+RUN yum install iproute initscripts python-pip python-devel -y
 
 # we can has SSH
 EXPOSE 22
 
-# This step is needed since standard CentOS docker image does not come with i
-# This package seems to be required for Mongo 3.2 and downwards
-RUN yum install initscripts -y
-
 # pepare for takeoff
 CMD ["/usr/sbin/init"]

tests/Dockerfile.centos_6-builded

@@ -1,10 +1,8 @@
 FROM centos:6
 
 # This is needed so that ansible managed to read "ansible_default_ipv4"
-RUN yum install iproute -y 
-
 # This step is needed since standard CentOS docker image does not come with EPEL installed by default
-RUN yum install epel-release -y
+RUN yum install iproute epel-release python-pip python-devel -y 
 
 # we can has SSH
 EXPOSE 22

tests/Dockerfile.centos_7-builded

@@ -1,10 +1,8 @@
 FROM centos:7
 
 # This is needed so that ansible managed to read "ansible_default_ipv4"
-RUN yum install iproute -y 
-
 # This step is needed since standard CentOS docker image does not come with EPEL installed by default
-RUN yum install epel-release -y
+RUN yum install iproute epel-release python-pip python-devel -y 
 
 # we can has SSH
 EXPOSE 22

tests/Dockerfile.debian_8-builded

@@ -3,5 +3,5 @@ FROM debian:8
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal && \
+    apt install --yes python-minimal python-pip && \
     rm /lib/systemd/system/[email protected]

tests/Dockerfile.debian_9-builded

@@ -3,7 +3,7 @@ FROM debian:9
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal systemd gnupg
+    apt install --yes python-minimal systemd gnupg python-pip
 
 RUN cd /lib/systemd/system/sysinit.target.wants/ && \
 		ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \

tests/Dockerfile.ubuntu-upstart_14_04-builded

@@ -0,0 +1,7 @@
+FROM ubuntu-upstart
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install --yes python-minimal python-pip python-dev iproute2
+

tests/Dockerfile.ubuntu_16_04-builded

@@ -3,5 +3,6 @@ FROM ubuntu:16.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal iproute2 && \
+    apt install --yes python-minimal python-pip python-dev iproute2 && \
     rm /lib/systemd/system/[email protected]
+

tests/Dockerfile.ubuntu_18_04-builded

@@ -3,7 +3,7 @@ FROM ubuntu:18.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt update && \
-    apt install --yes python-minimal systemd gnupg iproute2
+    apt install --yes python-minimal python-pip python-dev systemd gnupg iproute2
 
 RUN cd /lib/systemd/system/sysinit.target.wants/ && \
 		ls | grep -v systemd-tmpfiles-setup.service | xargs rm -f && \

tests/callback_plugins/profile_tasks.py

@@ -0,0 +1,74 @@
+# profile_tasks.py: an Ansible plugin for timing tasks
+
+# Copyright (C) 2014 Jharrod LaFon <[email protected]>
+# https://github.com/jlafon/ansible-profile/
+# Included with permission
+
+
+# The MIT License (MIT)
+#
+# Copyright (c) 2014 Jharrod LaFon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+from ansible.plugins.callback import CallbackBase
+import time
+
+
+class CallbackModule(CallbackBase):
+    """
+    A plugin for timing tasks
+    """
+    def __init__(self):
+        self.stats = {}
+        self.current = None
+
+    def playbook_on_task_start(self, name, is_conditional):
+        """
+        Logs the start of each task
+        """
+        if self.current is not None:
+            # Record the running time of the last executed task
+            self.stats[self.current] = time.time() - self.stats[self.current]
+
+        # Record the start time of the current task
+        self.current = name
+        self.stats[self.current] = time.time()
+
+    def playbook_on_stats(self, stats):
+        """
+        Prints the timings
+        """
+        # Record the timing of the very last task
+        if self.current is not None:
+            self.stats[self.current] = time.time() - self.stats[self.current]
+
+        # Sort the tasks by their running time
+        results = sorted(self.stats.items(),
+                         key=lambda value: value[1], reverse=True)
+
+        # Just keep the top 25
+        results = results[:25]
+
+        # Print the timings
+        for name, elapsed in results:
+            print("{0:-<70}{1:->9}".format(
+                    '{0} '.format(name),
+                    ' {0:.02f}s'.format(elapsed)))

tests/files/etc/apt/apt.conf.d/proxy.conf

@@ -0,0 +1,3 @@
+Acquire::http::Proxy "http://proxy:3128/";
+Acquire::https::Proxy "http://proxy:3128/";
+

tests/files/etc/environment

@@ -0,0 +1,2 @@
+PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+HTTP_PROXY="http://proxy:3128"