diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 9bba5a1c..7f5aa1e2 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -78,6 +78,7 @@ become: true roles: - elastic_stack + - metricbeat - name: Setup load tester hosts: load_test @@ -90,7 +91,8 @@ hosts: metrics become: true roles: - - role: metrics + - metrics + - metricbeat - name: Set up miners hosts: miners @@ -100,8 +102,8 @@ - role: dashd tags: - dashd - - role: elastic_beats - core_container_name: dashd + - core_filebeat + - metricbeat - name: Set up mixers hosts: mixer_nodes @@ -111,8 +113,8 @@ - role: dashd tags: - dashd - - role: elastic_beats - core_container_name: dashd + - core_filebeat + - metricbeat - name: Set up core and tenderdash on seed nodes hosts: seed_nodes @@ -133,8 +135,8 @@ tags: - dashd - role: tenderdash - - role: elastic_beats - core_container_name: dashd + - core_filebeat + - metricbeat - name: Set up core on masternodes hosts: masternodes @@ -158,8 +160,8 @@ tags: - dashd - mn_status_report - - role: elastic_beats - core_container_name: dashd + - core_filebeat + - metricbeat # Start network @@ -189,8 +191,8 @@ dashd_zmq: true dashd_listen: true - insight - - role: elastic_beats - core_container_name: dashd + - core_filebeat + - metricbeat tags: - web @@ -205,8 +207,8 @@ enable_wallet: true tags: - dashd - - role: elastic_beats - core_container_name: dashd + - core_filebeat + - metricbeat # Register masternodes and set sporks @@ -283,9 +285,7 @@ - role: dash_cli - role: dashmate - role: mn_status_report - - role: elastic_beats - core_container_name: core - abci_logs_path: "{{ dashmate_logs_dir }}" + - role: metricbeat - name: Set up protx diff script hosts: masternodes diff --git a/ansible/group_vars/all b/ansible/group_vars/all index 3f932252..774ce1f7 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -157,6 +157,7 @@ kibana_encryptionkey: # Set to 50% of instance memory # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html elastic_heap_size: 8g +metricbeat_enabled: true elastic_compose_project_name: elastic elastic_path: '{{ dashd_home }}/{{ elastic_compose_project_name }}' diff --git a/ansible/roles/core_filebeat/defaults/main.yml b/ansible/roles/core_filebeat/defaults/main.yml new file mode 100644 index 00000000..7da5862f --- /dev/null +++ b/ansible/roles/core_filebeat/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +core_container_name: "dashd" diff --git a/ansible/roles/core_filebeat/tasks/main.yml b/ansible/roles/core_filebeat/tasks/main.yml new file mode 100644 index 00000000..11cd8d77 --- /dev/null +++ b/ansible/roles/core_filebeat/tasks/main.yml @@ -0,0 +1,32 @@ +--- + +- name: Load common filebeat config + ansible.builtin.include_vars: + file: common.yml + +- name: Get core container host info + community.docker.docker_host_info: + containers: true + containers_filters: + name: '{{ core_container_name }}' + register: core_host_info + +- name: Set container ids for core if core is running + ansible.builtin.set_fact: + core_container_id: '{{ core_host_info.containers[0].Id }}' + when: core_host_info.containers | length > 0 + +- name: Load core input config if core is running + ansible.builtin.include_vars: + file: core.yml + when: core_container_id is defined + +- name: Set up filebeat log monitoring + ansible.builtin.include_role: + name: geerlingguy.filebeat + +- name: Make sure filebeat is restarted + service: + name: filebeat + state: restarted + enabled: true diff --git a/ansible/roles/core_filebeat/vars/common.yml b/ansible/roles/core_filebeat/vars/common.yml new file mode 100644 index 00000000..98c12315 --- /dev/null +++ b/ansible/roles/core_filebeat/vars/common.yml @@ -0,0 +1,12 @@ +--- + +filebeat_version: 8.x +filebeat_package: "filebeat={{ elastic_version }}" +filebeat_output_logstash_enabled: false +filebeat_output_elasticsearch_enabled: true +filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}" +filebeat_output_elasticsearch_auth: + username: "{{ elastic_username }}" + password: "{{ elastic_password }}" +filebeat_enable_logging: true +filebeat_log_level: info diff --git a/ansible/roles/core_filebeat/vars/core.yml b/ansible/roles/core_filebeat/vars/core.yml new file mode 100644 index 00000000..12810d32 --- /dev/null +++ b/ansible/roles/core_filebeat/vars/core.yml @@ -0,0 +1,17 @@ +--- + +filebeat_inputs: + - type: container + enabled: true + index: "logs-core-{{ dash_network_name }}-%{[agent.version]}" + paths: + - '/var/lib/docker/containers/{{ core_container_id }}/*.log' + processors: + - add_fields: + target: event + fields: + dataset: "core-{{ dash_network_name }}" + - dissect: + tokenizer: "%{?timestamp} %{message}" + overwrite_keys: true + target_prefix: "" diff --git a/ansible/roles/dashmate/defaults/filebeat.yml b/ansible/roles/dashmate/defaults/filebeat.yml new file mode 100644 index 00000000..34488bba --- /dev/null +++ b/ansible/roles/dashmate/defaults/filebeat.yml @@ -0,0 +1,72 @@ +--- + +filebeat_version: 8.x +filebeat_package: "filebeat={{ elastic_version }}" +filebeat_output_logstash_enabled: false +filebeat_output_elasticsearch_enabled: true +filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}" +filebeat_output_elasticsearch_auth: + username: "{{ elastic_username }}" + password: "{{ elastic_password }}" +filebeat_enable_logging: true +filebeat_log_level: info +filebeat_inputs: + - type: log + enabled: true + index: "logs-core-{{ dash_network_name }}-%{[agent.version]}" + paths: + - "{{ dashmate_logs_dir }}/core.log*" + processors: + - add_fields: + target: event + fields: + dataset: "core-{{ dash_network_name }}" + - dissect: + tokenizer: "%{?timestamp} %{message}" + overwrite_keys: true + target_prefix: "" + - type: log + enabled: "{{ dashmate_platform_enable }}" + json.message_key: message + exclude_files: ['\.gz$'] + index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}" + paths: + - "{{ dashmate_logs_dir }}/drive-json.log*" + processors: + - timestamp: + field: json.timestamp + layouts: + - UNIX_MS + - add_fields: + target: event + fields: + dataset: "drive.abci-{{ dash_network_name }}" + - rename: + fields: + - from: "json.fields.message" + to: "message" + - from: "json.level" + to: "log.level" + ignore_missing: true + fail_on_error: true + - type: log + enabled: "{{ dashmate_platform_enable }}" + json.message_key: message + index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}" + paths: + - "{{ dashmate_logs_dir }}/tenderdash.log*" + processors: + - add_fields: + target: event + fields: + dataset: "drive.tenderdash-{{ dash_network_name }}" + - rename: + fields: + - from: "json.message" + to: "message" + ignore_missing: true + fail_on_error: true + - rename: + fields: + - from: "json.level" + to: "log.level" diff --git a/ansible/roles/dashmate/tasks/logs.yml b/ansible/roles/dashmate/tasks/logs.yml new file mode 100644 index 00000000..b886c6d2 --- /dev/null +++ b/ansible/roles/dashmate/tasks/logs.yml @@ -0,0 +1,49 @@ +--- + +- name: Create logs dir + ansible.builtin.file: + path: '{{ dashmate_logs_dir }}' + state: directory + owner: '{{ dashmate_user }}' + group: '{{ dashmate_group }}' + recurse: true + +- name: Configure log rotation + ansible.builtin.include_role: + name: arillso.logrotate + vars: + logrotate_applications: + - name: platform-logs + definitions: + - logs: + - '{{ dashmate_logs_dir }}/*.log' + options: + - rotate 7 + - daily + - maxsize 1G + - missingok + - notifempty + - copytruncate + - compress + - delaycompress + +- name: Ensure logrotate runs hourly under systemd timer + ansible.builtin.lineinfile: + path: /lib/systemd/system/logrotate.timer + regexp: '^OnCalendar=hourly' + insertafter: '^OnCalendar=daily' + line: OnCalendar=hourly + +- name: Load common filebeat config + ansible.builtin.include_vars: + file: filebeat.yml + +- name: Set up filebeat log monitoring + ansible.builtin.include_role: + name: geerlingguy.filebeat + +- name: Make sure filebeat is restarted + service: + name: filebeat + state: restarted + enabled: true diff --git a/ansible/roles/dashmate/tasks/main.yml b/ansible/roles/dashmate/tasks/main.yml index 25e6ba6e..44b0aadd 100644 --- a/ansible/roles/dashmate/tasks/main.yml +++ b/ansible/roles/dashmate/tasks/main.yml @@ -42,39 +42,8 @@ dir: '{{ dashmate_home }}' users: '{{ system_users + [dashmate_user_dict] }}' -- name: Create logs dir - ansible.builtin.file: - path: '{{ dashmate_logs_dir }}' - state: directory - owner: '{{ dashmate_user }}' - group: '{{ dashmate_group }}' - recurse: true - -- name: Configure log rotation - ansible.builtin.include_role: - name: arillso.logrotate - vars: - logrotate_applications: - - name: platform-logs - definitions: - - logs: - - '{{ dashmate_logs_dir }}/*.log' - options: - - rotate 7 - - daily - - maxsize 1G - - missingok - - notifempty - - copytruncate - - compress - - delaycompress - -- name: Ensure logrotate runs hourly under systemd timer - ansible.builtin.lineinfile: - path: /lib/systemd/system/logrotate.timer - regexp: '^OnCalendar=hourly' - insertafter: '^OnCalendar=daily' - line: OnCalendar=hourly +- name: Configure logs + ansible.builtin.import_tasks: ./logs.yml - name: Create dashmate config dir ansible.builtin.file: diff --git a/ansible/roles/elastic_beats/tasks/main.yml b/ansible/roles/dashmate_elastic_beats/tasks/main.yml similarity index 97% rename from ansible/roles/elastic_beats/tasks/main.yml rename to ansible/roles/dashmate_elastic_beats/tasks/main.yml index 2646f6e0..88b4df77 100644 --- a/ansible/roles/elastic_beats/tasks/main.yml +++ b/ansible/roles/dashmate_elastic_beats/tasks/main.yml @@ -58,6 +58,12 @@ ansible.builtin.include_role: name: geerlingguy.filebeat +- name: Make sure Filebeat is restarted + service: + name: filebeat + state: restarted + enabled: true + # TODO: Make sure we have retention policy for metrics - name: Set up metricbeat ansible.builtin.include_role: diff --git a/ansible/roles/elastic_beats/vars/common.yml b/ansible/roles/dashmate_elastic_beats/vars/common.yml similarity index 100% rename from ansible/roles/elastic_beats/vars/common.yml rename to ansible/roles/dashmate_elastic_beats/vars/common.yml diff --git a/ansible/roles/elastic_beats/vars/core.yml b/ansible/roles/dashmate_elastic_beats/vars/core.yml similarity index 100% rename from ansible/roles/elastic_beats/vars/core.yml rename to ansible/roles/dashmate_elastic_beats/vars/core.yml diff --git a/ansible/roles/elastic_beats/vars/drive.yml b/ansible/roles/dashmate_elastic_beats/vars/drive.yml similarity index 100% rename from ansible/roles/elastic_beats/vars/drive.yml rename to ansible/roles/dashmate_elastic_beats/vars/drive.yml diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml similarity index 100% rename from ansible/roles/elastic_beats/vars/tenderdash.yml rename to ansible/roles/dashmate_elastic_beats/vars/tenderdash.yml diff --git a/ansible/roles/metricbeat/tasks/main.yml b/ansible/roles/metricbeat/tasks/main.yml new file mode 100644 index 00000000..9d8979b4 --- /dev/null +++ b/ansible/roles/metricbeat/tasks/main.yml @@ -0,0 +1,59 @@ +--- + +- name: Set up metricbeat + ansible.builtin.include_role: + name: elastic.beats + vars: + beats_version: "{{ elastic_version }}" + beat: metricbeat + beat_conf: + setup: + dashboards: + enabled: true + kibana: + host: "{{ hostvars['logs-1'].private_ip }}:5601" + username: "{{ elastic_username }}" + password: "{{ elastic_password }}" + metricbeat: + modules: + - module: system + metricsets: + - cpu # CPU usage + - load # CPU load averages + - memory # Memory usage + - network # Network IO + - process # Per process metrics + - process_summary # Process summary + - uptime # System Uptime + - socket_summary # Socket summary + - core # Per CPU core usage + - diskio # Disk IO + - fsstat # File system summary metrics + - socket # Sockets and connection info (linux only) + enabled: true + period: 10s + processes: ['.*'] + + # Configure the metric types that are included by these metricsets. + cpu.metrics: ["percentages", "normalized_percentages"] # The other available option is ticks. + core.metrics: ["percentages"] # The other available option is ticks. + - module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "event" + - "healthcheck" + - "info" + - "memory" + - "network" + # - "network_summary" + hosts: ["unix:///var/run/docker.sock"] + period: 10s + enabled: true + output_conf: + elasticsearch: + hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}" + username: "{{ elastic_username }}" + password: "{{ elastic_password }}" + when: metricbeat_enabled