refactor: split elastic_beat into core_filebeat, metricbeat and dashmate

ansible/deploy.yml

@@ -78,6 +78,7 @@
   become: true
   roles:
     - elastic_stack
+    - metricbeat
 
 - name: Setup load tester
   hosts: load_test
@@ -90,7 +91,8 @@
   hosts: metrics
   become: true
   roles:
-    - role: metrics
+    - metrics
+    - metricbeat
 
 - name: Set up miners
   hosts: miners
@@ -100,8 +102,8 @@
     - role: dashd
       tags:
         - dashd
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 - name: Set up mixers
   hosts: mixer_nodes
@@ -111,8 +113,8 @@
     - role: dashd
       tags:
         - dashd
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 - name: Set up core and tenderdash on seed nodes
   hosts: seed_nodes
@@ -133,8 +135,8 @@
       tags:
         - dashd
     - role: tenderdash
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 - name: Set up core on masternodes
   hosts: masternodes
@@ -158,8 +160,8 @@
       tags:
         - dashd
     - mn_status_report
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 
 # Start network
@@ -189,8 +191,8 @@
       dashd_zmq: true
       dashd_listen: true
     - insight
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
   tags:
     - web
 
@@ -205,8 +207,8 @@
       enable_wallet: true
       tags:
         - dashd
-    - role: elastic_beats
-      core_container_name: dashd
+    - core_filebeat
+    - metricbeat
 
 # Register masternodes and set sporks
 
@@ -283,9 +285,7 @@
     - role: dash_cli
     - role: dashmate
     - role: mn_status_report
-    - role: elastic_beats
-      core_container_name: core
-      abci_logs_path: "{{ dashmate_logs_dir }}"
+    - role: metricbeat
 
 - name: Set up protx diff script
   hosts: masternodes

ansible/group_vars/all

@@ -157,6 +157,7 @@ kibana_encryptionkey:
 # Set to 50% of instance memory
 # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
 elastic_heap_size: 8g
+metricbeat_enabled: true
 
 elastic_compose_project_name: elastic
 elastic_path: '{{ dashd_home }}/{{ elastic_compose_project_name }}'

ansible/roles/core_filebeat/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+
+core_container_name: "dashd"

ansible/roles/core_filebeat/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+
+- name: Load common filebeat config
+  ansible.builtin.include_vars:
+    file: common.yml
+
+- name: Get core container host info
+  community.docker.docker_host_info:
+    containers: true
+    containers_filters:
+      name: '{{ core_container_name }}'
+  register: core_host_info
+
+- name: Set container ids for core if core is running
+  ansible.builtin.set_fact:
+    core_container_id: '{{ core_host_info.containers[0].Id }}'
+  when: core_host_info.containers | length > 0
+
+- name: Load core input config if core is running
+  ansible.builtin.include_vars:
+    file: core.yml
+  when: core_container_id is defined
+
+- name: Set up filebeat log monitoring
+  ansible.builtin.include_role:
+    name: geerlingguy.filebeat
+
+- name: Make sure filebeat is restarted
+  service:
+    name: filebeat
+    state: restarted
+    enabled: true

ansible/roles/core_filebeat/vars/common.yml

@@ -0,0 +1,12 @@
+---
+
+filebeat_version: 8.x
+filebeat_package: "filebeat={{ elastic_version }}"
+filebeat_output_logstash_enabled: false
+filebeat_output_elasticsearch_enabled: true
+filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+filebeat_output_elasticsearch_auth:
+  username: "{{ elastic_username }}"
+  password: "{{ elastic_password }}"
+filebeat_enable_logging: true
+filebeat_log_level: info

ansible/roles/core_filebeat/vars/core.yml

@@ -0,0 +1,17 @@
+---
+
+filebeat_inputs:
+  - type: container
+    enabled: true
+    index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - '/var/lib/docker/containers/{{ core_container_id }}/*.log'
+    processors:
+      - add_fields:
+          target: event
+          fields:
+            dataset: "core-{{ dash_network_name }}"
+      - dissect:
+          tokenizer: "%{?timestamp} %{message}"
+          overwrite_keys: true
+          target_prefix: ""

ansible/roles/dashmate/defaults/filebeat.yml

@@ -0,0 +1,72 @@
+---
+
+filebeat_version: 8.x
+filebeat_package: "filebeat={{ elastic_version }}"
+filebeat_output_logstash_enabled: false
+filebeat_output_elasticsearch_enabled: true
+filebeat_output_elasticsearch_hosts: "{{ query('inventory_hostnames', 'logs') | map('extract', hostvars, ['private_ip']) | map('~(item) => item + \":9200\"') | list }}"
+filebeat_output_elasticsearch_auth:
+  username: "{{ elastic_username }}"
+  password: "{{ elastic_password }}"
+filebeat_enable_logging: true
+filebeat_log_level: info
+filebeat_inputs:
+  - type: log
+    enabled: true
+    index: "logs-core-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - "{{ dashmate_logs_dir }}/core.log*"
+    processors:
+      - add_fields:
+          target: event
+          fields:
+            dataset: "core-{{ dash_network_name }}"
+      - dissect:
+          tokenizer: "%{?timestamp} %{message}"
+          overwrite_keys: true
+          target_prefix: ""
+  - type: log
+    enabled: "{{ dashmate_platform_enable }}"
+    json.message_key: message
+    exclude_files: ['\.gz$']
+    index: "logs-drive.abci-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - "{{ dashmate_logs_dir }}/drive-json.log*"
+    processors:
+      - timestamp:
+          field: json.timestamp
+          layouts:
+            - UNIX_MS
+      - add_fields:
+          target: event
+          fields:
+            dataset: "drive.abci-{{ dash_network_name }}"
+      - rename:
+          fields:
+            - from: "json.fields.message"
+              to: "message"
+            - from: "json.level"
+              to: "log.level"
+          ignore_missing: true
+          fail_on_error: true
+  - type: log
+    enabled: "{{ dashmate_platform_enable }}"
+    json.message_key: message
+    index: "logs-drive.tenderdash-{{ dash_network_name }}-%{[agent.version]}"
+    paths:
+      - "{{ dashmate_logs_dir }}/tenderdash.log*"
+    processors:
+      - add_fields:
+          target: event
+          fields:
+            dataset: "drive.tenderdash-{{ dash_network_name }}"
+      - rename:
+          fields:
+            - from: "json.message"
+              to: "message"
+          ignore_missing: true
+          fail_on_error: true
+      - rename:
+          fields:
+            - from: "json.level"
+              to: "log.level"

ansible/roles/dashmate/tasks/logs.yml

@@ -0,0 +1,49 @@
+---
+
+- name: Create logs dir
+  ansible.builtin.file:
+    path: '{{ dashmate_logs_dir }}'
+    state: directory
+    owner: '{{ dashmate_user }}'
+    group: '{{ dashmate_group }}'
+    recurse: true
+
+- name: Configure log rotation
+  ansible.builtin.include_role:
+    name: arillso.logrotate
+  vars:
+    logrotate_applications:
+      - name: platform-logs
+        definitions:
+          - logs:
+              - '{{ dashmate_logs_dir }}/*.log'
+            options:
+              - rotate 7
+              - daily
+              - maxsize 1G
+              - missingok
+              - notifempty
+              - copytruncate
+              - compress
+              - delaycompress
+
+- name: Ensure logrotate runs hourly under systemd timer
+  ansible.builtin.lineinfile:
+    path: /lib/systemd/system/logrotate.timer
+    regexp: '^OnCalendar=hourly'
+    insertafter: '^OnCalendar=daily'
+    line: OnCalendar=hourly
+
+- name: Load common filebeat config
+  ansible.builtin.include_vars:
+    file: filebeat.yml
+
+- name: Set up filebeat log monitoring
+  ansible.builtin.include_role:
+    name: geerlingguy.filebeat
+
+- name: Make sure filebeat is restarted
+  service:
+    name: filebeat
+    state: restarted
+    enabled: true

ansible/roles/dashmate/tasks/main.yml

@@ -42,39 +42,8 @@
       dir: '{{ dashmate_home }}'
     users: '{{ system_users + [dashmate_user_dict] }}'
 
-- name: Create logs dir
-  ansible.builtin.file:
-    path: '{{ dashmate_logs_dir }}'
-    state: directory
-    owner: '{{ dashmate_user }}'
-    group: '{{ dashmate_group }}'
-    recurse: true
-
-- name: Configure log rotation
-  ansible.builtin.include_role:
-    name: arillso.logrotate
-  vars:
-    logrotate_applications:
-      - name: platform-logs
-        definitions:
-          - logs:
-              - '{{ dashmate_logs_dir }}/*.log'
-            options:
-              - rotate 7
-              - daily
-              - maxsize 1G
-              - missingok
-              - notifempty
-              - copytruncate
-              - compress
-              - delaycompress
-
-- name: Ensure logrotate runs hourly under systemd timer
-  ansible.builtin.lineinfile:
-    path: /lib/systemd/system/logrotate.timer
-    regexp: '^OnCalendar=hourly'
-    insertafter: '^OnCalendar=daily'
-    line: OnCalendar=hourly
+- name: Configure logs
+  ansible.builtin.import_tasks: ./logs.yml
 
 - name: Create dashmate config dir
   ansible.builtin.file:

ansible/roles/elastic_beats/tasks/main.yml → ansible/roles/dashmate_elastic_beats/tasks/main.yml

@@ -58,6 +58,12 @@
   ansible.builtin.include_role:
     name: geerlingguy.filebeat
 
+- name: Make sure Filebeat is restarted
+  service:
+    name: filebeat
+    state: restarted
+    enabled: true
+
 # TODO: Make sure we have retention policy for metrics
 - name: Set up metricbeat
   ansible.builtin.include_role: