Oonidata

ansible/deploy-clickhouse.yml

@@ -0,0 +1,13 @@
+---
+- name: Deploy oonidata clickhouse hosts
+  hosts:
+    - notebook.ooni.org
+    - data1.htz-fsn.prod.ooni.nu
+    #- data2.htz-fsn.prod.ooni.nu
+    - data3.htz-fsn.prod.ooni.nu
+  become: true
+  tags:
+    - clickhouse
+  roles:
+    - prometheus_node_exporter
+    - oonidata_clickhouse

ansible/deploy-monitoring.yml

@@ -0,0 +1,12 @@
+---
+- name: Update monitoring config
+  hosts: monitoring.ooni.org
+  become: true
+  tags:
+    - monitoring
+  roles:
+    - prometheus
+    - prometheus_blackbox_exporter
+    - prometheus_alertmanager
+
+

ansible/deploy-tier0.yml

@@ -0,0 +1,19 @@
+---
+- name: Include monitoring playbook
+  ansible.builtin.import_playbook: deploy-monitoring.yml
+
+- name: Include clickhouse playbook
+  ansible.builtin.import_playbook: deploy-clickhouse.yml
+
+- name: Deploy oonidata worker nodes
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+  become: true
+  tags:
+    - oonidata_worker
+  roles:
+    - oonidata
+  vars:
+    enable_jupyterhub: false
+    enable_oonipipeline_worker: true
+    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"

ansible/deploy-tier2.yml

@@ -0,0 +1,25 @@
+---
+- name: Setup OpenVPN server
+  hosts: openvpn-server1.ooni.io
+  become: true
+  remote_user: root
+  roles:
+    - ssh_users
+
+- name: Deploy notebook host
+  hosts: notebook.ooni.org
+  become: true
+  tags:
+    - notebook
+  vars:
+    enable_oonipipeline_worker: false
+  roles:
+    - oonidata
+
+# commented out due to the fact it requires manual config of ~/.ssh/config
+#- name: Setup codesign box
+#  hosts: codesign-box
+#  become: true
+#  remote_user: ubuntu
+#  roles:
+#    - codesign_box

ansible/group_vars/all/vars.yml

@@ -27,3 +27,5 @@ admin_usernames: [ art, mehul ]
 root_usernames: [ art, mehul ]
 non_admin_usernames: [ ]
 deactivated_usernames: [ sbs, federico, sarath ]
+
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"

ansible/group_vars/clickhouse/vars.yml

@@ -7,6 +7,8 @@ nftables_clickhouse_allow:
     ip: 168.119.7.188
   - fqdn: notebook.ooni.org
     ip: 138.201.19.39
+  - fqdn: clickhouseproxy.dev.ooni.io
+    ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}"
 
 nftables_zookeeper_allow:
   - fqdn: data1.htz-fsn.prod.ooni.nu
@@ -24,7 +26,7 @@ clickhouse_config:
   max_connections: 4096
   keep_alive_timeout: 3
   max_concurrent_queries: 100
-  max_server_memory_usage: 0
+  max_server_memory_usage: 21001001000
   max_thread_pool_size: 10000
   max_server_memory_usage_to_ram_ratio: 0.9
   total_memory_profiler_step: 4194304
@@ -154,6 +156,10 @@ clickhouse_distributed_ddl:
 clickhouse_default_profiles:
   default:
     readonly: 2
+    max_memory_usage: 11001001000
+    use_uncompressed_cache: 0
+    load_balancing: random
+    max_partitions_per_insert_block: 100
   readonly:
     readonly: 1
   write:
@@ -194,3 +200,17 @@ clickhouse_default_quotas:
     result_rows: 0
     read_rows: 0
     execution_time: 0
+
+clickhouse_prometheus:
+  endpoint: "/metrics"
+  port: 9363
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  status_info: true
+
+prometheus_nginx_proxy_config:
+  - location: /metrics/node_exporter
+    proxy_pass: http://127.0.0.1:8100/metrics
+  - location: /metrics/clickhouse
+    proxy_pass: http://127.0.0.1:9363/metrics

ansible/inventory

@@ -1,22 +1,24 @@
-[all]
-# This requires manual setup of ~/.ssh/config
-#codesign-box
+[all:children]
+htz-fsn
+ghs-ams
 
-[prod]
-data.ooni.org
-oonidata.ooni.org
-monitoring.ooni.org
-openvpn-server1.ooni.io
+## Role tags
+
+[clickhouse]
 notebook.ooni.org
 data1.htz-fsn.prod.ooni.nu
 data2.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
 
-[dev]
-oonidatatest.ooni.nu
+## Location tags
 
-[clickhouse]
+[htz-fsn]
+data.ooni.org
+monitoring.ooni.org
 notebook.ooni.org
 data1.htz-fsn.prod.ooni.nu
 data2.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
+
+[ghs-ams]
+openvpn-server1.ooni.io

ansible/playbook.yml

@@ -7,63 +7,8 @@
   tags:
     - bootstrap
 
-- name: Update monitoring config
-  hosts: monitoring.ooni.org
-  become: true
-  tags:
-    - monitoring
-  roles:
-    - prometheus
-    - prometheus_blackbox_exporter
-    - prometheus_alertmanager
-
-- name: Setup OpenVPN server
-  hosts: openvpn-server1.ooni.io
-  become: true
-  remote_user: root
-  roles:
-    - ssh_users
-
-- name: Deploy oonidata clickhouse hosts
-  hosts:
-    - data1.htz-fsn.prod.ooni.nu
-    #- data2.htz-fsn.prod.ooni.nu
-    - data3.htz-fsn.prod.ooni.nu
-    - notebook.ooni.org
-  become: true
-  tags:
-    - clickhouse
-  roles:
-    #- tailnet
-    - oonidata_clickhouse
-
-- name: Deploy oonidata worker nodes
-  hosts:
-    - data1.htz-fsn.prod.ooni.nu
-  become: true
-  tags:
-    - oonidata_worker
-  roles:
-    - oonidata
-  vars:
-    enable_jupyterhub: false
-    enable_oonipipeline_worker: true
-    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"
-
-- name: Deploy notebook host
-  hosts: notebook.ooni.org
-  become: true
-  tags:
-    - notebook
-  vars:
-    enable_oonipipeline_worker: false
-  roles:
-    - oonidata
+- name: Include tier0 playbook
+  ansible.builtin.import_playbook: deploy-tier0.yml
 
-# commented out due to the fact it requires manual config of ~/.ssh/config
-#- name: Setup codesign box
-#  hosts: codesign-box
-#  become: true
-#  remote_user: ubuntu
-#  roles:
-#    - codesign_box
+- name: Include tier2 playbook
+  ansible.builtin.import_playbook: deploy-tier2.yml

ansible/requirements.yml

@@ -1,7 +1,6 @@
 - src: willshersystems.sshd
 - src: nginxinc.nginx
 - src: geerlingguy.certbot
-- src: geerlingguy.node_exporter
 - src: artis3n.tailscale
 - src: https://github.com/idealista/clickhouse_role
   scm: git

ansible/roles/bootstrap/tasks/main.yml

@@ -55,11 +55,6 @@
   tags:
     - nftables
 
-- ansible.builtin.include_role:
-    name: prometheus_node_exporter
-  tags:
-    - node_exporter
-
 - name: Configure journald
   tags:
     - journald

ansible/roles/nginx/defaults/main.yml

@@ -0,0 +1 @@
+nginx_user: nginx