Prod pipeline v5 (#112)

Devops related work to get OONI Pipeline v5 production ready

ansible/group_vars/all/vars.yml

@@ -23,7 +23,7 @@ ssh_users:
     keys:
       - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBXprrutdT6AhrV9hWBKjyzq6RqGmCBWpWxi3qwJyRcBJfkiEYKV9QWl3H0g/Sg9JzLd9lWG2yfAai7cyBAT4Ih0+OhwQ0V7wkhBn4YkNjs7d4BGPHjuLIywS9VtmiyH7VafikMjmqPLL/uPBIbRrx9RuSfLkAuN9XFZpVmqzWY8ePpcRCvnG6ucPxEY8o+4j5nfTrgxSaIT31kH16/PFJe07tn1SZjxZE4sZTz/p9xKt6s8HXmlP3RdnXSpXWmH8ZwYDrNhkcH8m6mC3giiqSKThFdwvQVflRRvn9pAlUOhy6KIBtAt1KobVJtOCPrrkcLhQ1C+2P9wKhfYspCGrScFGnrUqumLxPpwlqILxJvmgqGAtkm8Ela9f2D9sEv8CUv5x9XptZKlyRhtOLixvLYoJlwfXXnmXa8T1pg8+4063BhHUOu/bg0InpSp3hdscOfk0R8FtDlXnn6COwbPXynIt4PxzIxD/WQhP0ymgH3ky6ClB5wRBVhOqYvxQw32n2QFS9A5ocga+nATiOE7BTOufgmDCA/OIXfJ/GukXRaMCBsvlx7tObHS1LOMt0I+WdoOEjI0ARUrFzwoiTrs9QYmd922e7S35EnheT3JjnCTjebJrCNtwritUy8vjsN/M27wJs7MAXleT7drwXXnm+3xYrH+4KQ+ru0dxMe1zfBw== [email protected]"
 
-admin_usernames: [ art, majakomel, mehul, norbel ]
+admin_usernames: [ art, mehul ]
 root_usernames: [ art, mehul ]
-non_admin_usernames: [ agrabeli ]
-deactivated_usernames: [ sbs, federico, sarath ]
+non_admin_usernames: [ ]
+deactivated_usernames: [ sbs, federico, sarath ]

ansible/group_vars/clickhouse/vars.yml

@@ -0,0 +1,177 @@
+nftables_clickhouse_allow:
+  - fqdn: data1.htz-fsn.prod.ooni.nu
+    ip: 142.132.254.225
+  - fqdn: data2.htz-fsn.prod.ooni.nu
+    ip: 88.198.54.12
+  - fqdn: data3.htz-fsn.prod.ooni.nu
+    ip: 168.119.7.188
+  - fqdn: notebook.ooni.org
+    ip: 138.201.19.39
+
+nftables_zookeeper_allow:
+  - fqdn: data1.htz-fsn.prod.ooni.nu
+    ip: 142.132.254.225
+  - fqdn: data2.htz-fsn.prod.ooni.nu
+    ip: 88.198.54.12
+  - fqdn: data3.htz-fsn.prod.ooni.nu
+    ip: 168.119.7.188
+  - fqdn: notebook.ooni.org
+    ip: 138.201.19.39
+
+clickhouse_version: 24.8.6.70
+
+clickhouse_config:
+  max_connections: 4096
+  keep_alive_timeout: 3
+  max_concurrent_queries: 100
+  max_server_memory_usage: 0
+  max_thread_pool_size: 10000
+  max_server_memory_usage_to_ram_ratio: 0.9
+  total_memory_profiler_step: 4194304
+  total_memory_tracker_sample_probability: 0
+  uncompressed_cache_size: 8589934592
+  mark_cache_size: 5368709120
+  # max_open_files: 262144
+  mmap_cache_size: 1000
+  compiled_expression_cache_size: 134217728
+  compiled_expression_cache_elements_size: 10000
+  # tmp_policy: tmp
+  default_profile: default
+  custom_settings_prefixes: ""
+  system_profile: write
+  # buffer_profile: default
+  default_database: default
+  # timezone:
+  # umask: 027
+  mlock_executable: true
+  remap_executable: true
+  builtin_dictionaries_reload_interval: 3600
+  max_session_timeout: 3600
+  default_session_timeout: 60
+  # regions_hierarchy_file: /opt/geo/regions_hierarchy.txt
+  # regions_names_files_path: /opt/geo/
+  # top_level_domains_path: /var/lib/clickhouse/top_level_domains/
+  # top_level_domains:  # Path to the list is under top_level_domains_path
+  #   - domain:
+  #     name: example_name
+  #     path: /path/to/example_name.dat
+  dictionaries_config: "*_dictionary.xml"
+  user_defined_executable_functions_config: "*_function.xml"
+  # max_table_size_to_drop: 0
+  # max_partition_size_to_drop: 0
+  format_schema_path: /var/lib/clickhouse/format_schemas/
+  # disable_internal_dns_cache: 1
+
+clickhouse_keeper:
+  tcp_port: 9181
+  log_storage_path: /var/lib/clickhouse/coordination/log
+  snapshot_storage_path: /var/lib/clickhouse/coordination/snapshots
+  coordination_settings:
+    operation_timeout_ms: 10000
+    session_timeout_ms: 30000
+    raft_logs_level: trace
+  keeper_servers:
+  - keeper_server:
+    server: data1.htz-fsn.prod.ooni.nu
+    id: 1
+    hostname: clickhouse1.prod.ooni.io
+    port: 9234
+
+  #- keeper_server:
+  #  server: data2.htz-fsn.prod.ooni.nu
+  #  id: 2
+  #  hostname: clickhouse2.prod.ooni.io
+  #  port: 9234
+
+  - keeper_server:
+    server: data3.htz-fsn.prod.ooni.nu
+    id: 3
+    hostname: clickhouse3.prod.ooni.io
+    port: 9234
+
+  - keeper_server:
+    server: notebook.ooni.org
+    id: 4
+    hostname: notebook.ooni.org
+    port: 9234
+
+clickhouse_zookeeper:
+  - node:
+    host: clickhouse1.prod.ooni.io
+    port: 9181
+  - node:
+    host: clickhouse3.prod.ooni.io
+    port: 9181
+  - node:
+    host: notebook.ooni.org
+    port: 9181
+
+clickhouse_remote_servers:
+  - server:
+    servername: oonidata_cluster
+    secret: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_oonidata_cluster_secret', profile='oonidevops_user_prod') }}"
+    shards:
+      - shard:
+        internal_replication: true
+        replicas:
+          - replica:
+            host: clickhouse1.prod.ooni.io
+            port: 9000
+          #- replica:
+          #  host: clickhouse2.prod.ooni.io
+          #  port: 9000
+          - replica:
+            host: clickhouse3.prod.ooni.io
+            port: 9000
+
+clickhouse_macros:
+  - macro: |
+      <shard>01</shard>
+      <replica>01</replica>
+    server:
+      - data1.htz-fsn.prod.ooni.nu
+  - macro: |
+      <shard>01</shard>
+      <replica>02</replica>
+    server:
+      - data2.htz-fsn.prod.ooni.nu
+  - macro: |
+      <shard>01</shard>
+      <replica>03</replica>
+    server:
+      - data3.htz-fsn.prod.ooni.nu
+  - macro: |
+      <cluster>oonidata_cluster</cluster>
+
+clickhouse_distributed_ddl:
+  path: "/clickhouse/task_queue/ddl"
+  profile: "write"
+  pool_size: 1
+  task_max_lifetime: 604800
+  cleanup_delay_period: 60
+  max_tasks_in_queue: 1000
+
+clickhouse_default_profiles:
+  default:
+    readonly: 2
+  write:
+    readonly: 0
+
+clickhouse_listen_hosts:
+  - "::"
+
+clickhouse_default_users:
+  - user:
+    name: default
+    password:
+    networks:
+      - "127.0.0.1"
+    profile: default
+    quota: default
+  - user:
+    name: write
+    password_sha256_hex: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}"
+    networks:
+      - "0.0.0.0/0"
+    profile: write
+    quota: default

ansible/group_vars/dev/vars.yml

@@ -1 +1,3 @@
-prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"
+admin_usernames: [ art, mehul, norbel, majakomel ]
+non_admin_usernames: [ agrabeli ]

ansible/group_vars/prod/vars.yml

@@ -1 +1,7 @@
-prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"
+tailscale_authkey: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/tailscale_authkey_devops', profile='oonidevops_user_prod') }}"
+tailscale_tags:
+  - "devops-prod"
+tailscale_oauth_ephemeral: false
+admin_usernames: [ art, mehul ]
+non_admin_usernames: [ ]

ansible/host_vars/data3.htz-fsn.prod.ooni.nu

@@ -0,0 +1,2 @@
+non_admin_usernames: [ ]
+clickhouse_base_path: /data/clickhouse

ansible/host_vars/notebook.ooni.org

@@ -64,8 +64,32 @@ ssh_users:
     comment: "Ben Ginoe"
     keys:
         - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOWdWCATiHUAzoS3mn3pFMIYDmi3n4Ekuzv5cEtvV0W1 root@parrot"
-
 admin_usernames: [ art, agrabeli, majakomel, mehul, norbel ]
 non_admin_usernames: [ ain, siti, ingrid, joss, vasilis, michael, benginoe ]
 jupyterhub_allowed_users: "{{ ssh_users }}"
 admin_group_name: admin
+
+clickhouse_default_profiles:
+  default:
+    readonly: 2
+  write:
+    readonly: 0
+
+clickhouse_listen_hosts:
+  - "127.0.0.1"
+
+clickhouse_default_users:
+  - user:
+    name: default
+    password:
+    networks:
+      - "127.0.0.1"
+    profile: default
+    quota: default
+  - user:
+    name: write
+    password_sha256_hex: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}"
+    networks:
+      - "127.0.0.1"
+    profile: write
+    quota: default

ansible/inventory

@@ -8,6 +8,15 @@ oonidata.ooni.org
 monitoring.ooni.org
 openvpn-server1.ooni.io
 notebook.ooni.org
+data1.htz-fsn.prod.ooni.nu
+data2.htz-fsn.prod.ooni.nu
+data3.htz-fsn.prod.ooni.nu
 
 [dev]
 oonidatatest.ooni.nu
+
+[clickhouse]
+notebook.ooni.org
+data1.htz-fsn.prod.ooni.nu
+data2.htz-fsn.prod.ooni.nu
+data3.htz-fsn.prod.ooni.nu

ansible/playbook-bootstrap.yml

@@ -4,5 +4,4 @@
   hosts: all
   remote_user: root
   roles:
-    - ssh_users
     - bootstrap

ansible/playbook.yml

@@ -4,53 +4,57 @@
   become: yes
   roles:
     - bootstrap
-
-- name: ClickHouse servers
-  hosts: clickhouse_servers
-  user: admin
-  become: true
-  vars:
-    clickhouse_reader_password: "{{ lookup('env', 'CLICKHOUSE_READER_PASSWORD') }}"
-  roles:
-    - clickhouse
-  handlers:
-    - name: Restart clickhouse-server
-      ansible.builtin.service:
-        name: clickhouse-server
-        state: restarted
+  tags:
+    - bootstrap
 
 - name: Update monitoring config
   hosts: monitoring.ooni.org
   become: true
+  tags:
+    - monitoring
   roles:
     - prometheus
     - prometheus_blackbox_exporter
     - prometheus_alertmanager
 
-- name: Deploy data.ooni.org host
-  hosts: data.ooni.org
-  become: true
-  roles:
-    #- clickhouse
-    - ssh_users
-    #- jupyterhub
-
 - name: Setup OpenVPN server
   hosts: openvpn-server1.ooni.io
   become: true
   remote_user: root
   roles:
     - ssh_users
 
-- name: Deploy oonidata hosts
-  hosts: oonidata.ooni.org
+- name: Deploy oonidata clickhouse hosts
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+    #- data2.htz-fsn.prod.ooni.nu
+    - data3.htz-fsn.prod.ooni.nu
+    - notebook.ooni.org
   become: true
+  tags:
+    - clickhouse
+  roles:
+    - tailnet
+    - oonidata_clickhouse
+
+- name: Deploy oonidata worker nodes
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+  become: true
+  tags:
+    - oonidata_worker
   roles:
     - oonidata
+  vars:
+    enable_jupyterhub: false
+    enable_oonipipeline_worker: true
+    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_secret', 'oonidevops/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"
 
-- name: Deploy notebook hosts
+- name: Deploy notebook host
   hosts: notebook.ooni.org
   become: true
+  tags:
+    - notebook
   vars:
     enable_oonipipeline_worker: false
   roles:

ansible/requirements.yml

@@ -1,4 +1,9 @@
 - src: willshersystems.sshd
 - src: nginxinc.nginx
 - src: geerlingguy.certbot
-- src: geerlingguy.node_exporter
+- src: geerlingguy.node_exporter
+- src: artis3n.tailscale
+- src: https://github.com/idealista/clickhouse_role
+  scm: git
+  version: 3.5.1
+  name: idealista.clickhouse_role

ansible/roles/bootstrap/handlers/main.yml

@@ -0,0 +1,18 @@
+- name: Restart chrony
+  ansible.builtin.systemd_service:
+    name: chrony.service
+    state: restarted
+
+- name: Restart systemd-resolved
+  ansible.builtin.systemd_service:
+    name: systemd-resolved.service
+    state: restarted
+
+- name: Test systemd-resolved
+  ansible.builtin.shell: resolvectl query go.dnscheck.tools --cache=no
+
+- name: Restart systemd-journald
+  ansible.builtin.systemd_service:
+    name: systemd-journald.service
+    state: restarted
+    enabled: yes