Data.ooni.org (#105)

Add support for production OONI data pipeline v5 deployment in ansible.

It also moves some of the ooni/sysadmin roles to devops and updates them
to new patterns.

There is also a new node_exporter role that's part of the bootstrap role
for performing host monitoring. Eventually this should be deployed on
every new host which we manually bootstrap outside of AWS.

Implements: #82

ansible/README.md

@@ -8,7 +8,12 @@ pyenv activate ooni-devops
 
 Install deps:
 ```
-pip install ansible dnspython boto3
+pip install ansible dnspython boto3 passlib
+```
+
+Install ansible galaxy modules:
+```
+ansible-galaxy install -r requirements.yml
 ```
 
 Setup AWS credentials, you should add 2 profiles called `oonidevops_user_dev` and `oonidevops_user_prod` which have access to the development and production environment respectively

ansible/group_vars/all/vars.yml

@@ -26,4 +26,4 @@ ssh_users:
 admin_usernames: [ art, majakomel, mehul, norbel ]
 root_usernames: [ art, mehul ]
 non_admin_usernames: [ agrabeli ]
-deactivated_usernames: [ sbs, federico, sarath ]
+deactivated_usernames: [ sbs, federico, sarath ]

ansible/group_vars/dev/vars.yml

@@ -0,0 +1 @@
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_dev') }}"

ansible/group_vars/prod/vars.yml

@@ -0,0 +1 @@
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"

ansible/host_vars/data.ooni.org

@@ -55,6 +55,6 @@ ssh_users:
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMJYsbeTjdma5cKyZISOFQfHbwwlZbWugPx9haeOx1UR"
      ]
 admin_usernames: [ art, majakomel, mehul, norbel ]
-non_admin_usernames: [ ain, siti, ingrid, joss ]
+non_admin_usernames: [ ain, siti, ingrid, joss, vasilis ]
 jupyterhub_allowed_users: "{{ ssh_users }}"
-admin_group_name: adm
+admin_group_name: adm

ansible/host_vars/oonidata.ooni.org

@@ -0,0 +1,2 @@
+admin_group_name: adm
+tls_cert_dir: /var/lib/dehydrated/certs

ansible/inventory

@@ -1,6 +1,13 @@
 [all]
-monitoring.ooni.org
-openvpn-server1.ooni.io
 # This requires manual setup of ~/.ssh/config
 #codesign-box
+
+[prod]
 data.ooni.org
+oonidata.ooni.org
+monitoring.ooni.org
+openvpn-server1.ooni.io
+notebook.ooni.org
+
+[dev]
+oonidatatest.ooni.nu

ansible/playbook-bootstrap.yml

@@ -0,0 +1,8 @@
+# This playbook is to be run on hosts that don't support bootstrapping the base
+# OS setup with something other than ansible (eg. cloud-init)
+- name: Bootstrap the ssh_users on target host
+  hosts: all
+  remote_user: root
+  roles:
+    - ssh_users
+    - bootstrap

ansible/playbook.yml

@@ -1,4 +1,10 @@
 ---
+- name: Ensure all hosts are bootstrapped correctly
+  hosts: all
+  become: yes
+  roles:
+    - bootstrap
+
 - name: ClickHouse servers
   hosts: clickhouse_servers
   user: admin
@@ -36,6 +42,12 @@
   roles:
     - ssh_users
 
+- name: Deploy oonidata hosts
+  hosts: oonidata.ooni.org
+  become: true
+  roles:
+    - oonidata
+
 # commented out due to the fact it requires manual config of ~/.ssh/config
 #- name: Setup codesign box
 #  hosts: codesign-box

ansible/requirements.yml

@@ -1 +1,4 @@
 - src: willshersystems.sshd
+- src: nginxinc.nginx
+- src: geerlingguy.certbot
+- src: geerlingguy.node_exporter

ansible/roles/bootstrap/tasks/main.yml

@@ -0,0 +1,45 @@
+- ansible.builtin.include_role:
+    name: ssh_users
+  tags:
+    - ssh_users
+
+- name: Set the hostname to inventory_hostname
+  ansible.builtin.hostname:
+    name: "{{ inventory_hostname }}"
+
+- name: Install common packages
+  ansible.builtin.apt:
+    name:
+      - bash-completion
+      - ca-certificates
+      - curl
+      - file
+      - git
+      - htop
+      - iotop
+      - lsof
+      - lvm2
+      - man-db
+      - mtr
+      - net-tools
+      - openssl
+      - python3-passlib
+      - rsync
+      - screen
+      - strace
+      - tcpdump
+      - tmux
+      - vim
+    state: latest
+    update_cache: yes
+    install_recommends: no
+
+- ansible.builtin.include_role:
+    name: nftables
+  tags:
+    - nftables
+
+- ansible.builtin.include_role:
+    name: prometheus_node_exporter
+  tags:
+    - node_exporter

ansible/roles/miniconda/defaults/main.yml

@@ -0,0 +1,2 @@
+miniconda_install_dir: /opt/miniconda
+admin_group_name: admin

ansible/roles/miniconda/tasks/install.yml

@@ -0,0 +1,23 @@
+---
+- name: Ensure miniconda directory exists
+  ansible.builtin.file:
+    path: "{{ miniconda_install_dir }}"
+    state: directory
+    owner: miniconda
+    group: "{{ admin_group_name }}"
+
+- name: Download the miniconda installer
+  ansible.builtin.get_url:
+    url: "https://repo.anaconda.com/miniconda/Miniconda3-py312_24.7.1-0-Linux-x86_64.sh"
+    dest: "{{ miniconda_install_dir }}/miniconda.sh"
+    checksum: "sha256:33442cd3813df33dcbb4a932b938ee95398be98344dff4c30f7e757cd2110e4f"
+    mode: "0700"
+
+- name: Run the miniconda installer
+  ansible.builtin.shell: |
+    bash {{ miniconda_install_dir }}/miniconda.sh -b -u -p {{ miniconda_install_dir }}
+
+- name: Delete installer
+  ansible.builtin.file:
+    path: "{{ miniconda_install_dir }}/miniconda.sh"
+    state: absent

ansible/roles/miniconda/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Ensure miniconda user exists
+  ansible.builtin.user:
+    name: miniconda
+    shell: /bin/false
+
+- name: Check if Miniconda is installed
+  ansible.builtin.stat:
+    path: "{{ miniconda_install_dir }}/bin/conda"
+  register: miniconda_bin
+
+- include_tasks: install.yml
+  when: not miniconda_bin.stat.exists
+
+- name: "install conda packages"
+  ansible.builtin.shell:
+    cmd: "{{ miniconda_install_dir }}/bin/conda install -y {{ item }}"
+  loop:
+    - pandas
+    - numpy
+    - altair

ansible/roles/nftables/README.adoc

@@ -0,0 +1,25 @@
+Install nftables based firewall
+
+Set up /etc/ooni/nftables/
+
+Rules for specific services are *not* configured by this role
+
+When creating rules to accept TCP traffic from any IPv4/6 address,
+files are named with the port number to detect collisions.
+
+Example (also see roles/nftables/tasks/main.yml):
+
+/etc/ooni/nftables/tcp/8080.nft
+
+```
+add rule inet filter input tcp dport 8080 counter accept comment "MyService"
+```
+
+
+Otherwise:
+
+/etc/ooni/nftables/tcp/5432_postgres_internal.nft
+
+```
+add rule inet filter input ip saddr { 10.0.0.0/8, 192.168.0.0/16 } tcp dport 5432 counter accept comment "Internal PostgreSQL"
+```

ansible/roles/nftables/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: Reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded

ansible/roles/nftables/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: Install nftables
+  ansible.builtin.apt:
+    cache_valid_time: 86400
+    name: nftables
+  tags:
+    - nftables
+
+- name: create config dir
+  ansible.builtin.file:
+    path: /etc/ooni/nftables/tcp
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  tags:
+    - nftables
+
+- name: allow SSH
+  ansible.builtin.blockinfile:
+    path: /etc/ooni/nftables/tcp/22.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 22 counter accept comment "Incoming SSH"
+  tags:
+    - nftables
+
+- name: Overwrite nftables.conf
+  ansible.builtin.template:
+    src: templates/nftables.conf
+    dest: /etc/nftables.conf
+    mode: 0755
+    owner: root
+  notify:
+    - Reload nftables
+  tags:
+    - nftables
+
+- name: enable nftables service
+  ansible.builtin.systemd_service:
+    name: nftables
+    enabled: yes
+    state: started
+  tags:
+    - nftables

ansible/roles/nftables/templates/nftables.conf

@@ -0,0 +1,41 @@
+#!/usr/sbin/nft -f
+#
+# Nftables configuration script
+#
+# Managed by ansible
+# roles/nftables/templates/nftables.conf
+#
+# The ruleset is applied atomically
+
+flush ruleset
+
+table inet filter {
+  chain input {
+    type filter hook input priority 0;
+    policy drop;
+    iif lo accept comment "Accept incoming traffic from localhost"
+    ct state invalid drop
+    ct state established,related accept comment "Accept traffic related to outgoing connections"
+    icmp type echo-request accept
+    icmpv6 type echo-request counter packets 0 bytes 0 accept
+    icmpv6 type { nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 1 accept
+    icmpv6 type { nd-router-advert, nd-neighbor-solicit, nd-neighbor-advert } ip6 hoplimit 255 counter packets 1 bytes 72 accept
+  }
+
+  chain forward {
+    type filter hook forward priority 0;
+    policy accept;
+  }
+
+  chain output {
+    type filter hook output priority 0;
+    policy accept;
+  }
+}
+
+# Configure TCP traffic rules
+include "/etc/ooni/nftables/tcp/*.nft"
+
+# Configure any other rule
+include "/etc/ooni/nftables/*.nft"
+