Merge pull request #10 from 116davinder/feature/mtls-support

zookeeper quorum sasl + mtls support

.gitignore

@@ -11,3 +11,5 @@ pulumi/__pycache__
 pulumi/*/*.pyc
 pulumi/*/venv
 pulumi/*/__pycache__
+files/certs
+*.tar.gz

Vagrantfile

@@ -6,16 +6,17 @@ Vagrant.configure("2") do |config|
     (1..cluster_nodes).each do |i|
       config.vm.define "zookeeper-#{i}" do |node|
         node.vm.box = "ubuntu/bionic64"
-        node.vm.hostname  = "zookeeper#{i}"
-        node.vm.network :private_network, ip: "192.168.56.10#{i}"
-        #node.vm.provision :hosts, :sync_hosts => true
+        node.vm.hostname  = "zookeeper#{i}.localhost"
+        node.vm.network :private_network, ip: "192.168.56.11#{i}"
+        node.vm.provision :hosts, :add_localhost_hostnames => false, :sync_hosts => true   # required to autogenerate /etc/hosts on all nodes
       end
   end
   # Setting CPU and Memory for All machines
   config.vm.provider "virtualbox" do |vb|
     vb.gui = false
-    vb.memory = "512"
+    vb.memory = "1024"
     vb.cpus =  1
+    vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ] # used for wsl2
   end
 
 # SSH config to use your local ssh key for auth instead of username/password

clusterJvmConfigs.yml → clusterConfigsUpdate.yml

@@ -1,14 +1,19 @@
 ---
 
+- hosts: clusterNodes
+  gather_facts: true
+  tasks:
+    - name: make sure gather facts is triggered on all nodes required for zoo.cfg
+      ansible.builtin.debug:
+        var: ansible_fqdn
+
 - hosts: clusterNodes
   gather_facts: true
   serial: 1
   tasks:
     - ansible.builtin.include_role:
         name: configure
         tasks_from: dynamicConfigs
-      vars:
-        - zookeeperConfigFile: java.env
 
     - name: Restarting all nodes
       ansible.builtin.import_role:

clusterCustomMetricExporterSetup.yml

@@ -1,11 +1,26 @@
 ---
 
-# Command to check Cron: crontab -u root -l
-#Ansible: zookeeper metric collector
-#* * * * * python3 /zookeeper/zooki.py /zookeeper /zookeeper/zookeeper-logs/
+# Command to check Cron: crontab -u zookeeper -l
+# Ansible: zookeeper metric collector
+# * * * * * python3 /zookeeper/zooki.py /zookeeper /zookeeper/zookeeper-logs/ false
 
 - hosts: clusterNodes
-  become: false
-  gather_facts: false
+  become: true
+  gather_facts: true
+  pre_tasks:
+    - name: install common utils
+      ansible.builtin.include_role:
+        name: common
+        tasks_from: commonUtils.yml
+
+    - name: copy python requirements.txt
+      ansible.builtin.copy:
+        src: "roles/customMetricExporter/files/requirements.txt"
+        dest: "/tmp/requirements.txt"
+
+    - name: installing requirements.txt with pip3
+      ansible.builtin.pip:
+        requirements: "/tmp/requirements.txt"
+        executable: pip3
   roles:
     - customMetricExporter

clusterMigrateToMtls.yml

@@ -0,0 +1,128 @@
+---
+
+# Note:
+#   this is one-time playbook to migrate non-mtls cluster to tls cluster config
+#   Ref: https://zookeeper.apache.org/doc/r3.8.0/zookeeperAdmin.html#Upgrading+existing+nonTLS+cluster
+
+- hosts: clusterNodes
+  gather_facts: true
+  tasks:
+    - name: MigrateToMtls | upload tls keystore and truststore to all nodes
+      ansible.builtin.include_role:
+        name: copyFiles
+
+- hosts: clusterNodes
+  gather_facts: true
+  become: true
+  serial: 1
+  tasks:
+    - name: MigrateToMtls | sslQuourm basic settings in zoo.cfg
+      ansible.builtin.lineinfile:
+        path: "{{ zookeeperInstallDir }}/zookeeper-{{ zookeeperVersion }}/conf/zoo.cfg"
+        regexp: "{{ item.regex }}"
+        line: "{{ item.line }}"
+      loop:
+        - { regex: "^sslQuorum=", line: "sslQuorum=false" }
+        - { regex: "^portUnification=", line: "portUnification=true" }
+        - { regex: "^serverCnxnFactory=", line: "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory" }
+        - { regex: "^ssl.quorum.keyStore.password=", line: "ssl.quorum.keyStore.password={{ zookeeperSslQuorumKeystorePassword }}" }
+        - { regex: "^ssl.quorum.keyStore.location=", line: "ssl.quorum.keyStore.location={{ zookeeperSslQuorumKeystoreLocation }}" }
+        - { regex: "^ssl.quorum.trustStore.location=", line: "ssl.quorum.trustStore.location={{ zookeeperSslQuorumTruststoreLocation }}" }
+        - { regex: "^ssl.quorum.trustStore.password=", line: "ssl.quorum.trustStore.password={{ zookeeperSslQuorumTruststorePassword }}" }
+
+    - name: MigrateToMtls | restarting zookeeper
+      ansible.builtin.import_role:
+        name: serviceState
+      vars:
+        serviceName: zookeeper
+        serviceState: restarted
+
+    - name: MigrateToMtls | zookeeper Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperClientPort }}"
+
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: please check logs of all nodes that "Creating TLS-enabled quorum server socket" message appears in logs file
+      ansible.builtin.pause:
+        prompt: "Press enter to confirm or ctrl-c to cancel"
+
+- hosts: clusterNodes
+  gather_facts: true
+  become: true
+  serial: 1
+  tasks:
+    - name: MigrateToMtls | sslQuourm enabled in zoo.cfg
+      ansible.builtin.include_role:
+        name: configure
+        tasks_from: dynamicConfigs
+      vars:
+        - zookeeperConfigFile: zoo.cfg
+        - zookeeperSslQuorum: true
+        - zookeeperPortUnification: "true" # force true
+
+    - name: MigrateToMtls | restarting zookeeper
+      ansible.builtin.import_role:
+        name: serviceState
+      vars:
+        serviceName: zookeeper
+        serviceState: restarted
+
+    - name: MigrateToMtls | zookeeper Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperClientPort }}"
+
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: please check logs of all nodes that cluster is working
+      ansible.builtin.pause:
+        prompt: "Press enter to confirm or ctrl-c to cancel"
+
+- hosts: clusterNodes
+  gather_facts: true
+  become: true
+  serial: 1
+  tasks:
+    - name: MigrateToMtls | regenerate zoo.cfg and portUnification disabled
+      ansible.builtin.include_role:
+        name: configure
+        tasks_from: dynamicConfigs
+      vars:
+        - zookeeperConfigFile: zoo.cfg
+        - zookeeperSslQuorum: true
+        - zookeeperPortUnification: "false" # force false
+
+    - name: MigrateToMtls | restarting zookeeper
+      ansible.builtin.import_role:
+        name: serviceState
+      vars:
+        serviceName: zookeeper
+        serviceState: restarted
+
+    - name: MigrateToMtls | zookeeper Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperClientPort }}"
+
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: please check logs of all nodes that cluster is working
+      ansible.builtin.debug:
+        msg: "please set zookeeperSslQuorum=true and zookeeperPortUnification=false in ansible variables for future updates"

clusterMigrateToSasLAuth.yml

@@ -0,0 +1,103 @@
+---
+
+# Note:
+#   this is one-time playbook to migrate non-sasl cluster to sasl cluster config
+#   Ref: https://cwiki.apache.org/confluence/display/ZOOKEEPER/Server-Server+mutual+authentication
+
+- hosts: clusterNodes
+  gather_facts: true
+  become: true
+  serial: 1
+  tasks:
+    - name: MigrateToSasL | regenerate jaas.conf
+      ansible.builtin.include_role:
+        name: configure
+        tasks_from: dynamicConfigs
+      vars:
+        - zookeeperConfigFile: jaas.conf
+
+    - name: MigrateToSasL | regenerate java.env to enable jaas.conf
+      ansible.builtin.include_role:
+        name: configure
+        tasks_from: dynamicConfigs
+      vars:
+        - zookeeperConfigFile: java.env
+        - zookeeperQuorumAuthEnableSasl: true
+
+    - name: MigrateToSasL | enableSasl in zoo.cfg
+      ansible.builtin.lineinfile:
+        path: "{{ zookeeperInstallDir }}/zookeeper-{{ zookeeperVersion }}/conf/zoo.cfg"
+        regexp: "^quorum.auth.enableSasl="
+        line: "quorum.auth.enableSasl=true"
+
+    - name: MigrateToSasL | restarting zookeeper
+      ansible.builtin.import_role:
+        name: serviceState
+      vars:
+        serviceName: zookeeper
+        serviceState: restarted
+
+    - name: MigrateToSasL | zookeeper Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperClientPort }}"
+
+- hosts: clusterNodes
+  gather_facts: true
+  become: true
+  serial: 1
+  tasks:
+    - name: MigrateToSasL | learnerRequireSasl in zoo.cfg
+      ansible.builtin.lineinfile:
+        path: "{{ zookeeperInstallDir }}/zookeeper-{{ zookeeperVersion }}/conf/zoo.cfg"
+        regexp: "^quorum.auth.learnerRequireSasl="
+        line: "quorum.auth.learnerRequireSasl=true"
+
+    - name: MigrateToSasL | restarting zookeeper
+      ansible.builtin.import_role:
+        name: serviceState
+      vars:
+        serviceName: zookeeper
+        serviceState: restarted
+
+    - name: MigrateToSasL | zookeeper Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperClientPort }}"
+
+- hosts: clusterNodes
+  gather_facts: true
+  become: true
+  serial: 1
+  tasks:
+    - name: MigrateToSasL | regenerate zoo.cfg with all parameters
+      ansible.builtin.include_role:
+        name: configure
+        tasks_from: dynamicConfigs
+      vars:
+        - zookeeperConfigFile: zoo.cfg
+        - zookeeperQuorumAuthEnableSasl: true
+
+    - name: MigrateToSasL | restarting zookeeper
+      ansible.builtin.import_role:
+        name: serviceState
+      vars:
+        serviceName: zookeeper
+        serviceState: restarted
+
+    - name: MigrateToSasL | zookeeper Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperClientPort }}"

clusterSetup.yml

@@ -16,6 +16,7 @@
   roles:
     - common
     - install
+    - copyFiles
     - configure
     - service
 
@@ -39,3 +40,15 @@
         PortStatus: started
       loop:
         - "{{ zookeeperClientPort }}"
+
+    - name: zookeeper Secure Port Status
+      ansible.builtin.include_role:
+        name: portCheck
+      vars:
+        PortNumber: "{{ item }}"
+        PortStatus: started
+      loop:
+        - "{{ zookeeperSecureClientPort }}"
+      when:
+        - zookeeperSslQuorum is defined
+        - zookeeperSslQuorum