spark2|3: add ha support for spark-hs

playbooks/kerberos_spnego_install.yml

@@ -0,0 +1,14 @@
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Kerberos spnego install
+  hosts: hdfs_nn:hdfs_jn:hdfs_dn:hdfs_httpfs:yarn_rm:yarn_nm:yarn_ats:mapred_jhs:hive_s2:hbase_rest:phoenix_queryserver_daemon:ranger_admin:ranger_kms:spark_hs:spark3_hs
+  tasks:
+    - tosit.tdp.resolve: # noqa unnamed-task
+        node_name: kerberos_spnego
+    - name: Install Kerberos spnego
+      ansible.builtin.import_role:
+        name: tosit.tdp.kerberos.spnego
+        tasks_from: install
+    - ansible.builtin.meta: clear_facts # noqa unnamed-task

playbooks/ranger_kerberos_install.yml

@@ -13,17 +13,6 @@
         name: tosit.tdp.ranger.admin
         tasks_from: kerberos
     - ansible.builtin.meta: clear_facts # noqa unnamed-task
-- name: Kerberos Ranger Admin HA install
-  hosts: spnego_ha
-  strategy: linear
-  tasks:
-    - tosit.tdp.resolve: # noqa unnamed-task
-        node_name: ranger_kerberos
-    - name: Install Ranger Admin Kerberos
-      ansible.builtin.import_role:
-        name: tosit.tdp.ranger.admin
-        tasks_from: kerberos-spnego-ha
-    - ansible.builtin.meta: clear_facts # noqa unnamed-task
 - name: Kerberos Ranger UserSync install
   hosts: ranger_usersync
   strategy: linear

playbooks/spark3_kerberos_install.yml

@@ -13,7 +13,6 @@
         name: tosit.tdp.spark.historyserver
         tasks_from: kerberos
     - ansible.builtin.meta: clear_facts # noqa unnamed-task
-
 - name: Spark3 Kerberos Client install
   hosts: spark3_client
   strategy: linear

plugins/module_utils/kerberos.py

@@ -1,6 +1,9 @@
 # Copyright 2022 TOSIT.IO
 # SPDX-License-Identifier: Apache-2.0
 
+import os
+import tempfile
+import shutil
 
 kerberos_spec = dict(
     kerberos=dict(type='bool', default=False),
@@ -42,6 +45,26 @@ def kinit(module):
     module.run_command(kinit_cmd, check_rc=True)
 
 
+def try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path):
+    """Try kinit, return True if success, False or exception otherwise"""
+    # Create a tmp dir to store the krb cache in order to not override
+    # an existing cache in default location
+    for principal in principals:
+        tmp_dir = tempfile.mkdtemp(suffix='_ansible_module_utils_kerberos')
+        try:
+            ccache = os.path.join(tmp_dir, "krb5cc")
+            kinit_cmd = get_kinit_cmd(kinit_bin, principal, keytab_path, ccache)
+            rc, stdout, stderr = module.run_command(kinit_cmd)
+            if rc == 0:
+                kdestroy_cmd = get_kdestroy_cmd(kdestroy_bin, ccache)
+                module.run_command(kdestroy_cmd)
+            else:
+                return False
+        finally:
+            shutil.rmtree(tmp_dir)
+    return True
+
+
 def get_kdestroy_cmd(kdestroy_bin, ccache=None):
     kdestroy_cmd = [kdestroy_bin]
     if ccache:

plugins/modules/krb_check_keytab.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+# Copyright 2022 TOSIT.IO
+# SPDX-License-Identifier: Apache-2.0
+
+# -*- coding: utf-8 -*-
+
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils._text import to_native
+from ansible_collections.tosit.tdp.plugins.module_utils.kerberos import try_kinit
+
+def main():
+    argument_spec = dict(
+        kinit_bin=dict(type='path', default='kinit'),
+        kdestroy_bin=dict(type='path', default='kdestroy'),
+        principal=dict(type='list', elements='str', required=True),
+        path=dict(type='path', required=True),
+    )
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        add_file_common_args=True,
+        supports_check_mode=True,
+    )
+
+    kinit_bin = module.params['kinit_bin']
+    kdestroy_bin = module.params['kdestroy_bin']
+    principals = module.params['principal']
+    keytab_path = module.params['path']
+
+    try:
+        results = {
+            'changed': False,
+        }
+
+        if not try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path):
+            raise RuntimeError("Keytab '{}' with principal '{}' is not working".format(keytab_path, principals))
+
+        module.exit_json(**results)
+
+    except Exception:
+        import traceback
+        module.fail_json(msg=to_native(traceback.format_exc()))
+
+if __name__ == '__main__':
+    main()

plugins/modules/krb_keytab.py

@@ -9,36 +9,16 @@
 __metaclass__ = type
 
 import os
-import tempfile
-import shutil
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
-from ansible_collections.tosit.tdp.plugins.module_utils.kerberos import get_kinit_cmd, get_kdestroy_cmd
+from ansible_collections.tosit.tdp.plugins.module_utils.kerberos import try_kinit
 from ansible_collections.tosit.tdp.plugins.module_utils.kerberos_admin import kerberos_admin_spec, kadmin
 
-def try_kinit(module, kinit_bin, kdestroy_bin, principal, keytab_path):
-    """Try kinit, return True if success, False or exception otherwise"""
-    # Create a tmp dir to store the krb cache in order to not override
-    # an existing cache in default location
-    tmp_dir = tempfile.mkdtemp(suffix='_ansible_module_krb_keytab')
-    try:
-        ccache = os.path.join(tmp_dir, "krb5cc")
-        kinit_cmd = get_kinit_cmd(kinit_bin, principal, keytab_path, ccache)
-        rc, stdout, stderr = module.run_command(kinit_cmd)
-        if rc == 0:
-            kdestroy_cmd = get_kdestroy_cmd(kdestroy_bin, ccache)
-            module.run_command(kdestroy_cmd)
-            return True
-        else:
-            return False
-    finally:
-        shutil.rmtree(tmp_dir)
-
 def main():
     argument_spec = dict(
         kinit_bin=dict(type='path', default='kinit'),
         kdestroy_bin=dict(type='path', default='kdestroy'),
-        principal=dict(type='str', required=True),
+        principal=dict(type='list', elements='str', required=True),
         path=dict(type='path', required=True),
         state=dict(type='str', choices=['present', 'absent'], default='present'),
         **kerberos_admin_spec
@@ -52,7 +32,7 @@ def main():
 
     kinit_bin = module.params['kinit_bin']
     kdestroy_bin = module.params['kdestroy_bin']
-    principal = module.params['principal']
+    principals = module.params['principal']
     keytab_path = module.params['path']
     state = module.params['state']
 
@@ -83,7 +63,7 @@ def main():
                 return module.exit_json(**results)
 
             # Case when keytab exists, try kinit to verify if the keytab is working.
-            if try_kinit(module, kinit_bin, kdestroy_bin, principal, keytab_path):
+            if try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path):
                 # Update file permissions for existing keytab if needed
                 file_args = module.load_file_common_arguments(module.params)
                 results['changed'] = module.set_fs_attributes_if_different(
@@ -95,16 +75,17 @@ def main():
         # Either the keytab does not exist or it is not valid, so it must be generated
         results['changed'] = True
         if not module.check_mode:
-            rc, stdout, stderr = kadmin(module, ['-q', 'ktadd -k {} {}'.format(keytab_path, principal)])
+            principal_args = ' '.join(principals)
+            rc, stdout, stderr = kadmin(module, ['-q', 'ktadd -k {} {}'.format(keytab_path, principal_args)])
             # rc is 0 when the principal does not exist...
             if 'Principal' in stderr and 'does not exist' in stderr:
-                raise RuntimeError("Failed to generate keytab for principal '{}': {}".format(principal, stderr))
+                raise RuntimeError("Failed to generate keytab for principal '{}': {}".format(principals, stderr))
             # Keytab generated is not guarantee to works so it must be verified,
             # for example, deleting a principal without deleting the keytab, then create the
             # same principal will reset the kvno, generate keytab in the same keytab file,
             # the keytab file will have the old kvno which is greater than the new kvno
-            if not try_kinit(module, kinit_bin, kdestroy_bin, principal, keytab_path):
-                raise RuntimeError("Keytab '{}' generated for principal '{}' is not working".format(keytab_path, principal))
+            if not try_kinit(module, kinit_bin, kdestroy_bin, principals, keytab_path):
+                raise RuntimeError("Keytab '{}' generated for principal '{}' is not working".format(keytab_path, principals))
 
         file_args = module.load_file_common_arguments(module.params)
 

plugins/modules/krb_principal.py

@@ -14,7 +14,7 @@
 
 def main():
     argument_spec = dict(
-        principal=dict(type='str', required=True),
+        principal=dict(type='list', elements='str', required=True),
         state=dict(type='str', choices=['present', 'absent'], default='present'),
         **kerberos_admin_spec
     )
@@ -24,34 +24,39 @@ def main():
         supports_check_mode=True,
     )
 
-    principal = module.params['principal']
+    principals = module.params['principal']
     state = module.params['state']
 
     try:
         results = {
             'changed': False,
+            'created': [],
+            'deleted': [],
         }
         current_state = None
 
-        rc, stdout, stderr = kadmin(module, ['-q', 'getprinc {}'.format(principal)])
-        if 'Principal does not exist' in stderr:
-            current_state = 'absent'
-        else:
-            current_state = 'present'
-
-        # Case when principal does not exist
-        if current_state == 'absent':
-            if state == 'absent':
-                return module.exit_json(**results)
-            results['changed'] = True
-            if not module.check_mode:
-                kadmin(module, ['-q', 'addprinc -randkey {}'.format(principal)])
-
-        # Case when principal exists and must be remove
-        if current_state == 'present' and state == 'absent':
-            results['changed'] = True
-            if not module.check_mode:
-                kadmin(module, ['-q', 'delprinc -force {}'.format(principal)])
+        for principal in principals:
+            rc, stdout, stderr = kadmin(module, ['-q', 'getprinc {}'.format(principal)])
+            if 'Principal does not exist' in stderr:
+                current_state = 'absent'
+            else:
+                current_state = 'present'
+
+            # Case when principal does not exist
+            if current_state == 'absent':
+                if state == 'absent':
+                    return module.exit_json(**results)
+                results['changed'] = True
+                results['created'].append(principal)
+                if not module.check_mode:
+                    kadmin(module, ['-q', 'addprinc -randkey {}'.format(principal)])
+
+            # Case when principal exists and must be remove
+            if current_state == 'present' and state == 'absent':
+                results['changed'] = True
+                results['deleted'].append(principal)
+                if not module.check_mode:
+                    kadmin(module, ['-q', 'delprinc -force {}'.format(principal)])
 
         module.exit_json(**results)
 

roles/hbase/phoenix/queryserver/daemon/tasks/kerberos.yml

@@ -7,51 +7,26 @@
     name: tosit.tdp.utils.kerberos
     tasks_from: install
 
-- when: krb_create_principals_keytabs
-  block:
-    - name: Ensure phoenix queryserver principal and keytab exist
-      ansible.builtin.import_role:
-        name: tosit.tdp.utils.kerberos
-        tasks_from: create_principal_keytab
-      vars:
-        principal: phoenixqueryserver/{{ ansible_fqdn }}
-        keytab: phoenixqueryserver.service.keytab
-        user: "{{ phoenix_queryserver_user }}"
-        group: "{{ hadoop_group }}"
-        mode: "0600"
-
-    - name: Ensure HTTP spnego principal and keytab exist
-      ansible.builtin.import_role:
-        name: tosit.tdp.utils.kerberos
-        tasks_from: create_principal_keytab
-      vars:
-        principal: HTTP/{{ ansible_fqdn }}
-        keytab: spnego.service.keytab
-        user: root
-        group: "{{ hadoop_group }}"
-        mode: "0640"
+- name: Ensure phoenix queryserver principal and keytab exist
+  ansible.builtin.import_role:
+    name: tosit.tdp.utils.kerberos
+    tasks_from: create_principal_keytab
+  vars:
+    principal: phoenixqueryserver/{{ ansible_fqdn }}
+    keytab: phoenixqueryserver.service.keytab
+    user: "{{ phoenix_queryserver_user }}"
+    group: "{{ hadoop_group }}"
+    mode: "0600"
+  when: krb_create_principals_keytabs
 
-- name: Phoenix QueryServer keytabs check
+- name: Ensure phoenix queryserver keytab works
+  ansible.builtin.import_role:
+    name: tosit.tdp.utils.kerberos
+    tasks_from: check_secure_keytab
+  vars:
+    principal: "phoenixqueryserver/{{ ansible_fqdn }}"
+    keytab: "phoenixqueryserver.service.keytab"
+    user: "{{ phoenix_queryserver_user }}"
+    group: "{{ hadoop_group }}"
+    mode: "0600"
   when: not krb_create_principals_keytabs
-  block:
-    - name: Ensure phoenix queryserver keytab works
-      ansible.builtin.import_role:
-        name: tosit.tdp.utils.kerberos
-        tasks_from: check_secure_keytab
-      vars:
-        principal: "phoenixqueryserver/{{ ansible_fqdn }}"
-        keytab: "phoenixqueryserver.service.keytab"
-        user: "{{ phoenix_queryserver_user }}"
-        group: "{{ hadoop_group }}"
-        mode: "0600"
-
-    - name: Ensure HTTP spnego keytab works
-      ansible.builtin.import_role:
-        name: tosit.tdp.utils.kerberos
-        tasks_from: check_secure_keytab
-      vars:
-        principal: HTTP/{{ ansible_fqdn }}
-        keytab: spnego.service.keytab
-        user: root
-        group: "{{ hadoop_group }}"
-        mode: "0640"