Refactor/ranger stack 3.3

roles/hbase/common/templates/hbase/install_hbase.properties.j2

@@ -161,3 +161,25 @@ CUSTOM_USER=hbase
 # CUSTOM_COMPONENT_GROUP=<custom-group>
 # keep blank if component group is default
 CUSTOM_GROUP=hadoop
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
+#Log4j Audit Provider
+XAAUDIT.LOG4J.ENABLE=true
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit

roles/hdfs/common/templates/install_hdfs.properties.j2

@@ -146,3 +146,25 @@ CUSTOM_USER=hdfs
 # CUSTOM_COMPONENT_GROUP=<custom-group>
 # keep blank if component group is default
 CUSTOM_GROUP=hadoop
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
+#Log4j Audit Provider
+XAAUDIT.LOG4J.ENABLE=true
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit

roles/hive/common/templates/install.properties.j2

@@ -158,3 +158,25 @@ CUSTOM_USER=hive
 # CUSTOM_COMPONENT_GROUP=<custom-group>
 # keep blank if component group is default
 CUSTOM_GROUP=hadoop
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
+#Log4j Audit Provider
+XAAUDIT.LOG4J.ENABLE=true
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit

roles/knox/common/templates/install.properties.j2

@@ -155,3 +155,25 @@ CUSTOM_USER=knox
 # CUSTOM_COMPONENT_GROUP=<custom-group>
 # keep blank if component group is default
 CUSTOM_GROUP=hadoop
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
+#Log4j Audit Provider
+XAAUDIT.LOG4J.ENABLE=true
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit

roles/ranger/admin/tasks/kerberos.yml

@@ -56,3 +56,11 @@
         user: "{{ ranger_user }}"
         group: "{{ hadoop_group }}"
         mode: "0600"
+
+- name: Template jaas.conf
+  ansible.builtin.template:
+    src: jaas.conf.j2
+    dest: "/etc/ranger/jaas.conf"
+    owner: root
+    group: root
+    mode: "777"

roles/ranger/common/templates/install.properties.j2

@@ -19,9 +19,9 @@
 
 #------------------------- DB CONFIG - BEGIN ----------------------------------
 # Uncomment the below if the DBA steps need to be run separately
-setup_mode=SeparateDBA
+setup_mode={{ install_properties.setup_mode }}
 
-PYTHON_COMMAND_INVOKER=python2
+PYTHON_COMMAND_INVOKER=python3
 
 #DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA
 DB_FLAVOR={{ install_properties.DB_FLAVOR }}
@@ -62,13 +62,24 @@ javax_net_ssl_keyStore=
 javax_net_ssl_keyStorePassword=
 javax_net_ssl_trustStore=
 javax_net_ssl_trustStorePassword=
+javax_net_ssl_trustStore_type=jks
+javax_net_ssl_keyStore_type=jks
+
+# For postgresql db
+# db_ssl_certificate_file=
+
 #
 # DB UserId used for the Ranger schema
 #
 db_name={{ install_properties.db_name }}
 db_user={{ install_properties.db_user }}
 db_password={{ install_properties.db_password }}
 
+#For over-riding the jdbc url.
+# is_override_db_connection_string=false
+# db_override_connection_string=
+
+
 # change password. Password for below mentioned users can be changed only once using this property.
 #PLEASE NOTE :: Password should be minimum 8 characters with min one alphabet and one numeric.
 rangerAdmin_password={{ ranger_admin_password }}
@@ -77,10 +88,20 @@ rangerUsersync_password={{ ranger_usersync_password }}
 keyadmin_password={{ ranger_keyadmin_password }}
 
 
-#Source for Audit Store. Currently only solr is supported.
+#Source for Audit Store. Currently solr, elasticsearch and cloudwatch logs are supported.
 # * audit_store is solr
 audit_store={{ install_properties.audit_store }}
 
+# * audit_solr_url Elasticsearch Host(s). E.g. 127.0.0.1
+audit_elasticsearch_urls=
+audit_elasticsearch_port=
+audit_elasticsearch_protocol=
+audit_elasticsearch_user=
+audit_elasticsearch_password=
+audit_elasticsearch_index=
+audit_elasticsearch_bootstrap_enabled=true
+
+
 # * audit_solr_url URL to Solr. E.g. http://<solr_host>:6083/solr/ranger_audits
 audit_solr_urls={{ install_properties.audit_solr_urls }}
 audit_solr_user=
@@ -90,11 +111,17 @@ audit_solr_zookeepers=
 audit_solr_collection_name=ranger_audits
 #solr Properties for cloud mode
 audit_solr_config_name=ranger_audits
+audit_solr_configset_location=
 audit_solr_no_shards=1
 audit_solr_no_replica=1
 audit_solr_max_shards_per_node=1
 audit_solr_acl_user_list_sasl=solr,infra-solr
+audit_solr_bootstrap_enabled=true
 
+# * audit to amazon cloudwatch properties
+audit_cloudwatch_region=
+audit_cloudwatch_log_group=
+audit_cloudwatch_log_stream_prefix=
 
 #------------------------- DB CONFIG - END ----------------------------------
 
@@ -216,9 +243,10 @@ sso_publickey=
 
 # Custom log directory path
 RANGER_ADMIN_LOG_DIR={{ ranger_log_dir }}
+RANGER_ADMIN_LOGBACK_CONF_FILE=
 
 # PID file path
-RANGER_PID_DIR_PATH=/var/run/ranger
+RANGER_PID_DIR_PATH={{ ranger_pid_dir }}
 
 # #################  DO NOT MODIFY ANY VARIABLES BELOW #########################
 #

roles/ranger/common/templates/jaas.conf.j2

@@ -0,0 +1,10 @@
+Client {
+  com.sun.security.auth.module.Krb5LoginModule required
+    debug=true
+    doNotPrompt=true
+    useKeyTab=true
+    keyTab="/etc/security/keytabs/rangeradmin.service.keytab"
+    principal="rangeradmin/{{ inventory_hostname }}.tdp@{{ realm }}"
+    storeKey=true
+    useTicketCache=false;
+};

roles/ranger/common/templates/kms_install.properties.j2

@@ -14,14 +14,14 @@
 # limitations under the License.
 
 #
-# This file provides a list of the deployment variables for the Ranger KMS Web Application
+# This file provides a list of the deployment variables for the Ranger KMS Web Application 
 #
 
 #------------------------- DB CONFIG - BEGIN ----------------------------------
 # Uncomment the below if the DBA steps need to be run separately
 setup_mode={{ kms_install_properties.setup_mode }}
 
-PYTHON_COMMAND_INVOKER=python2
+PYTHON_COMMAND_INVOKER=python3
 
 #DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA
 DB_FLAVOR={{ kms_install_properties.DB_FLAVOR }}
@@ -40,8 +40,8 @@ SQL_CONNECTOR_JAR={{ kms_install_properties.SQL_CONNECTOR_JAR }}
 #
 # DB password for the DB admin user-id
 # **************************************************************************
-# ** If the password is left empty or not-defined here,
-# ** it will be prompted to enter the password during installation process
+# ** If the password is left empty or not-defined here, 
+# ** it will be prompted to enter the password during installation process 
 # **************************************************************************
 #
 #db_root_user=root|SYS|postgres|sa|dba
@@ -52,6 +52,7 @@ SQL_CONNECTOR_JAR={{ kms_install_properties.SQL_CONNECTOR_JAR }}
 db_root_user=root
 db_root_password=
 db_host={{ kms_install_properties.db_host }}
+#SSL config
 db_ssl_enabled=false
 db_ssl_required=false
 db_ssl_verifyServerCertificate=false
@@ -61,20 +62,35 @@ javax_net_ssl_keyStore=
 javax_net_ssl_keyStorePassword=
 javax_net_ssl_trustStore=
 javax_net_ssl_trustStorePassword=
+javax_net_ssl_trustStore_type=jks
+javax_net_ssl_keyStore_type=jks
+
+# For postgresql db
+db_ssl_certificate_file=
+
 #
 # DB UserId used for the Ranger KMS schema
 #
 db_name={{ kms_install_properties.db_name }}
 db_user={{ kms_install_properties.db_user }}
 db_password={{ kms_install_properties.db_password }}
 
+#For over-riding the jdbc url.
+is_override_db_connection_string=false
+db_override_connection_string=
+
 #------------------------- DB CONFIG - END ----------------------------------
 #KMS Server config
 ranger_kms_http_enabled=false
 ranger_kms_https_keystore_file={{ ranger_keystore_location }}
 ranger_kms_https_keystore_keyalias={{ ansible_fqdn }}
 ranger_kms_https_keystore_password={{ ranger_keystore_password }}
 
+#------------------------- RANGER KMS Install Dir ------------------
+realScriptPath=`readlink -f $0`
+realScriptDir=`dirname $realScriptPath`
+COMPONENT_INSTALL_DIR_NAME=`(cd $realScriptDir; pwd)`
+
 #------------------------- RANGER KMS Master Key Crypt Key ------------------
 KMS_MASTER_KEY_PASSWD={{ ranger_keyadmin_password }}
 
@@ -99,7 +115,36 @@ KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn
 KEYSECURE_MASTER_KEY_SIZE=256
 KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg
 
-#
+#------------------------- Ranger Azure Key Vault ------------------------------
+AZURE_KEYVAULT_ENABLED=false
+AZURE_KEYVAULT_SSL_ENABLED=false
+AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42
+AZURE_CLIENT_SECRET=<AzureKeyVaultPassword>
+AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx
+# Initialize below prop if your certificate file has any password
+#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass
+AZURE_MASTERKEY_NAME=RangerMasterKey
+# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT
+AZURE_MASTER_KEY_TYPE=RSA
+# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP 
+ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP
+AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/
+
+#------------------------- Ranger Google Cloud HSM ------------------------------
+IS_GCP_ENABLED=false
+GCP_KEYRING_ID=
+GCP_CRED_JSON_FILE=/full/path/to/credfile.json
+GCP_PROJECT_ID=
+GCP_LOCATION_ID=
+GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt
+
+#------------------------- Ranger Tencent KMS ------------------------------
+TENCENT_KMS_ENABLED=false
+TENCENT_MASTERKEY_ID=
+TENCENT_CLIENT_ID=
+TENCENT_CLIENT_SECRET=
+TENCENT_CLIENT_REGION=
+
 # ------- UNIX User CONFIG ----------------
 #
 unix_user={{ ranger_kms_user }}
@@ -147,6 +192,20 @@ XAAUDIT.SOLR.PASSWORD=NONE
 XAAUDIT.SOLR.ZOOKEEPER=NONE
 XAAUDIT.SOLR.FILE_SPOOL_DIR={{ ranger_log_dir }}/kms/audit/solr/spool
 
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
 # Enable audit logs to HDFS
 #Example
 #XAAUDIT.HDFS.ENABLE=true
@@ -168,6 +227,27 @@ XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
 XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
 XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
 
+#Log4j Audit Provider
+XAAUDIT.LOG4J.ENABLE=true
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+# Enable audit logs to Amazon CloudWatch Logs
+#Example
+#XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=true
+#XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=ranger_audits
+#XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM={instance_id}
+#XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=/var/log/hive/audit/amazon_cloudwatch/spool
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
 # End of V3 properties
 
 
@@ -202,7 +282,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
-#Solr Audit Provder
+#Solr Audit Provider
 XAAUDIT.SOLR.IS_ENABLED=false
 XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000