From 8e95361b377f558b9a2005c0fb25085c2c2d8287 Mon Sep 17 00:00:00 2001 From: SteBaum Date: Fri, 20 Dec 2024 18:54:18 +0100 Subject: [PATCH 1/4] feat(ranger): to ranger version 2.5.0-0.0 and solr version 8.11.3 --- tdp_vars_defaults/hbase/hbase.yml | 2 +- tdp_vars_defaults/hdfs/hdfs.yml | 2 +- tdp_vars_defaults/hive/hive.yml | 2 +- tdp_vars_defaults/knox/knox.yml | 2 +- tdp_vars_defaults/ranger/ranger.yml | 8 ++++---- tdp_vars_defaults/yarn/yarn.yml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tdp_vars_defaults/hbase/hbase.yml b/tdp_vars_defaults/hbase/hbase.yml index da561263..5a3bded6 100644 --- a/tdp_vars_defaults/hbase/hbase.yml +++ b/tdp_vars_defaults/hbase/hbase.yml @@ -125,7 +125,7 @@ hbase_regionserver_kerberos_principal: "hbase/{{ ansible_fqdn }}@{{ realm }}" hbase_rest_kerberos_principal: "hbase/{{ ansible_fqdn }}@{{ realm }}" # HBase Ranger Plugin -ranger_hbase_release: ranger-2.0.0-1.0-hbase-plugin +ranger_hbase_release: ranger-2.5.0-0.0-hbase-plugin ranger_hbase_dist_file: "{{ ranger_hbase_release }}.tar.gz" ranger_hbase_install_dir: "{{ hbase_root_dir }}/ranger-hbase-plugin" ranger_hbase_install_properties: diff --git a/tdp_vars_defaults/hdfs/hdfs.yml b/tdp_vars_defaults/hdfs/hdfs.yml index 82a51c8c..387d11e0 100644 --- a/tdp_vars_defaults/hdfs/hdfs.yml +++ b/tdp_vars_defaults/hdfs/hdfs.yml @@ -59,7 +59,7 @@ hdfs_site: namenode_kerberos_principal: "nn/{{ ansible_fqdn }}@{{ realm }}" # Ranger HDFS properties -ranger_hdfs_release: ranger-2.0.0-1.0-hdfs-plugin +ranger_hdfs_release: ranger-2.5.0-0.0-hdfs-plugin ranger_hdfs_dist_file: "{{ ranger_hdfs_release }}.tar.gz" ranger_hdfs_install_dir: "{{ hadoop_root_dir }}/ranger-hdfs-plugin" ranger_hdfs_install_properties: diff --git a/tdp_vars_defaults/hive/hive.yml b/tdp_vars_defaults/hive/hive.yml index d91a7eed..492bd3e2 100644 --- a/tdp_vars_defaults/hive/hive.yml +++ b/tdp_vars_defaults/hive/hive.yml @@ -141,7 +141,7 @@ beeline_site: beeline.hs2.jdbc.url.default: zk_cluster # Ranger Hive properties -ranger_hive_release: ranger-2.0.0-1.0-hive-plugin +ranger_hive_release: ranger-2.5.0-0.0-hive-plugin ranger_hive_dist_file: "{{ ranger_hive_release }}.tar.gz" ranger_hive_install_dir: "{{ hive_root_dir }}/ranger-hive-plugin" ranger_hive_install_properties: diff --git a/tdp_vars_defaults/knox/knox.yml b/tdp_vars_defaults/knox/knox.yml index 290fd7b0..3e46f56e 100644 --- a/tdp_vars_defaults/knox/knox.yml +++ b/tdp_vars_defaults/knox/knox.yml @@ -192,7 +192,7 @@ knox_start_on_boot: no knox_restart: "no" # Ranger Knox properties -ranger_knox_release: ranger-2.0.0-1.0-knox-plugin +ranger_knox_release: ranger-2.5.0-0.0-knox-plugin ranger_knox_dist_file: "{{ ranger_knox_release }}.tar.gz" ranger_knox_install_dir: "{{ knox_root_dir }}/ranger-knox-plugin" ranger_knox_install_properties: diff --git a/tdp_vars_defaults/ranger/ranger.yml b/tdp_vars_defaults/ranger/ranger.yml index f32141a7..cec94e65 100644 --- a/tdp_vars_defaults/ranger/ranger.yml +++ b/tdp_vars_defaults/ranger/ranger.yml @@ -3,11 +3,11 @@ --- # Ranger version -ranger_release: ranger-2.0.0-1.0-admin +ranger_release: ranger-2.5.0-0.0-admin ranger_dist_file: "{{ ranger_release }}.tar.gz" -ranger_usersync_release: ranger-2.0.0-1.0-usersync +ranger_usersync_release: ranger-2.5.0-0.0-usersync ranger_usersync_dist_file: "{{ ranger_usersync_release }}.tar.gz" -ranger_kms_release: ranger-2.0.0-1.0-kms +ranger_kms_release: ranger-2.5.0-0.0-kms ranger_kms_dist_file: "{{ ranger_kms_release }}.tar.gz" # Ranger users and group @@ -124,7 +124,7 @@ ranger_usync_restart: "no" ranger_kms_restart: "no" # Solr version -solr_release: solr-7.7.3 +solr_release: solr-8.11.3 solr_dist_file: "{{ solr_release }}.tgz" # Solr users and group diff --git a/tdp_vars_defaults/yarn/yarn.yml b/tdp_vars_defaults/yarn/yarn.yml index 3663a442..6d1007fc 100644 --- a/tdp_vars_defaults/yarn/yarn.yml +++ b/tdp_vars_defaults/yarn/yarn.yml @@ -147,7 +147,7 @@ container_executor: # Ranger YARN properties -ranger_yarn_release: ranger-2.0.0-1.0-yarn-plugin +ranger_yarn_release: ranger-2.5.0-0.0-yarn-plugin ranger_yarn_dist_file: "{{ ranger_yarn_release }}.tar.gz" ranger_yarn_install_dir: "{{ hadoop_root_dir }}/ranger-yarn-plugin" ranger_yarn_install_properties: From 5593446f4378ecf5420e61ce4f67e568c3c4559b Mon Sep 17 00:00:00 2001 From: ElNeoX-dev <61790793+ElNeoX-dev@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:40:25 +0100 Subject: [PATCH 2/4] fix(ranger): add jaas to ranger --- roles/ranger/admin/tasks/kerberos.yml | 8 ++++++++ roles/ranger/common/templates/jaas.conf.j2 | 10 ++++++++++ 2 files changed, 18 insertions(+) create mode 100644 roles/ranger/common/templates/jaas.conf.j2 diff --git a/roles/ranger/admin/tasks/kerberos.yml b/roles/ranger/admin/tasks/kerberos.yml index e253a57f..88ba02bc 100644 --- a/roles/ranger/admin/tasks/kerberos.yml +++ b/roles/ranger/admin/tasks/kerberos.yml @@ -56,3 +56,11 @@ user: "{{ ranger_user }}" group: "{{ hadoop_group }}" mode: "0600" + +- name: Template jaas.conf + ansible.builtin.template: + src: jaas.conf.j2 + dest: "/etc/ranger/jaas.conf" + owner: root + group: root + mode: "777" diff --git a/roles/ranger/common/templates/jaas.conf.j2 b/roles/ranger/common/templates/jaas.conf.j2 new file mode 100644 index 00000000..bfa6dd0d --- /dev/null +++ b/roles/ranger/common/templates/jaas.conf.j2 @@ -0,0 +1,10 @@ +Client { + com.sun.security.auth.module.Krb5LoginModule required + debug=true + doNotPrompt=true + useKeyTab=true + keyTab="/etc/security/keytabs/rangeradmin.service.keytab" + principal="rangeradmin/{{ inventory_hostname }}.tdp@{{ realm }}" + storeKey=true + useTicketCache=false; +}; \ No newline at end of file From 32071c86429cfbf9c4f9639646a9be4beabbc974 Mon Sep 17 00:00:00 2001 From: SteBaum Date: Fri, 20 Dec 2024 19:03:45 +0100 Subject: [PATCH 3/4] fix(ranger): added properties --- .../hbase/install_hbase.properties.j2 | 22 ++++++++++++ .../templates/install_hdfs.properties.j2 | 22 ++++++++++++ .../common/templates/install.properties.j2 | 22 ++++++++++++ .../common/templates/install.properties.j2 | 22 ++++++++++++ .../common/templates/install.properties.j2 | 36 ++++++++++++++++--- .../templates/install_yarn.properties.j2 | 22 ++++++++++++ 6 files changed, 142 insertions(+), 4 deletions(-) diff --git a/roles/hbase/common/templates/hbase/install_hbase.properties.j2 b/roles/hbase/common/templates/hbase/install_hbase.properties.j2 index 61fc3220..b388e045 100644 --- a/roles/hbase/common/templates/hbase/install_hbase.properties.j2 +++ b/roles/hbase/common/templates/hbase/install_hbase.properties.j2 @@ -161,3 +161,25 @@ CUSTOM_USER=hbase # CUSTOM_COMPONENT_GROUP= # keep blank if component group is default CUSTOM_GROUP=hadoop + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +#Log4j Audit Provider +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit \ No newline at end of file diff --git a/roles/hdfs/common/templates/install_hdfs.properties.j2 b/roles/hdfs/common/templates/install_hdfs.properties.j2 index 75f7ff56..c45e3419 100644 --- a/roles/hdfs/common/templates/install_hdfs.properties.j2 +++ b/roles/hdfs/common/templates/install_hdfs.properties.j2 @@ -146,3 +146,25 @@ CUSTOM_USER=hdfs # CUSTOM_COMPONENT_GROUP= # keep blank if component group is default CUSTOM_GROUP=hadoop + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +#Log4j Audit Provider +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit diff --git a/roles/hive/common/templates/install.properties.j2 b/roles/hive/common/templates/install.properties.j2 index e7669997..59c2b262 100755 --- a/roles/hive/common/templates/install.properties.j2 +++ b/roles/hive/common/templates/install.properties.j2 @@ -158,3 +158,25 @@ CUSTOM_USER=hive # CUSTOM_COMPONENT_GROUP= # keep blank if component group is default CUSTOM_GROUP=hadoop + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +#Log4j Audit Provider +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit diff --git a/roles/knox/common/templates/install.properties.j2 b/roles/knox/common/templates/install.properties.j2 index 47509f7f..178100ff 100755 --- a/roles/knox/common/templates/install.properties.j2 +++ b/roles/knox/common/templates/install.properties.j2 @@ -155,3 +155,25 @@ CUSTOM_USER=knox # CUSTOM_COMPONENT_GROUP= # keep blank if component group is default CUSTOM_GROUP=hadoop + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +#Log4j Audit Provider +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit diff --git a/roles/ranger/common/templates/install.properties.j2 b/roles/ranger/common/templates/install.properties.j2 index 0b7f38e2..1329c6a6 100644 --- a/roles/ranger/common/templates/install.properties.j2 +++ b/roles/ranger/common/templates/install.properties.j2 @@ -19,9 +19,9 @@ #------------------------- DB CONFIG - BEGIN ---------------------------------- # Uncomment the below if the DBA steps need to be run separately -setup_mode=SeparateDBA +setup_mode={{ install_properties.setup_mode }} -PYTHON_COMMAND_INVOKER=python2 +PYTHON_COMMAND_INVOKER=python3 #DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA DB_FLAVOR={{ install_properties.DB_FLAVOR }} @@ -62,6 +62,12 @@ javax_net_ssl_keyStore= javax_net_ssl_keyStorePassword= javax_net_ssl_trustStore= javax_net_ssl_trustStorePassword= +javax_net_ssl_trustStore_type=jks +javax_net_ssl_keyStore_type=jks + +# For postgresql db +# db_ssl_certificate_file= + # # DB UserId used for the Ranger schema # @@ -69,6 +75,11 @@ db_name={{ install_properties.db_name }} db_user={{ install_properties.db_user }} db_password={{ install_properties.db_password }} +#For over-riding the jdbc url. +# is_override_db_connection_string=false +# db_override_connection_string= + + # change password. Password for below mentioned users can be changed only once using this property. #PLEASE NOTE :: Password should be minimum 8 characters with min one alphabet and one numeric. rangerAdmin_password={{ ranger_admin_password }} @@ -77,10 +88,20 @@ rangerUsersync_password={{ ranger_usersync_password }} keyadmin_password={{ ranger_keyadmin_password }} -#Source for Audit Store. Currently only solr is supported. +#Source for Audit Store. Currently solr, elasticsearch and cloudwatch logs are supported. # * audit_store is solr audit_store={{ install_properties.audit_store }} +# * audit_solr_url Elasticsearch Host(s). E.g. 127.0.0.1 +audit_elasticsearch_urls= +audit_elasticsearch_port= +audit_elasticsearch_protocol= +audit_elasticsearch_user= +audit_elasticsearch_password= +audit_elasticsearch_index= +audit_elasticsearch_bootstrap_enabled=true + + # * audit_solr_url URL to Solr. E.g. http://:6083/solr/ranger_audits audit_solr_urls={{ install_properties.audit_solr_urls }} audit_solr_user= @@ -90,11 +111,17 @@ audit_solr_zookeepers= audit_solr_collection_name=ranger_audits #solr Properties for cloud mode audit_solr_config_name=ranger_audits +audit_solr_configset_location= audit_solr_no_shards=1 audit_solr_no_replica=1 audit_solr_max_shards_per_node=1 audit_solr_acl_user_list_sasl=solr,infra-solr +audit_solr_bootstrap_enabled=true +# * audit to amazon cloudwatch properties +audit_cloudwatch_region= +audit_cloudwatch_log_group= +audit_cloudwatch_log_stream_prefix= #------------------------- DB CONFIG - END ---------------------------------- @@ -216,9 +243,10 @@ sso_publickey= # Custom log directory path RANGER_ADMIN_LOG_DIR={{ ranger_log_dir }} +RANGER_ADMIN_LOGBACK_CONF_FILE= # PID file path -RANGER_PID_DIR_PATH=/var/run/ranger +RANGER_PID_DIR_PATH={{ ranger_pid_dir }} # ################# DO NOT MODIFY ANY VARIABLES BELOW ######################### # diff --git a/roles/yarn/common/templates/install_yarn.properties.j2 b/roles/yarn/common/templates/install_yarn.properties.j2 index df7aed31..af6e523b 100644 --- a/roles/yarn/common/templates/install_yarn.properties.j2 +++ b/roles/yarn/common/templates/install_yarn.properties.j2 @@ -146,3 +146,25 @@ CUSTOM_USER=yarn # CUSTOM_COMPONENT_GROUP= # keep blank if component group is default CUSTOM_GROUP=hadoop + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +#Log4j Audit Provider +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit \ No newline at end of file From 9a9216e7bb8e1dd0265a5909229b51e147e269ba Mon Sep 17 00:00:00 2001 From: SteBaum Date: Thu, 9 Jan 2025 17:36:29 +0100 Subject: [PATCH 4/4] feat(kms): modifications for ranger kms --- .../templates/kms_install.properties.j2 | 92 +++++++++++++++++-- roles/ranger/common/templates/ranger-kms.j2 | 25 +++-- tdp_vars_defaults/ranger/ranger.yml | 2 +- 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/roles/ranger/common/templates/kms_install.properties.j2 b/roles/ranger/common/templates/kms_install.properties.j2 index c419af8a..962cafb7 100644 --- a/roles/ranger/common/templates/kms_install.properties.j2 +++ b/roles/ranger/common/templates/kms_install.properties.j2 @@ -14,14 +14,14 @@ # limitations under the License. # -# This file provides a list of the deployment variables for the Ranger KMS Web Application +# This file provides a list of the deployment variables for the Ranger KMS Web Application # #------------------------- DB CONFIG - BEGIN ---------------------------------- # Uncomment the below if the DBA steps need to be run separately setup_mode={{ kms_install_properties.setup_mode }} -PYTHON_COMMAND_INVOKER=python2 +PYTHON_COMMAND_INVOKER=python3 #DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA DB_FLAVOR={{ kms_install_properties.DB_FLAVOR }} @@ -40,8 +40,8 @@ SQL_CONNECTOR_JAR={{ kms_install_properties.SQL_CONNECTOR_JAR }} # # DB password for the DB admin user-id # ************************************************************************** -# ** If the password is left empty or not-defined here, -# ** it will be prompted to enter the password during installation process +# ** If the password is left empty or not-defined here, +# ** it will be prompted to enter the password during installation process # ************************************************************************** # #db_root_user=root|SYS|postgres|sa|dba @@ -52,6 +52,7 @@ SQL_CONNECTOR_JAR={{ kms_install_properties.SQL_CONNECTOR_JAR }} db_root_user=root db_root_password= db_host={{ kms_install_properties.db_host }} +#SSL config db_ssl_enabled=false db_ssl_required=false db_ssl_verifyServerCertificate=false @@ -61,6 +62,12 @@ javax_net_ssl_keyStore= javax_net_ssl_keyStorePassword= javax_net_ssl_trustStore= javax_net_ssl_trustStorePassword= +javax_net_ssl_trustStore_type=jks +javax_net_ssl_keyStore_type=jks + +# For postgresql db +db_ssl_certificate_file= + # # DB UserId used for the Ranger KMS schema # @@ -68,6 +75,10 @@ db_name={{ kms_install_properties.db_name }} db_user={{ kms_install_properties.db_user }} db_password={{ kms_install_properties.db_password }} +#For over-riding the jdbc url. +is_override_db_connection_string=false +db_override_connection_string= + #------------------------- DB CONFIG - END ---------------------------------- #KMS Server config ranger_kms_http_enabled=false @@ -75,6 +86,11 @@ ranger_kms_https_keystore_file={{ ranger_keystore_location }} ranger_kms_https_keystore_keyalias={{ ansible_fqdn }} ranger_kms_https_keystore_password={{ ranger_keystore_password }} +#------------------------- RANGER KMS Install Dir ------------------ +realScriptPath=`readlink -f $0` +realScriptDir=`dirname $realScriptPath` +COMPONENT_INSTALL_DIR_NAME=`(cd $realScriptDir; pwd)` + #------------------------- RANGER KMS Master Key Crypt Key ------------------ KMS_MASTER_KEY_PASSWD={{ ranger_keyadmin_password }} @@ -99,7 +115,36 @@ KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn KEYSECURE_MASTER_KEY_SIZE=256 KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg -# +#------------------------- Ranger Azure Key Vault ------------------------------ +AZURE_KEYVAULT_ENABLED=false +AZURE_KEYVAULT_SSL_ENABLED=false +AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42 +AZURE_CLIENT_SECRET= +AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx +# Initialize below prop if your certificate file has any password +#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass +AZURE_MASTERKEY_NAME=RangerMasterKey +# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT +AZURE_MASTER_KEY_TYPE=RSA +# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP +ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP +AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/ + +#------------------------- Ranger Google Cloud HSM ------------------------------ +IS_GCP_ENABLED=false +GCP_KEYRING_ID= +GCP_CRED_JSON_FILE=/full/path/to/credfile.json +GCP_PROJECT_ID= +GCP_LOCATION_ID= +GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt + +#------------------------- Ranger Tencent KMS ------------------------------ +TENCENT_KMS_ENABLED=false +TENCENT_MASTERKEY_ID= +TENCENT_CLIENT_ID= +TENCENT_CLIENT_SECRET= +TENCENT_CLIENT_REGION= + # ------- UNIX User CONFIG ---------------- # unix_user={{ ranger_kms_user }} @@ -147,6 +192,20 @@ XAAUDIT.SOLR.PASSWORD=NONE XAAUDIT.SOLR.ZOOKEEPER=NONE XAAUDIT.SOLR.FILE_SPOOL_DIR={{ ranger_log_dir }}/kms/audit/solr/spool +# Enable audit logs to ElasticSearch +#Example +#XAAUDIT.ELASTICSEARCH.ENABLE=true +#XAAUDIT.ELASTICSEARCH.URL=localhost +#XAAUDIT.ELASTICSEARCH.INDEX=audit + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + # Enable audit logs to HDFS #Example #XAAUDIT.HDFS.ENABLE=true @@ -168,6 +227,27 @@ XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER +#Log4j Audit Provider +XAAUDIT.LOG4J.ENABLE=true +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=true +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit + +# Enable audit logs to Amazon CloudWatch Logs +#Example +#XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=true +#XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=ranger_audits +#XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM={instance_id} +#XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=/var/log/hive/audit/amazon_cloudwatch/spool + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + # End of V3 properties @@ -202,7 +282,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 -#Solr Audit Provder +#Solr Audit Provider XAAUDIT.SOLR.IS_ENABLED=false XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 diff --git a/roles/ranger/common/templates/ranger-kms.j2 b/roles/ranger/common/templates/ranger-kms.j2 index 705cb821..49c8af72 100755 --- a/roles/ranger/common/templates/ranger-kms.j2 +++ b/roles/ranger/common/templates/ranger-kms.j2 @@ -25,15 +25,20 @@ fi action=$1 arg2=$2 arg3=$3 + +if [ -z "${arg3}" ] +then + arg3="hsmenabled" +fi + action=`echo $action | tr '[:lower:]' '[:upper:]'` realScriptPath=`readlink -f $0` realScriptDir=`dirname $realScriptPath` RANGER_KMS_DIR=`(cd $realScriptDir; pwd)` RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews RANGER_KMS_EWS_CONF_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf" -RANGER_KMS_EWS_LIB_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/lib" -ranger_kms_max_heap_size={{ ranger_kms_heapsize }} +ranger_kms_max_heap_size=1g if [ -f ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then . ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh @@ -45,9 +50,7 @@ for custom_env_script in `find ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf fi done -JMX_OPTS=" {{ jmx_common_opts }} {{ jmx_exporter_kms_opts }} " -RANGER_LOGS_OPTS="-Dlog4j.configuration=file:{{ ranger_kms_conf_dir }}/conf/log4j.properties -Dranger.log.dir={{ ranger_log_dir }} -Dranger.log.file={{ ranger_kms_log_file }} -Dranger.root.logger={{ ranger_root_logger_level }},{{ ranger_root_logger }}" -JAVA_OPTS="${RANGER_LOGS_OPTS} ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=256m -Xmx${ranger_kms_max_heap_size}" +JAVA_OPTS=" ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=256m -Xmx${ranger_kms_max_heap_size} -Xms1g " if [ "$JAVA_HOME" != "" ]; then export PATH=$JAVA_HOME/bin:$PATH @@ -98,8 +101,9 @@ KMS_CONFIG_FILENAME=ranger-kms-site.xml TOMCAT_LOG_DIR=${RANGER_KMS_LOG_DIR} -TOMCAT_LOG_FILE=${TOMCAT_LOG_DIR}/catalina-ranger-kms.out -TOMCAT_STOP_LOG_FILE=${TOMCAT_LOG_DIR}/catalina-ranger-kms.out +TOMCAT_LOG_FILE=${TOMCAT_LOG_DIR}/catalina.out +TOMCAT_STOP_LOG_FILE=${TOMCAT_LOG_DIR}/stop_catalina.out +KMS_LOG_PROPERTIES_FILE=${RANGER_KMS_EWS_CONF_DIR}/kms-logback.xml if [ ! -d ${TOMCAT_LOG_DIR} ] then @@ -108,10 +112,11 @@ fi KMS_CONF_DIR=${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf SERVER_NAME=rangerkms -JAVA_OPTS="${JAVA_OPTS} ${DB_SSL_PARAM} -Duser=${USER} -Dhostname=${HOSTNAME} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH " +cp="-cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_DIR}/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" +JAVA_OPTS="${JAVA_OPTS} ${DB_SSL_PARAM} -Dmetric.type=${arg3} -Duser=${USER} -Dhostname=${HOSTNAME} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dlogback.configurationFile=file:${KMS_LOG_PROPERTIES_FILE} -Dkms.log.dir=${TOMCAT_LOG_DIR} $cp" createRangerKMSPid () { SLEEP_TIME_AFTER_START=5 - nohup java -D${PROC_NAME} ${JAVA_OPTS} ${JMX_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} > ${TOMCAT_LOG_FILE} 2>&1 & + nohup java -D${PROC_NAME} ${JAVA_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} > ${TOMCAT_LOG_FILE} 2>&1 & VALUE_OF_PID=$! echo "Starting Apache Ranger KMS Service" sleep $SLEEP_TIME_AFTER_START @@ -193,7 +198,7 @@ elif [ "${action}" == "METRIC" ]; then metric; exit elif [ "${action}" == "VERSION" ]; then - ( cd ${RANGER_KMS_LIB_DIR} ; java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo ) + ( cd ${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/lib ; java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo ) exit else echo "Invalid argument [$1];" diff --git a/tdp_vars_defaults/ranger/ranger.yml b/tdp_vars_defaults/ranger/ranger.yml index cec94e65..8b7e8744 100644 --- a/tdp_vars_defaults/ranger/ranger.yml +++ b/tdp_vars_defaults/ranger/ranger.yml @@ -29,7 +29,7 @@ ranger_kms_conf_dir: "/etc/kms" # Ranger pid directories ranger_pid_dir: /var/run/ranger ranger_usersync_pid_dir: /var/run/ranger-usersync -ranger_kms_pid_dir: /var/run/ranger-kms +ranger_kms_pid_dir: /var/run/ranger_kms # Ranger logging configuration # Root logger should be: [RFA | DRFA]