From f82518740c4aefe413cd53998e29b1ea00eb4e29 Mon Sep 17 00:00:00 2001 From: Paul-Adrien Cordonnier Date: Fri, 13 Dec 2024 11:49:03 +0100 Subject: [PATCH] feat(templates): add custom opts to *-env.sh --- roles/hadoop/common/templates/hadoop-env.sh.j2 | 5 +++++ .../hbase/common/templates/hbase/hbase-env.sh.j2 | 14 ++++++++++++++ roles/hdfs/common/templates/hadoop-env.sh.j2 | 13 +++++++++++++ roles/hdfs/common/templates/httpfs-env.sh.j2 | 5 +++++ roles/hive/common/templates/hive-env.sh.j2 | 15 +++++++++++++++ roles/yarn/common/templates/hadoop-env.sh.j2 | 14 ++++++++++++++ tdp_vars_defaults/hadoop/hadoop.yml | 3 +++ tdp_vars_defaults/hbase/hbase.yml | 6 ++++++ tdp_vars_defaults/hdfs/hdfs.yml | 5 +++++ tdp_vars_defaults/hive/hive.yml | 4 ++++ tdp_vars_defaults/yarn/yarn.yml | 5 +++++ 11 files changed, 89 insertions(+) diff --git a/roles/hadoop/common/templates/hadoop-env.sh.j2 b/roles/hadoop/common/templates/hadoop-env.sh.j2 index cd8ca878..bf0d3cfc 100644 --- a/roles/hadoop/common/templates/hadoop-env.sh.j2 +++ b/roles/hadoop/common/templates/hadoop-env.sh.j2 @@ -433,3 +433,8 @@ export YARN_NODEMANAGER_OPTS="${JMX_OPTS} {{ jmx_exporter_nm_opts }} $YARN_NODEM export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} $YARN_TIMELINESERVER_OPTS" export MAPRED_HISTORYSERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_jhs_opts }} $MAPRED_HISTORYSERVER_OPTS" + +# TDP CUSTOM OPTS + +export TDP_CUSTOM_HADOOP_OPTS="{{ hadoop_client_custom_opts | default('') }}" +export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HADOOP_OPTS" diff --git a/roles/hbase/common/templates/hbase/hbase-env.sh.j2 b/roles/hbase/common/templates/hbase/hbase-env.sh.j2 index de137989..a7ddf8cf 100644 --- a/roles/hbase/common/templates/hbase/hbase-env.sh.j2 +++ b/roles/hbase/common/templates/hbase/hbase-env.sh.j2 @@ -153,3 +153,17 @@ export HBASE_REST_OPTS="$HBASE_REST_OPTS -Djava.security.auth.login.config={{ hb # Export hadoop lib native export LD_LIBRARY_PATH="{{ hadoop_home }}/lib/native/:$LD_LIBRARY_PATH" + +# TDP CUSTOM OPTS + +export TDP_CUSTOM_HBASE_OPTS="{{ hbase_client_custom_opts | default('') }}" +export HBASE_OPTS="$HBASE_OPTS $TDP_CUSTOM_HBASE_OPTS" + +export TDP_CUSTOM_HBASE_MASTER_OPTS="{{ hbase_master_custom_opts | default('') }}" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $TDP_CUSTOM_HBASE_MASTER_OPTS" + +export TDP_CUSTOM_HBASE_REGIONSERVER_OPTS="{{ hbase_rs_custom_opts | default('') }}" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $TDP_CUSTOM_HBASE_REGIONSERVER_OPTS" + +export TDP_CUSTOM_HBASE_REST_OPTS="{{ hbase_rest_custom_opts | default('') }}" +export HBASE_REST_OPTS="$HBASE_REST_OPTS $TDP_CUSTOM_HBASE_REST_OPTS" diff --git a/roles/hdfs/common/templates/hadoop-env.sh.j2 b/roles/hdfs/common/templates/hadoop-env.sh.j2 index 5978fcb6..2c4afec8 100644 --- a/roles/hdfs/common/templates/hadoop-env.sh.j2 +++ b/roles/hdfs/common/templates/hadoop-env.sh.j2 @@ -437,3 +437,16 @@ export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} $YARN_T export MAPRED_HISTORYSERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_jhs_opts }} $MAPRED_HISTORYSERVER_OPTS" +# TDP CUSTOM OPTS + +export TDP_CUSTOM_HADOOP_OPTS="{{ hadoop_client_custom_opts | default('') }}" +export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HADOOP_OPTS" + +export TDP_CUSTOM_HDFS_NAMENODE_OPTS="{{ hdfs_namenodenode_custom_opts | default('') }}" +export HDFS_NAMENODE_OPTS="$HDFS_NAMENODE_OPTS $TDP_CUSTOM_HDFS_NAMENODE_OPTS" + +export TDP_CUSTOM_HDFS_DATANODE_OPTS="{{ hdfs_datanode_custom_opts | default('') }}" +export HDFS_DATANODE_OPTS="$HDFS_DATANODE_OPTS $TDP_CUSTOM_HDFS_DATANODE_OPTS" + +export TDP_CUSTOM_HDFS_JOURNALNODE_OPTS="{{ hdfs_journalnode_custom_opts | default('') }}" +export HDFS_JOURNALNODE_OPTS="$HDFS_JOURNALNODE_OPTS $TDP_CUSTOM_HDFS_JOURNALNODE_OPTS" diff --git a/roles/hdfs/common/templates/httpfs-env.sh.j2 b/roles/hdfs/common/templates/httpfs-env.sh.j2 index 92493aec..3111094a 100644 --- a/roles/hdfs/common/templates/httpfs-env.sh.j2 +++ b/roles/hdfs/common/templates/httpfs-env.sh.j2 @@ -52,3 +52,8 @@ export JMX_OPTS="{{ jmx_common_opts }} {{ jmx_exporter_httpfs_opts }}" export HDFS_HTTPFS_LOGS_OPTS="-Dhadoop.log.file={{ hadoop_hdfs_httpfs_log_file }}" export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS $HDFS_HTTPFS_LOGS_OPTS" + +# TDP CUSTOM OPTS + +export TDP_CUSTOM_HDFS_HTTPFS_OPTS="{{ hdfs_httpfs_custom_opts | default('') }}" +export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HDFS_HTTPFS_OPTS" diff --git a/roles/hive/common/templates/hive-env.sh.j2 b/roles/hive/common/templates/hive-env.sh.j2 index 7bdfff40..f0b8b3e2 100644 --- a/roles/hive/common/templates/hive-env.sh.j2 +++ b/roles/hive/common/templates/hive-env.sh.j2 @@ -65,6 +65,11 @@ if [ "$SERVICE" = "metastore" ]; then export HADOOP_GC_OPTS="-Xloggc:{{ hive_log_dir }}/metastore-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ hive_log_dir }}/hms_heapdump.hprof" export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS ${HADOOP_LOGS_OPTS}" + # TDP CUSTOM OPTS + + export TDP_CUSTOM_METASTORE_HADOOP_OPTS="{{ hive_metastore_custom_opts | default('') }}" + export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_METASTORE_HADOOP_OPTS" + fi if [ "$SERVICE" = "hiveserver2" ]; then @@ -76,6 +81,11 @@ if [ "$SERVICE" = "hiveserver2" ]; then export HADOOP_GC_OPTS="-Xloggc:{{ hive_log_dir }}/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ hive_log_dir }}/hs2_heapdump.hprof" export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS ${HADOOP_LOGS_OPTS}" + # TDP CUSTOM OPTS + + export TDP_CUSTOM_HIVESERVER2_HADOOP_OPTS="{{ hive_hiveserver2_custom_opts | default('') }}" + export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HIVESERVER2_HADOOP_OPTS" + fi export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}" @@ -89,3 +99,8 @@ export HIVE_PID_DIR={{ hive_pid_dir }} # Folder containing extra libraries required for hive compilation/execution can be controlled by: # export HIVE_AUX_JARS_PATH= + +# TDP CUSTOM OPTS + +EXPORT TDP_CUSTOM_HIVE_HADOOP_CLIENT_OPTS="{{ hive_client_custom_opts | default('') }}" +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS $TDP_CUSTOM_HIVE_HADOOP_CLIENT_OPTS" diff --git a/roles/yarn/common/templates/hadoop-env.sh.j2 b/roles/yarn/common/templates/hadoop-env.sh.j2 index cc873394..ce261c14 100644 --- a/roles/yarn/common/templates/hadoop-env.sh.j2 +++ b/roles/yarn/common/templates/hadoop-env.sh.j2 @@ -439,3 +439,17 @@ export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} ${YARN_ export MAPRED_HISTORYSERVER_LOGS_OPTS="-Dhadoop.log.file={{ hadoop_mapred_historyserver_log_file }}" export MAPRED_HISTORYSERVER_OPTS="-Xmx{{ yarn_jobhistoryserver_heapsize }} ${JMX_OPTS} {{ jmx_exporter_jhs_opts }} ${MAPRED_HISTORYSERVER_LOGS_OPTS} $MAPRED_HISTORYSERVER_OPTS" + +# TDP CUSTOM OPTS + +export TDP_CUSTOM_YARN_RESOURCEMANAGER_OPTS="{{ yarn_resourcemanager_custom_opts | default('') }}" +export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS $TDP_CUSTOM_YARN_RESOURCEMANAGER_OPTS" + +export TDP_CUSTOM_YARN_NODEMANAGER_OPTS="{{ yarn_nodemanager_custom_opts | default('') }}" +export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS $TDP_CUSTOM_YARN_NODEMANAGER_OPTS" + +export TDP_CUSTOM_YARN_TIMELINESERVER_OPTS="{{ yarn_timelineserver_custom_opts | default('') }}" +export YARN_TIMELINESERVER_OPTS="$YARN_TIMELINESERVER_OPTS $TDP_CUSTOM_YARN_TIMELINESERVER_OPT" + +export TDP_CUSTOM_MAPRED_HISTORYSERVER_OPTS="{{ mapred_historyserver_custom_opts | default('') }}" +export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS $TDP_CUSTOM_MAPRED_HISTORYSERVER_OPTS" diff --git a/tdp_vars_defaults/hadoop/hadoop.yml b/tdp_vars_defaults/hadoop/hadoop.yml index 7fb1853e..71c33a9e 100644 --- a/tdp_vars_defaults/hadoop/hadoop.yml +++ b/tdp_vars_defaults/hadoop/hadoop.yml @@ -242,3 +242,6 @@ ranger_kms_hosts: |- hadoop_optional_tools: - hadoop-aws + +# Custom opts +hadoop_client_custom_opts: "" diff --git a/tdp_vars_defaults/hbase/hbase.yml b/tdp_vars_defaults/hbase/hbase.yml index eb0070d3..67af7741 100644 --- a/tdp_vars_defaults/hbase/hbase.yml +++ b/tdp_vars_defaults/hbase/hbase.yml @@ -242,3 +242,9 @@ jmx_exporter: password: "{{ hbase_keystore_password }}" certificate: alias: "{{ ansible_fqdn }}" + +# Custom opts +hbase_client_custom_opts: "" +hbase_master_custom_opts: "" +hbase_rs_custom_opts: "" +hbase_rest_custom_opts: "" diff --git a/tdp_vars_defaults/hdfs/hdfs.yml b/tdp_vars_defaults/hdfs/hdfs.yml index 3672c0d5..08f930ba 100644 --- a/tdp_vars_defaults/hdfs/hdfs.yml +++ b/tdp_vars_defaults/hdfs/hdfs.yml @@ -113,3 +113,8 @@ jmx_exporter: certificate: alias: "{{ ansible_fqdn }}" +# Custom opts +hadoop_client_custom_opts: "" +hdfs_namenodenode_custom_opts: "" +hdfs_datanode_custom_opts: "" +hdfs_journalnode_custom_opts: "" diff --git a/tdp_vars_defaults/hive/hive.yml b/tdp_vars_defaults/hive/hive.yml index 65b843d6..d91a7eed 100644 --- a/tdp_vars_defaults/hive/hive.yml +++ b/tdp_vars_defaults/hive/hive.yml @@ -228,3 +228,7 @@ jmx_exporter: certificate: alias: "{{ ansible_fqdn }}" +# Custom opts +hive_metastore_custom_opts: "" +hive_hiveserver2_custom_opts: "" +hive_client_custom_opts: "" diff --git a/tdp_vars_defaults/yarn/yarn.yml b/tdp_vars_defaults/yarn/yarn.yml index 9239a525..3663a442 100644 --- a/tdp_vars_defaults/yarn/yarn.yml +++ b/tdp_vars_defaults/yarn/yarn.yml @@ -215,3 +215,8 @@ jmx_exporter: certificate: alias: "{{ ansible_fqdn }}" +# Custom opts +yarn_resourcemanager_custom_opts: "" +yarn_nodemanager_custom_opts: "" +yarn_timelineserver_custom_opts: "" +mapred_historyserver_custom_opts: ""