feat(templates): add custom opts to *-env.sh

TOSIT-IO · Dec 20, 2024 · bc7a365 · bc7a365
1 parent 1370860
commit bc7a365
Show file tree

Hide file tree

Showing 12 changed files with 96 additions and 0 deletions.
diff --git a/roles/hadoop/common/templates/hadoop-env.sh.j2 b/roles/hadoop/common/templates/hadoop-env.sh.j2
@@ -433,3 +433,8 @@ export YARN_NODEMANAGER_OPTS="${JMX_OPTS} {{ jmx_exporter_nm_opts }} $YARN_NODEM
 export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} $YARN_TIMELINESERVER_OPTS"
 
 export MAPRED_HISTORYSERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_jhs_opts }} $MAPRED_HISTORYSERVER_OPTS"
+
+# TDP CUSTOM OPTS
+
+export TDP_CUSTOM_HADOOP_OPTS="{{ hadoop_client_custom_opts | default('') }}"
+export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HADOOP_OPTS"
diff --git a/roles/hbase/common/templates/hbase/hbase-env.sh.j2 b/roles/hbase/common/templates/hbase/hbase-env.sh.j2
@@ -153,3 +153,17 @@ export HBASE_REST_OPTS="$HBASE_REST_OPTS -Djava.security.auth.login.config={{ hb
 
 # Export hadoop lib native
 export LD_LIBRARY_PATH="{{ hadoop_home }}/lib/native/:$LD_LIBRARY_PATH"
+
+# TDP CUSTOM OPTS
+
+export TDP_CUSTOM_HBASE_OPTS="{{ hbase_client_custom_opts | default('') }}"
+export HBASE_OPTS="$HBASE_OPTS $TDP_CUSTOM_HBASE_OPTS"
+
+export TDP_CUSTOM_HBASE_MASTER_OPTS="{{ hbase_master_custom_opts | default('') }}"
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $TDP_CUSTOM_HBASE_MASTER_OPTS"
+
+export TDP_CUSTOM_HBASE_REGIONSERVER_OPTS="{{ hbase_rs_custom_opts | default('') }}"
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $TDP_CUSTOM_HBASE_REGIONSERVER_OPTS"
+
+export TDP_CUSTOM_HBASE_REST_OPTS="{{ hbase_rest_custom_opts | default('') }}"
+export HBASE_REST_OPTS="$HBASE_REST_OPTS $TDP_CUSTOM_HBASE_REST_OPTS"
diff --git a/roles/hbase/common/templates/phoenix_queryserver/hbase-env.sh.j2 b/roles/hbase/common/templates/phoenix_queryserver/hbase-env.sh.j2
@@ -142,3 +142,8 @@ export HBASE_MANAGES_ZK=false
 
 # Export hadoop lib native
 export LD_LIBRARY_PATH="{{ hadoop_home }}/lib/native/:$LD_LIBRARY_PATH"
+
+# TDP CUSTOM OPTS
+
+export TDP_CUSTOM_PHOENIX_QUERYSERVER_OPTS="{{ hbase_phoenix_queryserver_custom_opts | default('') }}"
+export PHOENIX_QUERYSERVER_OPTS="$PHOENIX_QUERYSERVER_OPTS $TDP_CUSTOM_PHOENIX_QUERYSERVER_OPTS"
diff --git a/roles/hdfs/common/templates/hadoop-env.sh.j2 b/roles/hdfs/common/templates/hadoop-env.sh.j2
@@ -437,3 +437,16 @@ export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} $YARN_T
 
 export MAPRED_HISTORYSERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_jhs_opts }} $MAPRED_HISTORYSERVER_OPTS"
 
+# TDP CUSTOM OPTS
+
+export TDP_CUSTOM_HADOOP_OPTS="{{ hadoop_client_custom_opts | default('') }}"
+export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HADOOP_OPTS"
+
+export TDP_CUSTOM_HDFS_NAMENODE_OPTS="{{ hdfs_namenode_custom_opts | default('') }}"
+export HDFS_NAMENODE_OPTS="$HDFS_NAMENODE_OPTS $TDP_CUSTOM_HDFS_NAMENODE_OPTS"
+
+export TDP_CUSTOM_HDFS_DATANODE_OPTS="{{ hdfs_datanode_custom_opts | default('') }}"
+export HDFS_DATANODE_OPTS="$HDFS_DATANODE_OPTS $TDP_CUSTOM_HDFS_DATANODE_OPTS"
+
+export TDP_CUSTOM_HDFS_JOURNALNODE_OPTS="{{ hdfs_journalnode_custom_opts | default('') }}"
+export HDFS_JOURNALNODE_OPTS="$HDFS_JOURNALNODE_OPTS $TDP_CUSTOM_HDFS_JOURNALNODE_OPTS"
diff --git a/roles/hdfs/common/templates/httpfs-env.sh.j2 b/roles/hdfs/common/templates/httpfs-env.sh.j2
@@ -52,3 +52,8 @@ export JMX_OPTS="{{ jmx_common_opts }} {{ jmx_exporter_httpfs_opts }}"
 export HDFS_HTTPFS_LOGS_OPTS="-Dhadoop.log.file={{ hadoop_hdfs_httpfs_log_file }}"
 
 export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS $HDFS_HTTPFS_LOGS_OPTS"
+
+# TDP CUSTOM OPTS
+
+export TDP_CUSTOM_HDFS_HTTPFS_OPTS="{{ hdfs_httpfs_custom_opts | default('') }}"
+export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HDFS_HTTPFS_OPTS"
diff --git a/roles/hive/common/templates/hive-env.sh.j2 b/roles/hive/common/templates/hive-env.sh.j2
@@ -65,6 +65,11 @@ if [ "$SERVICE" = "metastore" ]; then
   export HADOOP_GC_OPTS="-Xloggc:{{ hive_log_dir }}/metastore-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ hive_log_dir }}/hms_heapdump.hprof"
   export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS ${HADOOP_LOGS_OPTS}"
 
+  # TDP CUSTOM OPTS
+
+  export TDP_CUSTOM_METASTORE_HADOOP_OPTS="{{ hive_metastore_custom_opts | default('') }}"
+  export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_METASTORE_HADOOP_OPTS"
+
 fi
 
 if [ "$SERVICE" = "hiveserver2" ]; then
@@ -76,6 +81,11 @@ if [ "$SERVICE" = "hiveserver2" ]; then
   export HADOOP_GC_OPTS="-Xloggc:{{ hive_log_dir }}/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ hive_log_dir }}/hs2_heapdump.hprof" 
   export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS ${HADOOP_LOGS_OPTS}"
 
+  # TDP CUSTOM OPTS
+
+  export TDP_CUSTOM_HIVESERVER2_HADOOP_OPTS="{{ hive_hiveserver2_custom_opts | default('') }}"
+  export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HIVESERVER2_HADOOP_OPTS"
+
 fi
 
 export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}"
@@ -89,3 +99,8 @@ export HIVE_PID_DIR={{ hive_pid_dir }}
 
 # Folder containing extra libraries required for hive compilation/execution can be controlled by:
 # export HIVE_AUX_JARS_PATH=
+
+# TDP CUSTOM OPTS
+
+EXPORT TDP_CUSTOM_HIVE_HADOOP_CLIENT_OPTS="{{ hive_client_custom_opts | default('') }}"
+export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS $TDP_CUSTOM_HIVE_HADOOP_CLIENT_OPTS"
diff --git a/roles/yarn/common/templates/hadoop-env.sh.j2 b/roles/yarn/common/templates/hadoop-env.sh.j2
@@ -439,3 +439,17 @@ export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} ${YARN_
 
 export MAPRED_HISTORYSERVER_LOGS_OPTS="-Dhadoop.log.file={{ hadoop_mapred_historyserver_log_file }}"
 export MAPRED_HISTORYSERVER_OPTS="-Xmx{{ yarn_jobhistoryserver_heapsize }} ${JMX_OPTS} {{ jmx_exporter_jhs_opts }} ${MAPRED_HISTORYSERVER_LOGS_OPTS} $MAPRED_HISTORYSERVER_OPTS"
+
+# TDP CUSTOM OPTS
+
+export TDP_CUSTOM_YARN_RESOURCEMANAGER_OPTS="{{ yarn_resourcemanager_custom_opts | default('') }}"
+export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS $TDP_CUSTOM_YARN_RESOURCEMANAGER_OPTS"
+
+export TDP_CUSTOM_YARN_NODEMANAGER_OPTS="{{ yarn_nodemanager_custom_opts | default('') }}"
+export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS $TDP_CUSTOM_YARN_NODEMANAGER_OPTS"
+
+export TDP_CUSTOM_YARN_TIMELINESERVER_OPTS="{{ yarn_timelineserver_custom_opts | default('') }}"
+export YARN_TIMELINESERVER_OPTS="$YARN_TIMELINESERVER_OPTS $TDP_CUSTOM_YARN_TIMELINESERVER_OPT"
+
+export TDP_CUSTOM_MAPRED_HISTORYSERVER_OPTS="{{ mapred_historyserver_custom_opts | default('') }}"
+export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS $TDP_CUSTOM_MAPRED_HISTORYSERVER_OPTS"
diff --git a/tdp_vars_defaults/hadoop/hadoop.yml b/tdp_vars_defaults/hadoop/hadoop.yml
@@ -242,3 +242,6 @@ ranger_kms_hosts: |-
 
 hadoop_optional_tools:
   - hadoop-aws
+
+# Custom opts
+hadoop_client_custom_opts: ""
diff --git a/tdp_vars_defaults/hbase/hbase.yml b/tdp_vars_defaults/hbase/hbase.yml
@@ -242,3 +242,10 @@ jmx_exporter:
         password: "{{ hbase_keystore_password }}"
       certificate:
         alias: "{{ ansible_fqdn }}"
+
+# Custom opts
+hbase_client_custom_opts: ""
+hbase_master_custom_opts: ""
+hbase_rs_custom_opts: ""
+hbase_rest_custom_opts: ""
+hbase_phoenix_queryserver_custom_opts: ""
diff --git a/tdp_vars_defaults/hdfs/hdfs.yml b/tdp_vars_defaults/hdfs/hdfs.yml
@@ -113,3 +113,9 @@ jmx_exporter:
       certificate:
         alias: "{{ ansible_fqdn }}"
 
+# Custom opts
+hadoop_client_custom_opts: ""
+hdfs_namenode_custom_opts: ""
+hdfs_datanode_custom_opts: ""
+hdfs_journalnode_custom_opts: ""
+hdfs_httpfs_custom_opts: ""
diff --git a/tdp_vars_defaults/hive/hive.yml b/tdp_vars_defaults/hive/hive.yml
@@ -228,3 +228,7 @@ jmx_exporter:
       certificate:
         alias: "{{ ansible_fqdn }}"
 
+# Custom opts
+hive_metastore_custom_opts: ""
+hive_hiveserver2_custom_opts: ""
+hive_client_custom_opts: ""
diff --git a/tdp_vars_defaults/yarn/yarn.yml b/tdp_vars_defaults/yarn/yarn.yml
@@ -215,3 +215,8 @@ jmx_exporter:
       certificate:
         alias: "{{ ansible_fqdn }}"
 
+# Custom opts
+yarn_resourcemanager_custom_opts: ""
+yarn_nodemanager_custom_opts: ""
+yarn_timelineserver_custom_opts: ""
+mapred_historyserver_custom_opts: ""