Skip to content

Commit

Permalink
feat(templates): add custom opts to *-env.sh
Browse files Browse the repository at this point in the history
  • Loading branch information
PACordonnier committed Dec 13, 2024
1 parent 1370860 commit f825187
Show file tree
Hide file tree
Showing 11 changed files with 89 additions and 0 deletions.
5 changes: 5 additions & 0 deletions roles/hadoop/common/templates/hadoop-env.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -433,3 +433,8 @@ export YARN_NODEMANAGER_OPTS="${JMX_OPTS} {{ jmx_exporter_nm_opts }} $YARN_NODEM
export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} $YARN_TIMELINESERVER_OPTS"

export MAPRED_HISTORYSERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_jhs_opts }} $MAPRED_HISTORYSERVER_OPTS"

# TDP CUSTOM OPTS

export TDP_CUSTOM_HADOOP_OPTS="{{ hadoop_client_custom_opts | default('') }}"
export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HADOOP_OPTS"
14 changes: 14 additions & 0 deletions roles/hbase/common/templates/hbase/hbase-env.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -153,3 +153,17 @@ export HBASE_REST_OPTS="$HBASE_REST_OPTS -Djava.security.auth.login.config={{ hb

# Export hadoop lib native
export LD_LIBRARY_PATH="{{ hadoop_home }}/lib/native/:$LD_LIBRARY_PATH"

# TDP CUSTOM OPTS

export TDP_CUSTOM_HBASE_OPTS="{{ hbase_client_custom_opts | default('') }}"
export HBASE_OPTS="$HBASE_OPTS $TDP_CUSTOM_HBASE_OPTS"

export TDP_CUSTOM_HBASE_MASTER_OPTS="{{ hbase_master_custom_opts | default('') }}"
export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $TDP_CUSTOM_HBASE_MASTER_OPTS"

export TDP_CUSTOM_HBASE_REGIONSERVER_OPTS="{{ hbase_rs_custom_opts | default('') }}"
export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $TDP_CUSTOM_HBASE_REGIONSERVER_OPTS"

export TDP_CUSTOM_HBASE_REST_OPTS="{{ hbase_rest_custom_opts | default('') }}"
export HBASE_REST_OPTS="$HBASE_REST_OPTS $TDP_CUSTOM_HBASE_REST_OPTS"
13 changes: 13 additions & 0 deletions roles/hdfs/common/templates/hadoop-env.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -437,3 +437,16 @@ export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} $YARN_T

export MAPRED_HISTORYSERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_jhs_opts }} $MAPRED_HISTORYSERVER_OPTS"

# TDP CUSTOM OPTS

export TDP_CUSTOM_HADOOP_OPTS="{{ hadoop_client_custom_opts | default('') }}"
export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HADOOP_OPTS"

export TDP_CUSTOM_HDFS_NAMENODE_OPTS="{{ hdfs_namenodenode_custom_opts | default('') }}"
export HDFS_NAMENODE_OPTS="$HDFS_NAMENODE_OPTS $TDP_CUSTOM_HDFS_NAMENODE_OPTS"

export TDP_CUSTOM_HDFS_DATANODE_OPTS="{{ hdfs_datanode_custom_opts | default('') }}"
export HDFS_DATANODE_OPTS="$HDFS_DATANODE_OPTS $TDP_CUSTOM_HDFS_DATANODE_OPTS"

export TDP_CUSTOM_HDFS_JOURNALNODE_OPTS="{{ hdfs_journalnode_custom_opts | default('') }}"
export HDFS_JOURNALNODE_OPTS="$HDFS_JOURNALNODE_OPTS $TDP_CUSTOM_HDFS_JOURNALNODE_OPTS"
5 changes: 5 additions & 0 deletions roles/hdfs/common/templates/httpfs-env.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,8 @@ export JMX_OPTS="{{ jmx_common_opts }} {{ jmx_exporter_httpfs_opts }}"
export HDFS_HTTPFS_LOGS_OPTS="-Dhadoop.log.file={{ hadoop_hdfs_httpfs_log_file }}"

export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS $HDFS_HTTPFS_LOGS_OPTS"

# TDP CUSTOM OPTS

export TDP_CUSTOM_HDFS_HTTPFS_OPTS="{{ hdfs_httpfs_custom_opts | default('') }}"
export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HDFS_HTTPFS_OPTS"
15 changes: 15 additions & 0 deletions roles/hive/common/templates/hive-env.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ if [ "$SERVICE" = "metastore" ]; then
export HADOOP_GC_OPTS="-Xloggc:{{ hive_log_dir }}/metastore-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ hive_log_dir }}/hms_heapdump.hprof"
export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS ${HADOOP_LOGS_OPTS}"

# TDP CUSTOM OPTS

export TDP_CUSTOM_METASTORE_HADOOP_OPTS="{{ hive_metastore_custom_opts | default('') }}"
export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_METASTORE_HADOOP_OPTS"

fi

if [ "$SERVICE" = "hiveserver2" ]; then
Expand All @@ -76,6 +81,11 @@ if [ "$SERVICE" = "hiveserver2" ]; then
export HADOOP_GC_OPTS="-Xloggc:{{ hive_log_dir }}/hiveserver2-gc-%t.log -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={{ hive_log_dir }}/hs2_heapdump.hprof"
export HADOOP_OPTS="$HADOOP_OPTS $JMX_OPTS ${HADOOP_LOGS_OPTS}"

# TDP CUSTOM OPTS

export TDP_CUSTOM_HIVESERVER2_HADOOP_OPTS="{{ hive_hiveserver2_custom_opts | default('') }}"
export HADOOP_OPTS="$HADOOP_OPTS $TDP_CUSTOM_HIVESERVER2_HADOOP_OPTS"

fi

export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}"
Expand All @@ -89,3 +99,8 @@ export HIVE_PID_DIR={{ hive_pid_dir }}

# Folder containing extra libraries required for hive compilation/execution can be controlled by:
# export HIVE_AUX_JARS_PATH=

# TDP CUSTOM OPTS

EXPORT TDP_CUSTOM_HIVE_HADOOP_CLIENT_OPTS="{{ hive_client_custom_opts | default('') }}"
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS $TDP_CUSTOM_HIVE_HADOOP_CLIENT_OPTS"
14 changes: 14 additions & 0 deletions roles/yarn/common/templates/hadoop-env.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -439,3 +439,17 @@ export YARN_TIMELINESERVER_OPTS="${JMX_OPTS} {{ jmx_exporter_ats_opts }} ${YARN_

export MAPRED_HISTORYSERVER_LOGS_OPTS="-Dhadoop.log.file={{ hadoop_mapred_historyserver_log_file }}"
export MAPRED_HISTORYSERVER_OPTS="-Xmx{{ yarn_jobhistoryserver_heapsize }} ${JMX_OPTS} {{ jmx_exporter_jhs_opts }} ${MAPRED_HISTORYSERVER_LOGS_OPTS} $MAPRED_HISTORYSERVER_OPTS"

# TDP CUSTOM OPTS

export TDP_CUSTOM_YARN_RESOURCEMANAGER_OPTS="{{ yarn_resourcemanager_custom_opts | default('') }}"
export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS $TDP_CUSTOM_YARN_RESOURCEMANAGER_OPTS"

export TDP_CUSTOM_YARN_NODEMANAGER_OPTS="{{ yarn_nodemanager_custom_opts | default('') }}"
export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS $TDP_CUSTOM_YARN_NODEMANAGER_OPTS"

export TDP_CUSTOM_YARN_TIMELINESERVER_OPTS="{{ yarn_timelineserver_custom_opts | default('') }}"
export YARN_TIMELINESERVER_OPTS="$YARN_TIMELINESERVER_OPTS $TDP_CUSTOM_YARN_TIMELINESERVER_OPT"

export TDP_CUSTOM_MAPRED_HISTORYSERVER_OPTS="{{ mapred_historyserver_custom_opts | default('') }}"
export MAPRED_HISTORYSERVER_OPTS="$MAPRED_HISTORYSERVER_OPTS $TDP_CUSTOM_MAPRED_HISTORYSERVER_OPTS"
3 changes: 3 additions & 0 deletions tdp_vars_defaults/hadoop/hadoop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -242,3 +242,6 @@ ranger_kms_hosts: |-
hadoop_optional_tools:
- hadoop-aws

# Custom opts
hadoop_client_custom_opts: ""
6 changes: 6 additions & 0 deletions tdp_vars_defaults/hbase/hbase.yml
Original file line number Diff line number Diff line change
Expand Up @@ -242,3 +242,9 @@ jmx_exporter:
password: "{{ hbase_keystore_password }}"
certificate:
alias: "{{ ansible_fqdn }}"

# Custom opts
hbase_client_custom_opts: ""
hbase_master_custom_opts: ""
hbase_rs_custom_opts: ""
hbase_rest_custom_opts: ""
5 changes: 5 additions & 0 deletions tdp_vars_defaults/hdfs/hdfs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,8 @@ jmx_exporter:
certificate:
alias: "{{ ansible_fqdn }}"

# Custom opts
hadoop_client_custom_opts: ""
hdfs_namenodenode_custom_opts: ""
hdfs_datanode_custom_opts: ""
hdfs_journalnode_custom_opts: ""
4 changes: 4 additions & 0 deletions tdp_vars_defaults/hive/hive.yml
Original file line number Diff line number Diff line change
Expand Up @@ -228,3 +228,7 @@ jmx_exporter:
certificate:
alias: "{{ ansible_fqdn }}"

# Custom opts
hive_metastore_custom_opts: ""
hive_hiveserver2_custom_opts: ""
hive_client_custom_opts: ""
5 changes: 5 additions & 0 deletions tdp_vars_defaults/yarn/yarn.yml
Original file line number Diff line number Diff line change
Expand Up @@ -215,3 +215,8 @@ jmx_exporter:
certificate:
alias: "{{ ansible_fqdn }}"

# Custom opts
yarn_resourcemanager_custom_opts: ""
yarn_nodemanager_custom_opts: ""
yarn_timelineserver_custom_opts: ""
mapred_historyserver_custom_opts: ""

0 comments on commit f825187

Please sign in to comment.