From 73cad4d9b71ec63b0c47236f41ba7606861e9cd8 Mon Sep 17 00:00:00 2001 From: Jean-Marie Gervais Date: Tue, 26 Sep 2023 13:59:24 +0200 Subject: [PATCH] fix: enforce ISO8601 standard on log timestamps --- tdp_vars_defaults/hadoop/hadoop.yml | 2 +- tdp_vars_defaults/hbase/hbase.yml | 4 +- tdp_vars_defaults/hive/hive.yml | 2 +- tdp_vars_defaults/knox/knox.yml | 2 +- tdp_vars_defaults/ranger/ranger.yml | 2 +- tdp_vars_defaults/spark/spark.yml | 2 +- tdp_vars_defaults/spark3/spark3.yml | 2 +- tdp_vars_defaults/tdp-cluster/tdp-cluster.yml | 43 +++++++++++-------- tdp_vars_defaults/zookeeper/zookeeper.yml | 2 +- 9 files changed, 33 insertions(+), 28 deletions(-) diff --git a/tdp_vars_defaults/hadoop/hadoop.yml b/tdp_vars_defaults/hadoop/hadoop.yml index 71b711e6..c6a0e2fe 100644 --- a/tdp_vars_defaults/hadoop/hadoop.yml +++ b/tdp_vars_defaults/hadoop/hadoop.yml @@ -64,7 +64,7 @@ hadoop_root_logger_level: INFO hadoop_log_file: hadoop.log hadoop_env_client_log_dir: /var/log/hadoop/$USER hadoop_log4j_client_log_dir: . -hadoop_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +hadoop_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config hadoop_log_drfa_date_pattern: "'.'yyyy-MM-dd" # RFA appenders config diff --git a/tdp_vars_defaults/hbase/hbase.yml b/tdp_vars_defaults/hbase/hbase.yml index 93b7c036..7c2f25e4 100644 --- a/tdp_vars_defaults/hbase/hbase.yml +++ b/tdp_vars_defaults/hbase/hbase.yml @@ -55,7 +55,7 @@ hbase_root_logger_level: INFO # Root logger should be: [RFA | DRFA] hbase_root_logger: RFA # Common appenders config -hbase_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +hbase_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config hbase_log_drfa_date_pattern: "'.'yyyy-MM-dd" # RFA appenders config @@ -211,7 +211,7 @@ phoenix_root_logger_level: INFO # Root logger should be: [RFA | DRFA] phoenix_root_logger: RFA # Common appenders config -phoenix_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +phoenix_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config phoenix_log_drfa_date_pattern: "'.'yyyy-MM-dd" # RFA appenders config diff --git a/tdp_vars_defaults/hive/hive.yml b/tdp_vars_defaults/hive/hive.yml index 5544c44d..cdf30ef6 100644 --- a/tdp_vars_defaults/hive/hive.yml +++ b/tdp_vars_defaults/hive/hive.yml @@ -30,7 +30,7 @@ hive_root_logger: RFA # Root logger level should be: [FATAL| ERROR| WARN| INFO| DEBUG| TRACE] hive_root_logger_level: INFO # Common appenders config -hive_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +hive_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # RFA appender config hive_log_rfa_maxfilesize: 256MB hive_log_rfa_maxbackupindex: 10 diff --git a/tdp_vars_defaults/knox/knox.yml b/tdp_vars_defaults/knox/knox.yml index 1d775904..9c97eb85 100644 --- a/tdp_vars_defaults/knox/knox.yml +++ b/tdp_vars_defaults/knox/knox.yml @@ -29,7 +29,7 @@ knox_root_logger_level: INFO knox_root_logger: RFA # Common appenders config -knox_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +knox_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config knox_log_drfa_date_pattern: "'.'yyyy-MM-dd" # RFA appenders config diff --git a/tdp_vars_defaults/ranger/ranger.yml b/tdp_vars_defaults/ranger/ranger.yml index e5debdcf..4c272055 100644 --- a/tdp_vars_defaults/ranger/ranger.yml +++ b/tdp_vars_defaults/ranger/ranger.yml @@ -37,7 +37,7 @@ ranger_root_logger: RFA # Root logger level should be: [FATAL| ERROR| WARN| INFO| DEBUG| TRACE] ranger_root_logger_level: INFO # Common appenders config -ranger_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +ranger_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config ranger_log_drfa_date_pattern: "'.'yyyy-MM-dd-HH-mm" ranger_solr_log_drfa_date_pattern: "%d{yyyy-MM-dd-HH-mm}" diff --git a/tdp_vars_defaults/spark/spark.yml b/tdp_vars_defaults/spark/spark.yml index f168ac8a..c284c1c5 100644 --- a/tdp_vars_defaults/spark/spark.yml +++ b/tdp_vars_defaults/spark/spark.yml @@ -40,7 +40,7 @@ spark_root_logger_level: INFO # Common appenders config spark_log_dir: "{{ spark2_log_dir }}" spark_hs_log_file: "{{ spark2_hs_log_file }}" -spark_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +spark_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config spark_log_drfa_date_pattern: "'.'yyyy-MM-dd-HH-mm" # RFA appenders config diff --git a/tdp_vars_defaults/spark3/spark3.yml b/tdp_vars_defaults/spark3/spark3.yml index fc293839..b936040a 100644 --- a/tdp_vars_defaults/spark3/spark3.yml +++ b/tdp_vars_defaults/spark3/spark3.yml @@ -35,7 +35,7 @@ spark_root_logger_level: INFO # Common appenders config spark_log_dir: "{{ spark3_log_dir }}" spark_hs_log_file: "{{ spark3_hs_log_file }}" -spark_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +spark_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config spark_log_drfa_date_pattern: "'.'yyyy-MM-dd-HH-mm" # RFA appenders config diff --git a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml index e1910499..fc50bf66 100644 --- a/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml +++ b/tdp_vars_defaults/tdp-cluster/tdp-cluster.yml @@ -164,45 +164,50 @@ exporter_hdfs_httpfs_http_port: 18122 ################################# # Service & Component logs dirs # ################################# +# format : log4j 1.x "%d{ISO8601}" is not fully compliant with ISO8601 standard +# (no 'T' as date/time separator) we specify it here. +tdp_date_iso8601_with_tz: "%d{yyyy-MM-dd'T'hh:mm:ss.SSSXXX}" +tdp_default_log_layout_pattern: '{{ tdp_date_iso8601_with_tz }} - %-5p [%t:%C{1}@%L] - %m%n' + hadoop_log_dir: /var/log/hadoop hdfs_log_dir: /var/log/hdfs -hadoop_hdfs_namenode_log_file: "hdfs-namenode_${HOSTNAME}.log" -hadoop_hdfs_journalnode_log_file: "hdfs-journalnode_${HOSTNAME}.log" -hadoop_hdfs_zkfc_log_file: "hdfs-zkfc_${HOSTNAME}.log" -hadoop_hdfs_datanode_log_file: "hdfs-datanode_${HOSTNAME}.log" -hadoop_hdfs_httpfs_log_file: "hdfs-httpfs_${HOSTNAME}.log" +hadoop_hdfs_namenode_log_file: "hdfs-namenode_{{ ansible_fqdn }}.log" +hadoop_hdfs_journalnode_log_file: "hdfs-journalnode_{{ ansible_fqdn }}.log" +hadoop_hdfs_zkfc_log_file: "hdfs-zkfc_{{ ansible_fqdn }}.log" +hadoop_hdfs_datanode_log_file: "hdfs-datanode_{{ ansible_fqdn }}.log" +hadoop_hdfs_httpfs_log_file: "hdfs-httpfs_{{ ansible_fqdn }}.log" yarn_log_dir: /var/log/yarn -hadoop_yarn_resourcemanager_log_file: "yarn-resourcemanager_${HOSTNAME}.log" -hadoop_yarn_nodemanager_log_file: "yarn-nodemanager_${HOSTNAME}.log" -hadoop_yarn_timelineserver_log_file: "yarn-timelineserver_${HOSTNAME}.log" +hadoop_yarn_resourcemanager_log_file: "yarn-resourcemanager_{{ ansible_fqdn }}.log" +hadoop_yarn_nodemanager_log_file: "yarn-nodemanager_{{ ansible_fqdn }}.log" +hadoop_yarn_timelineserver_log_file: "yarn-timelineserver_{{ ansible_fqdn }}.log" mapred_log_dir: /var/log/mapred -hadoop_mapred_historyserver_log_file: "mapred-historyserver_${HOSTNAME}.log" +hadoop_mapred_historyserver_log_file: "mapred-historyserver_{{ ansible_fqdn }}.log" hbase_log_dir: /var/log/hbase -hbase_hm_log_file: "hbase-master_${HOSTNAME}.log" -hbase_hrs_log_file: "hbase-regionserver_${HOSTNAME}.log" -hbase_hr_log_file: "hbase-rest_${HOSTNAME}.log" +hbase_hm_log_file: "hbase-master_{{ ansible_fqdn }}.log" +hbase_hrs_log_file: "hbase-regionserver_{{ ansible_fqdn }}.log" +hbase_hr_log_file: "hbase-rest_{{ ansible_fqdn }}.log" phoenix_log_dir: /var/log/phoenix -phoenix_queryserver_log_file: "phoenix-queryserver_${HOSTNAME}.log" +phoenix_queryserver_log_file: "phoenix-queryserver_{{ ansible_fqdn }}.log" hive_log_dir: /var/log/hive -hive_s2_log_file: "hive-hiveserver2_${HOSTNAME}.log" -hive_ms_log_file: "hive-metastore_${HOSTNAME}.log" +hive_s2_log_file: "hive-hiveserver2_{{ ansible_fqdn }}.log" +hive_ms_log_file: "hive-metastore_{{ ansible_fqdn }}.log" knox_log_dir: /var/log/knox -knox_gateway_log_file: "knox-gateway_${HOSTNAME}.log" +knox_gateway_log_file: "knox-gateway_{{ ansible_fqdn }}.log" ranger_log_dir: /var/log/ranger -ranger_admin_log_file: "ranger-admin_${HOSTNAME}.log" -ranger_usersync_log_file: "ranger-usersync_${HOSTNAME}.log" +ranger_admin_log_file: "ranger-admin_{{ ansible_fqdn }}.log" +ranger_usersync_log_file: "ranger-usersync_{{ ansible_fqdn }}.log" ranger_solr_log_file: "ranger-solr_{{ ansible_fqdn }}.log" ranger_kms_log_dir: /var/log/kms -ranger_kms_log_file: "ranger-kms_${HOSTNAME}.log" +ranger_kms_log_file: "ranger-kms_{{ ansible_fqdn }}.log" spark2_log_dir: /var/log/spark spark2_hs_log_file: "spark-historyserver_{{ ansible_fqdn }}.log" diff --git a/tdp_vars_defaults/zookeeper/zookeeper.yml b/tdp_vars_defaults/zookeeper/zookeeper.yml index 683763dd..f5921a58 100644 --- a/tdp_vars_defaults/zookeeper/zookeeper.yml +++ b/tdp_vars_defaults/zookeeper/zookeeper.yml @@ -46,7 +46,7 @@ zookeeper_root_logger_level: INFO # Root logger should be: [RFA | DRFA] zookeeper_root_logger: RFA # Common appenders config -zookeeper_log_layout_pattern: "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" +zookeeper_log_layout_pattern: "{{ tdp_default_log_layout_pattern }}" # DRFA appenders config zookeeper_log_drfa_date_pattern: "'.'yyyy-MM-dd" # RFA appenders config