From ea78ada55af3ffcbb3d4cd2b8a8a96f79a95efa2 Mon Sep 17 00:00:00 2001 From: GuillaumeHold Date: Tue, 26 Sep 2023 16:32:48 +0200 Subject: [PATCH 1/3] feat: add hive compaction initiator configuration --- tdp_vars_defaults/hive/hive_metastore.yml | 3 +++ topology.ini | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tdp_vars_defaults/hive/hive_metastore.yml b/tdp_vars_defaults/hive/hive_metastore.yml index 3ed977c6..71d0ce5f 100644 --- a/tdp_vars_defaults/hive/hive_metastore.yml +++ b/tdp_vars_defaults/hive/hive_metastore.yml @@ -6,5 +6,8 @@ hive_ms_credentials_store_path: "{{ hive_ms_conf_dir }}/hive.jceks" hive_ms_credentials_store_uri: localjceks://file{{ hive_ms_credentials_store_path }} +compaction_enabled: true + hive_site: hadoop.security.credential.provider.path: "{{ hive_ms_credentials_store_uri }}" + hive.compactor.initiator.on: "{{ compaction_enabled and ('hive_compaction_initiator' in group_names ) | string | lower }}" diff --git a/topology.ini b/topology.ini index 1d5c4f56..d69b100d 100644 --- a/topology.ini +++ b/topology.ini @@ -71,7 +71,13 @@ master2 master3 [hive_ms:children] +hive_compaction_initiator +hive_ms_ha + +[hive_compaction_initiator:children] master2 + +[hive_ms_ha:children] master3 [hive_client:children] From a247faa0a7a624ba17d45b7029ba4facc2e98470 Mon Sep 17 00:00:00 2001 From: GuillaumeHold Date: Thu, 28 Sep 2023 11:51:25 +0200 Subject: [PATCH 2/3] fix(hive): correct hive compaction initiator configuration --- tdp_vars_defaults/hive/hive_metastore.yml | 2 +- topology.ini | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/tdp_vars_defaults/hive/hive_metastore.yml b/tdp_vars_defaults/hive/hive_metastore.yml index 71d0ce5f..d68a3007 100644 --- a/tdp_vars_defaults/hive/hive_metastore.yml +++ b/tdp_vars_defaults/hive/hive_metastore.yml @@ -10,4 +10,4 @@ compaction_enabled: true hive_site: hadoop.security.credential.provider.path: "{{ hive_ms_credentials_store_uri }}" - hive.compactor.initiator.on: "{{ compaction_enabled and ('hive_compaction_initiator' in group_names ) | string | lower }}" + hive.compactor.initiator.on: "{{ compaction_enabled and (inventory_hostname == groups['hive_ms'][0]) | string | lower }}" \ No newline at end of file diff --git a/topology.ini b/topology.ini index d69b100d..1d5c4f56 100644 --- a/topology.ini +++ b/topology.ini @@ -71,13 +71,7 @@ master2 master3 [hive_ms:children] -hive_compaction_initiator -hive_ms_ha - -[hive_compaction_initiator:children] master2 - -[hive_ms_ha:children] master3 [hive_client:children] From c29e03de3653378fdd403eaa22aa545315ce8d17 Mon Sep 17 00:00:00 2001 From: GuillaumeHold Date: Fri, 27 Oct 2023 10:17:29 +0200 Subject: [PATCH 3/3] fix: disable transactions and compactions by default --- tdp_vars_defaults/hive/hive_metastore.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdp_vars_defaults/hive/hive_metastore.yml b/tdp_vars_defaults/hive/hive_metastore.yml index d68a3007..8dd08c44 100644 --- a/tdp_vars_defaults/hive/hive_metastore.yml +++ b/tdp_vars_defaults/hive/hive_metastore.yml @@ -6,7 +6,9 @@ hive_ms_credentials_store_path: "{{ hive_ms_conf_dir }}/hive.jceks" hive_ms_credentials_store_uri: localjceks://file{{ hive_ms_credentials_store_path }} -compaction_enabled: true +# Hive Transactions variables +hive_transactions_enabled: false +compaction_enabled: "{{ hive_transactions_enabled }}" hive_site: hadoop.security.credential.provider.path: "{{ hive_ms_credentials_store_uri }}"