diff --git a/playbooks/yarn_resourcemanager_capacityscheduler.yml b/playbooks/utils/yarn_capacity_scheduler.yml similarity index 51% rename from playbooks/yarn_resourcemanager_capacityscheduler.yml rename to playbooks/utils/yarn_capacity_scheduler.yml index 133f6235..e7596913 100644 --- a/playbooks/yarn_resourcemanager_capacityscheduler.yml +++ b/playbooks/utils/yarn_capacity_scheduler.yml @@ -2,13 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 --- -- name: Hadoop Yarn ResourceManager Capacity Scheduler +- name: Update and refresh Hadoop Yarn ResourceManager Capacity Scheduler hosts: yarn_rm tasks: - tosit.tdp.resolve: # noqa unnamed-task node_name: yarn_resourcemanager - - name: Configure YARN RM capacity scheduler - ansible.builtin.import_role: - name: tosit.tdp.yarn.resourcemanager - tasks_from: capacity_scheduler + - ansible.builtin.import_role: + name: tosit.tdp.utils.yarn_capacity_scheduler - ansible.builtin.meta: clear_facts # noqa unnamed-task diff --git a/roles/utils/yarn_capacity_scheduler/defaults/main.yml b/roles/utils/yarn_capacity_scheduler/defaults/main.yml new file mode 100644 index 00000000..06fc148a --- /dev/null +++ b/roles/utils/yarn_capacity_scheduler/defaults/main.yml @@ -0,0 +1,5 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +yarn_capacity_scheduler: "{{ default_capacity_scheduler }}" diff --git a/roles/utils/yarn_capacity_scheduler/tasks/main.yml b/roles/utils/yarn_capacity_scheduler/tasks/main.yml new file mode 100644 index 00000000..2cb6df68 --- /dev/null +++ b/roles/utils/yarn_capacity_scheduler/tasks/main.yml @@ -0,0 +1,24 @@ +# Copyright 2022 TOSIT.IO +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Render capacity-scheduler.xml + ansible.builtin.template: + src: capacity-scheduler.xml.j2 + dest: "{{ hadoop_rm_conf_dir }}/capacity-scheduler.xml" + owner: root + group: root + mode: "644" + +- name: Kinit for yarn + run_once: true + ansible.builtin.command: kinit -kt /etc/security/keytabs/yarn.headless.keytab {{ yarn_headless_principal }}@{{ realm }} + become: true + become_user: "{{ yarn_user }}" + changed_when: false + +- name: Yarn capacity scheduler refresh queues + run_once: true + ansible.builtin.command: "{{ hadoop_root_dir }}/hadoop/bin/yarn rmadmin -refreshQueues" + become: true + become_user: "{{ yarn_user }}" diff --git a/roles/utils/yarn_capacity_scheduler/templates/capacity-scheduler.xml.j2 b/roles/utils/yarn_capacity_scheduler/templates/capacity-scheduler.xml.j2 new file mode 100644 index 00000000..51928a7c --- /dev/null +++ b/roles/utils/yarn_capacity_scheduler/templates/capacity-scheduler.xml.j2 @@ -0,0 +1,8 @@ + + {% for name, value in yarn_capacity_scheduler.items() %} + + {{ name }} + {{ value }} + + {% endfor %} + diff --git a/roles/yarn/common/templates/capacity-scheduler.xml.j2 b/roles/yarn/common/templates/capacity-scheduler.xml.j2 index b8b1cd77..3d417fb7 100644 --- a/roles/yarn/common/templates/capacity-scheduler.xml.j2 +++ b/roles/yarn/common/templates/capacity-scheduler.xml.j2 @@ -1,5 +1,5 @@ - {% for name, value in capacity_scheduler.items() %} + {% for name, value in default_capacity_scheduler.items() %} {{ name }} {{ value }} diff --git a/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml b/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml deleted file mode 100644 index ac84d94c..00000000 --- a/roles/yarn/resourcemanager/tasks/capacity_scheduler.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2022 TOSIT.IO -# SPDX-License-Identifier: Apache-2.0 - ---- -- name: Render capacity-scheduler.xml - ansible.builtin.template: - src: capacity-scheduler.xml.j2 - dest: '{{ hadoop_rm_conf_dir }}/capacity-scheduler.xml' - owner: root - group: root - mode: "644" - -- name: Restart YARN Resource Manager - ansible.builtin.service: - name: hadoop-yarn-resourcemanager - state: restarted diff --git a/roles/yarn/resourcemanager/tasks/config.yml b/roles/yarn/resourcemanager/tasks/config.yml index 5d79c733..5a5b9928 100644 --- a/roles/yarn/resourcemanager/tasks/config.yml +++ b/roles/yarn/resourcemanager/tasks/config.yml @@ -73,6 +73,7 @@ owner: root group: root mode: "644" + force: false # the file will only be rendered if the destination does not exist - name: Render jmxremote.password ansible.builtin.template: diff --git a/tdp_vars_defaults/yarn/yarn.yml b/tdp_vars_defaults/yarn/yarn.yml index 9583a397..980628c3 100644 --- a/tdp_vars_defaults/yarn/yarn.yml +++ b/tdp_vars_defaults/yarn/yarn.yml @@ -156,7 +156,7 @@ ranger_yarn_install_properties: XAAUDIT_SOLR_ENABLE: "{% if 'ranger_solr' in groups and groups['ranger_solr'] %}true{% else %}false{% endif %}" XAAUDIT_SOLR_URL: "{% if 'ranger_solr' in groups and groups['ranger_solr'] %}http://{{ groups['ranger_solr'][0] | tosit.tdp.access_fqdn(hostvars) }}:{{ ranger_solr_http_port }}/solr/ranger_audits{% else %}NONE{% endif %}" -capacity_scheduler: +default_capacity_scheduler: yarn.scheduler.capacity.maximum-applications: 10000 yarn.scheduler.capacity.maximum-am-resource-percent: 0.1 yarn.scheduler.capacity.resource-calculator: org.apache.hadoop.yarn.util.resource.DominantResourceCalculator