diff --git a/roles/spark/client/tasks/install.yml b/roles/spark/client/tasks/install.yml index 27a2652e..df0aced5 100644 --- a/roles/spark/client/tasks/install.yml +++ b/roles/spark/client/tasks/install.yml @@ -3,12 +3,12 @@ --- - name: Ensure spark common installation steps are performed - import_role: + ansible.builtin.import_role: name: tosit.tdp.spark.common tasks_from: install - name: Upload spark hbase jar - copy: + ansible.builtin.copy: src: "{{ binaries_local_dir }}/{{ spark_hbase_dist_file }}" dest: "{{ spark_root_dir }}/{{ spark_release }}/jars" owner: root @@ -18,7 +18,7 @@ diff: false - name: Copy audience-annotations dependency - copy: + ansible.builtin.copy: src: "{{ hbase_install_dir }}/lib/client-facing-thirdparty/audience-annotations-0.5.0.jar" dest: "{{ spark_root_dir }}/{{ spark_release }}/jars" owner: root @@ -29,7 +29,7 @@ diff: false - name: Create configuration directory - file: + ansible.builtin.file: path: "{{ spark_client_conf_dir }}" state: directory owner: root @@ -37,7 +37,7 @@ mode: "755" - name: "Render /usr/bin/{{ spark_version }}-submit command" - template: + ansible.builtin.template: src: spark-submit-command.j2 dest: "/usr/bin/{{ spark_version }}-submit" owner: root @@ -45,7 +45,7 @@ mode: "755" - name: "Render /usr/bin/{{ spark_version }}-shell command" - template: + ansible.builtin.template: src: spark-shell-command.j2 dest: "/usr/bin/{{ spark_version }}-shell" owner: root @@ -53,7 +53,7 @@ mode: "755" - name: "Render /usr/bin/{{ spark_version }}-sql command" - template: + ansible.builtin.template: src: spark-sql-command.j2 dest: "/usr/bin/{{ spark_version }}-sql" owner: root @@ -61,9 +61,25 @@ mode: "755" - name: "Render /usr/bin/py{{ spark_version }} command" - template: + ansible.builtin.template: src: pyspark-command.j2 dest: "/usr/bin/py{{ spark_version }}" owner: root group: root mode: "755" + +- name: "Render /usr/bin/{{ spark_version }}R-shell command" + ansible.builtin.template: + src: sparkr-shell-command.j2 + dest: "/usr/bin/{{ spark_version }}r-shell" + owner: root + group: root + mode: "755" + when: (spark_version == "spark3") and spark_enable_r + +- name: Create symbolic link to R library + ansible.builtin.file: + src: "{{ spark_install_dir }}/R/lib/SparkR" + dest: "/usr/lib64/R/library/SparkR" + state: link + when: (spark_version == "spark3") and spark_enable_r diff --git a/roles/spark/common/tasks/install.yml b/roles/spark/common/tasks/install.yml index db3391c4..27760345 100644 --- a/roles/spark/common/tasks/install.yml +++ b/roles/spark/common/tasks/install.yml @@ -3,7 +3,7 @@ --- - name: Upload {{ spark_dist_file }} - copy: + ansible.builtin.copy: src: "{{ binaries_local_dir }}/{{ spark_dist_file }}" dest: "{{ binaries_upload_dir }}" owner: root @@ -12,7 +12,7 @@ diff: false - name: Extract {{ spark_dist_file }} - unarchive: + ansible.builtin.unarchive: src: "{{ binaries_upload_dir }}/{{ spark_dist_file }}" dest: "{{ spark_root_dir }}" owner: root @@ -22,20 +22,20 @@ creates: "{{ spark_root_dir }}/{{ spark_release }}" - name: Create symbolic link to Spark installation - file: + ansible.builtin.file: src: "{{ spark_root_dir }}/{{ spark_release }}" dest: "{{ spark_install_dir }}" state: link - name: Ensure spark user exists - include_role: + ansible.builtin.include_role: name: tosit.tdp.utils.user vars: user: "{{ spark_user }}" group: "{{ hadoop_group }}" - name: Create directory for pid - file: + ansible.builtin.file: path: "{{ spark_pid_dir }}" state: directory owner: "{{ spark_user }}" @@ -43,7 +43,7 @@ mode: "750" - name: Template spark tmpfiles.d - template: + ansible.builtin.template: src: tmpfiles-spark.conf.j2 dest: "/etc/tmpfiles.d/{{ spark_version }}.conf" owner: root @@ -51,7 +51,7 @@ mode: "644" - name: Create log directory - file: + ansible.builtin.file: path: "{{ spark_log_dir }}" state: directory owner: "{{ spark_user }}" diff --git a/roles/spark/common/templates/sparkr-shell-command.j2 b/roles/spark/common/templates/sparkr-shell-command.j2 new file mode 100644 index 00000000..a35e0ca6 --- /dev/null +++ b/roles/spark/common/templates/sparkr-shell-command.j2 @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + + +export YARN_CONF_DIR=/etc/hadoop/conf + + +/opt/tdp/spark3/bin/sparkR "$@" diff --git a/tdp_vars_defaults/spark/spark.yml b/tdp_vars_defaults/spark/spark.yml index c0ddd0de..2a970d77 100644 --- a/tdp_vars_defaults/spark/spark.yml +++ b/tdp_vars_defaults/spark/spark.yml @@ -32,6 +32,9 @@ hbase_install_dir: "{{ spark_root_dir }}/hbase" # Spark pid directories spark_pid_dir: /var/run/spark +# SparkR +spark_enable_r: false + #Spark logging configuration # Root logger should be: [RFA | DRFA] spark_root_logger: RFA diff --git a/tdp_vars_defaults/spark3/spark3.yml b/tdp_vars_defaults/spark3/spark3.yml index fc293839..f10cacd9 100644 --- a/tdp_vars_defaults/spark3/spark3.yml +++ b/tdp_vars_defaults/spark3/spark3.yml @@ -27,6 +27,9 @@ spark_hs_conf_dir: "{{ spark_conf_dir }}/conf.hs" # Spark pid directories spark_pid_dir: /var/run/spark3 +# SparkR +spark_enable_r: false + #Spark3 logging configuration # Root logger should be: [RFA | DRFA] spark_root_logger: RFA