feat(sparkR): integration of sparkr

roles/spark/client/tasks/install.yml

@@ -3,12 +3,12 @@
 
 ---
 - name: Ensure spark common installation steps are performed
-  import_role:
+  ansible.builtin.import_role:
     name: tosit.tdp.spark.common
     tasks_from: install
 
 - name: Upload spark hbase jar
-  copy:
+  ansible.builtin.copy:
     src: "{{ binaries_local_dir }}/{{ spark_hbase_dist_file }}"
     dest: "{{ spark_root_dir }}/{{ spark_release }}/jars"
     owner: root
@@ -18,7 +18,7 @@
   diff: false
 
 - name: Copy audience-annotations dependency
-  copy:
+  ansible.builtin.copy:
     src: "{{ hbase_install_dir }}/lib/client-facing-thirdparty/audience-annotations-0.5.0.jar"
     dest: "{{ spark_root_dir }}/{{ spark_release }}/jars"
     owner: root
@@ -29,41 +29,57 @@
   diff: false
 
 - name: Create configuration directory
-  file:
+  ansible.builtin.file:
     path: "{{ spark_client_conf_dir }}"
     state: directory
     owner: root
     group: root
     mode: "755"
 
 - name: "Render /usr/bin/{{ spark_version }}-submit command"
-  template:
+  ansible.builtin.template:
     src: spark-submit-command.j2
     dest: "/usr/bin/{{ spark_version }}-submit"
     owner: root
     group: root
     mode: "755"
 
 - name: "Render /usr/bin/{{ spark_version }}-shell command"
-  template:
+  ansible.builtin.template:
     src: spark-shell-command.j2
     dest: "/usr/bin/{{ spark_version }}-shell"
     owner: root
     group: root
     mode: "755"
 
 - name: "Render /usr/bin/{{ spark_version }}-sql command"
-  template:
+  ansible.builtin.template:
     src: spark-sql-command.j2
     dest: "/usr/bin/{{ spark_version }}-sql"
     owner: root
     group: root
     mode: "755"
 
 - name: "Render /usr/bin/py{{ spark_version }} command"
-  template:
+  ansible.builtin.template:
     src: pyspark-command.j2
     dest: "/usr/bin/py{{ spark_version }}"
     owner: root
     group: root
     mode: "755"
+
+- name: "Render /usr/bin/{{ spark_version }}R-shell command"
+  ansible.builtin.template:
+    src: sparkr-shell-command.j2
+    dest: "/usr/bin/{{ spark_version }}r-shell"
+    owner: root
+    group: root
+    mode: "755"
+  when: (spark_version == "spark3") and spark_enable_r
+
+- name: Create symbolic link to R library
+  ansible.builtin.file:
+    src: "{{ spark_install_dir }}/R/lib/SparkR"
+    dest: "/usr/lib64/R/library/SparkR"
+    state: link
+  when: (spark_version == "spark3") and spark_enable_r

roles/spark/common/tasks/install.yml

@@ -3,7 +3,7 @@
 
 ---
 - name: Upload {{ spark_dist_file }}
-  copy:
+  ansible.builtin.copy:
     src: "{{ binaries_local_dir }}/{{ spark_dist_file }}"
     dest: "{{ binaries_upload_dir }}"
     owner: root
@@ -12,7 +12,7 @@
   diff: false
 
 - name: Extract {{ spark_dist_file }}
-  unarchive:
+  ansible.builtin.unarchive:
     src: "{{ binaries_upload_dir }}/{{ spark_dist_file }}"
     dest: "{{ spark_root_dir }}"
     owner: root
@@ -22,36 +22,36 @@
     creates: "{{ spark_root_dir }}/{{ spark_release }}"
 
 - name: Create symbolic link to Spark installation
-  file:
+  ansible.builtin.file:
     src: "{{ spark_root_dir }}/{{ spark_release }}"
     dest: "{{ spark_install_dir }}"
     state: link
 
 - name: Ensure spark user exists
-  include_role:
+  ansible.builtin.include_role:
     name: tosit.tdp.utils.user
   vars:
     user: "{{ spark_user }}"
     group: "{{ hadoop_group }}"
 
 - name: Create directory for pid
-  file:
+  ansible.builtin.file:
     path: "{{ spark_pid_dir }}"
     state: directory
     owner: "{{ spark_user }}"
     group: "{{ hadoop_group }}"
     mode: "750"
 
 - name: Template spark tmpfiles.d
-  template:
+  ansible.builtin.template:
     src: tmpfiles-spark.conf.j2
     dest: "/etc/tmpfiles.d/{{ spark_version }}.conf"
     owner: root
     group: root
     mode: "644"
 
 - name: Create log directory
-  file:
+  ansible.builtin.file:
     path: "{{ spark_log_dir }}"
     state: directory
     owner: "{{ spark_user }}"

roles/spark/common/templates/sparkr-shell-command.j2

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+
+export YARN_CONF_DIR=/etc/hadoop/conf
+
+
+/opt/tdp/spark3/bin/sparkR "$@"

tdp_vars_defaults/spark/spark.yml

@@ -32,6 +32,9 @@ hbase_install_dir: "{{ spark_root_dir }}/hbase"
 # Spark pid directories
 spark_pid_dir: /var/run/spark
 
+# SparkR
+spark_enable_r: false
+
 #Spark logging configuration
 # Root logger should be: [RFA | DRFA]
 spark_root_logger: RFA

tdp_vars_defaults/spark3/spark3.yml

@@ -27,6 +27,9 @@ spark_hs_conf_dir: "{{ spark_conf_dir }}/conf.hs"
 # Spark pid directories
 spark_pid_dir: /var/run/spark3
 
+# SparkR
+spark_enable_r: false
+
 #Spark3 logging configuration
 # Root logger should be: [RFA | DRFA]
 spark_root_logger: RFA