From 91b6c7cdd0717b050a88ed702ff36d1f41ebb44f Mon Sep 17 00:00:00 2001
From: Mohamed Mehdi BEN AISSA
 <diode-ben-aissa.consultant@dgfip.finances.gouv.fr>
Date: Wed, 25 Oct 2023 17:44:43 +0200
Subject: [PATCH] feat(knox): add support of two instances of livy with
 spark2(livy) and spark3 (livy3)

---
 .../services/livy3/0.4.3/rewrite.xml.j2       | 58 +++++++++++++++++++
 .../services/livy3/0.4.3/service.xml.j2       | 35 +++++++++++
 roles/knox/gateway/tasks/config.yml           | 25 ++++++++
 tdp_vars_defaults/knox/knox.yml               |  3 +
 4 files changed, 121 insertions(+)
 create mode 100644 roles/knox/common/templates/services/livy3/0.4.3/rewrite.xml.j2
 create mode 100644 roles/knox/common/templates/services/livy3/0.4.3/service.xml.j2

diff --git a/roles/knox/common/templates/services/livy3/0.4.3/rewrite.xml.j2 b/roles/knox/common/templates/services/livy3/0.4.3/rewrite.xml.j2
new file mode 100644
index 00000000..05a48391
--- /dev/null
+++ b/roles/knox/common/templates/services/livy3/0.4.3/rewrite.xml.j2
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<rules>
+  <!-- These should be removed at some point since Livy
+       doesn't have /v1 in the REST API specification -->
+  <rule dir="IN" name="LIVYSERVER3/livy3/root/v1/inbound/" pattern="*://*:*/**/livy3/v1">
+    <rewrite template="{$serviceUrl[LIVYSERVER3]}"/>
+  </rule>
+  <rule dir="IN" name="LIVYSERVER3/livy3/path/v1/inbound" pattern="*://*:*/**/livy3/v1/{path=**}?{**}">
+    <rewrite template="{$serviceUrl[LIVYSERVER3]}/{path=**}?{**}"/>
+  </rule>
+
+  <!-- Prefer these rules without v1 in the url -->
+  <rule dir="IN" name="LIVYSERVER3/livy3/root/inbound" pattern="*://*:*/**/livy3">
+    <rewrite template="{$serviceUrl[LIVYSERVER3]}"/>
+  </rule>
+  <rule dir="IN" name="LIVYSERVER3/livy3/path/inbound" pattern="*://*:*/**/livy3/{path=**}?{**}">
+    <rewrite template="{$serviceUrl[LIVYSERVER3]}/{path=**}?{**}"/>
+  </rule>
+
+  <rule dir="OUT" name="LIVYSERVER3/livy3/outbound/sparkurl" pattern="*://*:*/proxy/{**}">
+    <rewrite template="{$frontend[url]}/yarn/proxy/{**}"/>
+  </rule>
+
+  <rule dir="OUT" name="LIVYSERVER3/livy3/outbound/logs" pattern="{scheme}://{host}:{port}/node/containerlogs/{**}?{**}">
+    <rewrite template="{$frontend[url]}/yarn/node/containerlogs/{**}?{**}?{scheme}?host={$hostmap(host)}?{port}"/>
+  </rule>
+
+  <rule dir="OUT" name="LIVYSERVER3/livy3/outbound/logs2" pattern="{scheme}://{host}:{port}/node/containerlogs/{**}">
+    <rewrite template="{$frontend[url]}/yarn/node/containerlogs/{**}?{scheme}?host={$hostmap(host)}?{port}"/>
+  </rule>
+
+  <rule dir="OUT" name="LIVYSERVER3/livy3/outbound/headers/ui" pattern="{scheme}://{host}:{port}/ui/">
+    <rewrite template="{$frontend[url]}/livy3/ui/"/>
+  </rule>
+
+  <filter name="LIVYSERVER3/livy3/outbound/headers">
+    <content type="application/x-http-headers">
+      <apply path="Location" rule="LIVYSERVER3/livy3/outbound/headers/ui"/>
+    </content>
+  </filter>
+
+</rules>
diff --git a/roles/knox/common/templates/services/livy3/0.4.3/service.xml.j2 b/roles/knox/common/templates/services/livy3/0.4.3/service.xml.j2
new file mode 100644
index 00000000..b78dc072
--- /dev/null
+++ b/roles/knox/common/templates/services/livy3/0.4.3/service.xml.j2
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<service role="LIVYSERVER3" name="livy3" version="0.4.3">
+  <metadata>
+    <type>API_AND_UI</type>
+    <context>/livy3/</context>
+    <shortDesc>Livy Server</shortDesc>
+    <description>Apache Livy is a service that enables easy interaction with a Spark cluster over a REST interface.</description>
+  </metadata>
+  <routes>
+    <route path="/livy3/**?**"/>
+    <route path="/livy3">
+      <rewrite apply="LIVYSERVER3/livy3/outbound/headers" to="response.headers"/>
+    </route>
+    <route path="/livy3/">
+      <rewrite apply="LIVYSERVER3/livy3/outbound/headers" to="response.headers"/>
+    </route>
+  </routes>
+  <dispatch classname="org.apache.knox.gateway.livy.LivyDispatch"/>
+</service>
diff --git a/roles/knox/gateway/tasks/config.yml b/roles/knox/gateway/tasks/config.yml
index 0214af2f..9c645a92 100644
--- a/roles/knox/gateway/tasks/config.yml
+++ b/roles/knox/gateway/tasks/config.yml
@@ -72,6 +72,31 @@
     group: "{{ knox_group }}"
     mode: "644"
 
+# Livy Spark 3 service definition
+- name: Create Livy Spark 3 service dir
+  ansible.builtin.file:
+    path: "{{ knox_data_dir }}/data/services/livy3/0.4.3"
+    state: directory
+    owner: "{{ knox_user }}"
+    group: "{{ knox_group }}"
+    mode: "755"
+
+- name: Template Livy Spark 3 service.xml
+  ansible.builtin.template:
+    src: services/livy3/0.4.3/service.xml.j2
+    dest: "{{ knox_data_dir }}/data/services/livy3/0.4.3/service.xml"
+    owner: "{{ knox_user }}"
+    group: "{{ knox_group }}"
+    mode: "644"
+
+- name: Template Livy Spark 3 rewrite.xml
+  ansible.builtin.template:
+    src: services/livy3/0.4.3/rewrite.xml.j2
+    dest: "{{ knox_data_dir }}/data/services/livy3/0.4.3/rewrite.xml"
+    owner: "{{ knox_user }}"
+    group: "{{ knox_group }}"
+    mode: "644"
+
 - name: Template Knox gateway-site.xml
   ansible.builtin.template:
     src: gateway-site.xml.j2
diff --git a/tdp_vars_defaults/knox/knox.yml b/tdp_vars_defaults/knox/knox.yml
index be9de811..77748914 100644
--- a/tdp_vars_defaults/knox/knox.yml
+++ b/tdp_vars_defaults/knox/knox.yml
@@ -166,6 +166,9 @@ gateway_topology:
         hosts: "{{ groups['hbase_master'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}"
         port: "{{ hbase_master_info_port }}"
       LIVYSERVER:
+        hosts: "{% if groups['livy_server'] is defined %}{{ groups['livy_server'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% else %}{% endif %}"
+        port: "8998"
+      LIVYSERVER3:
         hosts: "{% if groups['livy_spark3_server'] is defined %}{{ groups['livy_spark3_server'] | default([]) | map('tosit.tdp.access_fqdn', hostvars) | list }}{% else %}{% endif %}"
         port: "8999"