fix(knox): fix webhdfs service definition for better support of path …

…parameter
TOSIT-IO · Nov 24, 2023 · bd3f24e · bd3f24e
1 parent 755575d
commit bd3f24e
Show file tree

Hide file tree

Showing 3 changed files with 161 additions and 0 deletions.
diff --git a/roles/knox/common/templates/services/webhdfs/2.4.0/rewrite.xml.j2 b/roles/knox/common/templates/services/webhdfs/2.4.0/rewrite.xml.j2
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<rules>
+
+    <rule dir="OUT" scope="WEBHDFS" name="WEBHDFS/webhdfs/outbound" pattern="hdfs://*:*/{path=**}?{**}">
+        <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="webhdfs://*:*/{path=**}?{**}">
+        <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound/namenode/headers/location">
+        <match pattern="{scheme}://{host}:{port}/{path=**}?{**}"/>
+        <rewrite template="{$frontend[url]}/webhdfs/data/v1/{path=**}?{scheme}?host={$hostmap(host)}?{port}?{**}"/>
+        <encrypt-query/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/hdfs" pattern="hdfs:/{path=**}?{**}">
+        <rewrite template="{$serviceMappedUrl[NAMENODE]}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/webhdfs" pattern="webhdfs:/{path=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/root" pattern="*://*:*/**/webhdfs/{version}/?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/file" pattern="*://*:*/**/webhdfs/{version}/{path_from_url=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/{path_from_url=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home" pattern="*://*:*/**/webhdfs/{version}/~?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home/file" pattern="*://*:*/**/webhdfs/{version}/~/{path_from_url=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}/{path_from_url=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/datanode">
+        <decrypt-query/>
+        <match pattern="*://*:*/**/webhdfs/data/*/{path=**}?{scheme}?{host}?{port}?{**}"/>
+        <rewrite template="{scheme}://{host}:{port}/{path=**}?{**}"/>
+    </rule>
+
+    <filter name="WEBHDFS/webhdfs/outbound/namenode/headers">
+        <content type="application/x-http-headers">
+            <apply path="Location" rule="WEBHDFS/webhdfs/outbound/namenode/headers/location"/>
+        </content>
+        <content type="application/json">
+            <apply path="$.Location" rule="WEBHDFS/webhdfs/outbound/namenode/headers/location"/>
+        </content>
+    </filter>
+
+</rules>
diff --git a/roles/knox/common/templates/services/webhdfs/2.4.0/service.xml.j2 b/roles/knox/common/templates/services/webhdfs/2.4.0/service.xml.j2
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<service role="WEBHDFS" name="webhdfs" version="2.4.0">
+    <metadata>
+        <type>API</type>
+        <context>/webhdfs</context>
+        <shortDesc>Web HDFS</shortDesc>
+        <description>An HTTP REST API which supports the complete FileSystem interface for HDFS.</description>
+        <samples>
+            <sample>
+                <description>List all files under 'testPath'</description>
+                <method>GET</method>
+                <path>v1/testPath?op=LISTSTATUS</path>
+            </sample>
+            <sample>
+                <description>Rename a File/Directory under </description>
+                <method>PUT</method>
+                <path>v1/testPath/testFile?op=RENAME&amp;destination=testPath/renamedFile</path>
+            </sample>
+            <sample>
+                <description>Get Home Directory</description>
+                <method>GET</method>
+                <path>v1/?op=GETHOMEDIRECTORY</path>
+            </sample>
+            <sample>
+                <description>You may check out Apache WebHDFS's REST API documentation here</description>
+                <value>https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html</value>
+            </sample>
+        </samples>
+    </metadata>
+    <routes>
+        <route path="/webhdfs/v1/?**">
+            <rewrite apply="WEBHDFS/webhdfs/inbound/namenode/root" to="request.url"/>
+        </route>
+        <route path="/webhdfs/v1/**?**">
+            <rewrite apply="WEBHDFS/webhdfs/inbound/namenode/file" to="request.url"/>
+            <rewrite apply="WEBHDFS/webhdfs/outbound/namenode/headers" to="response.headers"/>
+            <rewrite apply="WEBHDFS/webhdfs/outbound/namenode/headers" to="response.body"/>
+        </route>
+        <route path="/webhdfs/v1/~?**">
+            <rewrite apply="WEBHDFS/webhdfs/inbound/namenode/home" to="request.url"/>
+        </route>
+        <route path="/webhdfs/v1/~/**?**">
+            <rewrite apply="WEBHDFS/webhdfs/inbound/namenode/home/file" to="request.url"/>
+            <rewrite apply="WEBHDFS/webhdfs/outbound/namenode/headers" to="response.headers"/>
+        </route>
+        <route path="/webhdfs/data/v1/**?**">
+            <rewrite apply="WEBHDFS/webhdfs/inbound/datanode" to="request.url"/>
+            <dispatch contributor-name="http-client" ha-classname="org.apache.knox.gateway.dispatch.DefaultDispatch" />
+        </route>
+    </routes>
+    <dispatch classname="org.apache.knox.gateway.hdfs.dispatch.HdfsHttpClientDispatch" ha-classname="org.apache.knox.gateway.hdfs.dispatch.WebHdfsHaDispatch"/>
+    <testURLs>
+        <testURL>/webhdfs/v1/?op=LISTSTATUS</testURL>
+    </testURLs>
+</service>
diff --git a/roles/knox/gateway/tasks/config.yml b/roles/knox/gateway/tasks/config.yml
@@ -122,6 +122,23 @@
     group: "{{ knox_group }}"
     mode: "644"
 
+# Webhdfs service definition
+- name: Template Webhdfs service.xml
+  ansible.builtin.template:
+    src: services/webhdfs/2.4.0/service.xml.j2
+    dest: "{{ knox_data_dir }}/data/services/webhdfs/2.4.0/service.xml"
+    owner: "{{ knox_user }}"
+    group: "{{ knox_group }}"
+    mode: "644"
+
+- name: Template Webhdfs rewrite.xml
+  ansible.builtin.template:
+    src: services/webhdfs/2.4.0/rewrite.xml.j2
+    dest: "{{ knox_data_dir }}/data/services/webhdfs/2.4.0/rewrite.xml"
+    owner: "{{ knox_user }}"
+    group: "{{ knox_group }}"
+    mode: "644"
+
 - name: Template Knox gateway-site.xml
   ansible.builtin.template:
     src: gateway-site.xml.j2