diff --git a/tests/conftest.py b/tests/conftest.py index 0146404..b3c26d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -193,6 +193,11 @@ def knox_gateway(host: Host, user: str) -> Dict[str, str]: } +@pytest.fixture(scope="session") +def webhdfs_gateway(host: Host, user: str) -> str: + return f"https://{host.backend.get_hosts('hdfs_nn')[0]}:9871" + + @pytest.fixture(scope="session") def hbase_rest(host: Host) -> str: return f"https://{host.backend.get_hosts('hbase_rest')[0]}:8080" diff --git a/tests/test_hdfs.py b/tests/test_hdfs.py index 819c5bb..2eb6116 100644 --- a/tests/test_hdfs.py +++ b/tests/test_hdfs.py @@ -1,8 +1,14 @@ +import json import os -from typing import Dict +from typing import Callable, Dict, List + +import pytest from testinfra import host +from .conftest import USERS, retry + + testinfra_hosts = ["edge"] @@ -55,3 +61,69 @@ def test_create_temporary_file_in_user_directory( ) assert "No such file or directory" in hdfs_cmd.stderr, hdfs_cmd assert hdfs_cmd.stdout == "", hdfs_cmd + + +def test_create_webhdfs_temporary_file_in_user_directory( + host: host.Host, + user: str, + user_file: Dict[str, str], + webhdfs_gateway: str, + curl: Callable, +): + distant_file = user_file["distant_file"] + distant_hdfs_path = user_file["distant_hdfs_path"] + file_content = user_file["file_content"] + webhdfs_url = webhdfs_gateway + webhdfs_gateway_url = f"{webhdfs_url}/webhdfs/v1" + + with host.sudo(user): + curl_result = retry( + lambda: curl( + f"-L -T {distant_file} --negotiate -u : -X PUT '{webhdfs_gateway_url}/user/{user}/{distant_hdfs_path}?op=CREATE'" + ) + )() + assert curl_result["http_status"] == 201, curl_result + + try: + curl_result = curl( + f"-L --negotiate -u : -X GET '{webhdfs_gateway_url}/user/{user}/{distant_hdfs_path}?op=OPEN'" + ) + assert file_content in curl_result["command"].stdout, curl_result + + curl_result = curl( + f"-L --negotiate -u : -X GET '{webhdfs_gateway_url}/user/{user}/{distant_hdfs_path}?op=LISTSTATUS'" + ) + assert curl_result["http_status"] == 200, curl_result + + liststatus = json.loads(curl_result["command"].stdout) + assert "FileStatuses" in liststatus, curl_result + assert "FileStatus" in liststatus["FileStatuses"], curl_result + assert len(liststatus["FileStatuses"]["FileStatus"]) == 1, curl_result + + filestatus = liststatus["FileStatuses"]["FileStatus"][0] + assert filestatus["group"] == user, curl_result + assert filestatus["owner"] == user, curl_result + assert filestatus["type"] == "FILE", curl_result + finally: + curl_result = curl( + f"-L --negotiate -u : -X DELETE '{webhdfs_gateway_url}/user/{user}/{distant_hdfs_path}?op=DELETE'" + ) + assert curl_result["http_status"] == 200, curl_result + + curl_result = curl( + f"-L --negotiate -u : -X GET '{webhdfs_gateway_url}/user/{user}/{distant_hdfs_path}?op=LISTSTATUS'", + check_status_code=False, + ) + liststatus = json.loads(curl_result["command"].stdout) + assert "RemoteException" in liststatus, curl_result + + assert "exception" in liststatus["RemoteException"], curl_result + assert ( + "FileNotFoundException" in liststatus["RemoteException"]["exception"] + ), curl_result + + assert "message" in liststatus["RemoteException"], curl_result + assert ( + f"File /user/{user}/{distant_hdfs_path} does not exist." + in liststatus["RemoteException"]["message"] + ), curl_result