diff --git a/pytest_tests/lib/helpers/http_gate.py b/pytest_tests/lib/helpers/http_gate.py index 51074521d..01d00265b 100644 --- a/pytest_tests/lib/helpers/http_gate.py +++ b/pytest_tests/lib/helpers/http_gate.py @@ -143,7 +143,13 @@ def get_via_http_gate_by_attribute( @allure.step("Upload via HTTP Gate") def upload_via_http_gate( - cid: str, path: str, endpoint: str, headers: dict = None, file_content_type: str = None + cid: str, + path: str, + endpoint: str, + headers: dict = None, + cookies: dict = None, + file_content_type: str = None, + error_pattern: Optional[str] = None, ) -> str: """ This function upload given object through HTTP gate @@ -159,9 +165,13 @@ def upload_via_http_gate( else: files = {"upload_file": (path, open(path, "rb"), file_content_type)} body = {"filename": path} - resp = requests.post(request, files=files, data=body, headers=headers) + resp = requests.post(request, files=files, data=body, headers=headers, cookies=cookies) if not resp.ok: + if error_pattern: + match = error_pattern.casefold() in str(resp.text).casefold() + assert match, f"Expected {resp.text} to match {error_pattern}" + return "" raise Exception( f"""Failed to get object via HTTP gate: request: {resp.request.path_url}, @@ -198,7 +208,6 @@ def upload_via_http_gate_curl( endpoint: str, headers: list = None, error_pattern: Optional[str] = None, - cookies: dict = None, ) -> str: """ This function upload given object through HTTP gate using curl utility. @@ -214,24 +223,9 @@ def upload_via_http_gate_curl( # parse attributes attributes = " ".join(headers) - cookies_attr = "" - if cookies: - for k, v in cookies.items(): - cookies_attr += f" -b '{k}={v}'" - - large_object = is_object_large(filepath) - if large_object: - # pre-clean - _cmd_run("rm pipe -f || true") - files = f"file=@pipe;filename={os.path.basename(filepath)}" - cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --silent --no-buffer -F '{files}' {attributes}{cookies_attr} {request}" - output = _cmd_run(cmd, LONG_TIMEOUT) - # clean up pipe - _cmd_run("rm pipe") - else: - files = f"file=@{filepath};filename={os.path.basename(filepath)}" - cmd = f"curl --silent -F '{files}' {attributes}{cookies_attr} {request}" - output = _cmd_run(cmd) + files = f"file=@{filepath};filename={os.path.basename(filepath)}" + cmd = f"curl --silent -F '{files}' {attributes} {request}" + output = _cmd_run(cmd) if error_pattern: match = error_pattern.casefold() in str(output).casefold() @@ -247,23 +241,6 @@ def upload_via_http_gate_curl( return response_json["object_id"] -@allure.step("Get via HTTP Gate using Curl") -def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str: - """ - This function gets given object from HTTP gate using curl utility. - cid: CID to get object from - oid: object OID - endpoint: http gate endpoint - """ - request = f"{endpoint}/get/{cid}/{oid}" - file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}") - - cmd = f"curl {request} > {file_path}" - _cmd_run(cmd) - - return file_path - - def _attach_allure_step(request: str, status_code: int, req_type="GET"): command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n" with allure.step(f"{req_type} Request"): @@ -344,6 +321,11 @@ def attr_into_header(attrs: dict) -> dict: return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()} +@allure.step("Convert each attribute (Key=Value) to the following format: 'X-Attribute-Key: Value'") +def attr_into_str_header(attrs: dict) -> list: + return {f"X-Attribute-{k}": {v} for k, v in attrs.items()} + + @allure.step( "Convert each attribute (Key=Value) to the following format: -H 'X-Attribute-Key: Value'" ) diff --git a/pytest_tests/tests/services/rest_gate/test_rest_bearer.py b/pytest_tests/tests/services/rest_gate/test_rest_bearer.py index 12d4c6e8a..d78782c5f 100644 --- a/pytest_tests/tests/services/rest_gate/test_rest_bearer.py +++ b/pytest_tests/tests/services/rest_gate/test_rest_bearer.py @@ -16,7 +16,7 @@ ) from helpers.container import create_container from helpers.file_helper import generate_file -from helpers.http_gate import upload_via_http_gate_curl +from helpers.http_gate import upload_via_http_gate from helpers.wellknown_acl import PUBLIC_ACL from http_gw.http_utils import get_object_and_verify_hashes from neofs_env.neofs_env_test_base import NeofsEnvTestBase @@ -88,9 +88,9 @@ def test_unable_put_without_bearer_token( self, simple_object_size: int, user_container: str, eacl_deny_for_others, gw_endpoint ): eacl_deny_for_others - upload_via_http_gate_curl( + upload_via_http_gate( cid=user_container, - filepath=generate_file(simple_object_size), + path=generate_file(simple_object_size), endpoint=gw_endpoint, error_pattern="access to object operation denied", ) @@ -119,13 +119,13 @@ def test_put_with_bearer_when_eacl_restrict( headers = None cookies = None if bearer_type == "header": - headers = [f" -H 'Authorization: Bearer {bearer}'"] + headers = {"Authorization": f"Bearer {bearer}"} if bearer_type == "cookie": cookies = {"Bearer": bearer} - oid = upload_via_http_gate_curl( + oid = upload_via_http_gate( cid=user_container, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, headers=headers, cookies=cookies, diff --git a/pytest_tests/tests/services/rest_gate/test_rest_gate.py b/pytest_tests/tests/services/rest_gate/test_rest_gate.py index d0a23309a..e75b65d59 100644 --- a/pytest_tests/tests/services/rest_gate/test_rest_gate.py +++ b/pytest_tests/tests/services/rest_gate/test_rest_gate.py @@ -6,16 +6,13 @@ import neofs_env.neofs_epoch as neofs_epoch import pytest from helpers.container import create_container -from helpers.file_helper import generate_file, generate_file_with_content, get_file_hash +from helpers.file_helper import generate_file, generate_file_with_content from helpers.http_gate import ( attr_into_header, get_object_by_attr_and_verify_hashes, - get_via_http_curl, get_via_http_gate, - get_via_zip_http_gate, try_to_get_object_and_expect_error, upload_via_http_gate, - upload_via_http_gate_curl, ) from helpers.neofs_verbs import put_object_to_random_node from helpers.utility import wait_for_gc_pass_on_storage_nodes @@ -443,7 +440,7 @@ def test_expiration_epoch_in_http(self, simple_object_size, gw_params): get_via_http_gate(cid=cid, oid=oid, endpoint=gw_params["endpoint"]) @pytest.mark.long - @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object") + @allure.title("Test Put over HTTP, Get over HTTP for large object") def test_put_http_get_http_large_file(self, complex_object_size, gw_params): """ This test checks upload and download using curl with 'large' object. @@ -462,11 +459,6 @@ def test_put_http_get_http_large_file(self, complex_object_size, gw_params): with allure.step("Put objects using HTTP"): oid_gate = upload_via_http_gate(cid=cid, path=file_path, endpoint=gw_params["endpoint"]) - oid_curl = upload_via_http_gate_curl( - cid=cid, - filepath=file_path, - endpoint=gw_params["endpoint"], - ) get_object_and_verify_hashes( oid=oid_gate, @@ -477,53 +469,3 @@ def test_put_http_get_http_large_file(self, complex_object_size, gw_params): nodes=self.neofs_env.storage_nodes, endpoint=gw_params["endpoint"], ) - get_object_and_verify_hashes( - oid=oid_curl, - file_name=file_path, - wallet=self.wallet.path, - cid=cid, - shell=self.shell, - nodes=self.neofs_env.storage_nodes, - endpoint=gw_params["endpoint"], - object_getter=get_via_http_curl, - ) - - @allure.title("Test Put/Get over HTTP using Curl utility") - def test_put_http_get_http_curl(self, complex_object_size, simple_object_size, gw_params): - """ - Test checks upload and download over HTTP using curl utility. - """ - cid = create_container( - self.wallet.path, - shell=self.shell, - endpoint=self.neofs_env.sn_rpc, - rule=self.PLACEMENT_RULE_2, - basic_acl=PUBLIC_ACL, - ) - file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( - complex_object_size - ) - - with allure.step("Put objects using curl utility"): - oid_simple = upload_via_http_gate_curl( - cid=cid, - filepath=file_path_simple, - endpoint=gw_params["endpoint"], - ) - oid_large = upload_via_http_gate_curl( - cid=cid, - filepath=file_path_large, - endpoint=gw_params["endpoint"], - ) - - for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): - get_object_and_verify_hashes( - oid=oid, - file_name=file_path, - wallet=self.wallet.path, - cid=cid, - shell=self.shell, - nodes=self.neofs_env.storage_nodes, - endpoint=gw_params["endpoint"], - object_getter=get_via_http_curl, - ) diff --git a/pytest_tests/tests/services/rest_gate/test_rest_headers.py b/pytest_tests/tests/services/rest_gate/test_rest_headers.py index cf3e7072c..be90d92e6 100644 --- a/pytest_tests/tests/services/rest_gate/test_rest_headers.py +++ b/pytest_tests/tests/services/rest_gate/test_rest_headers.py @@ -11,10 +11,12 @@ ) from helpers.file_helper import generate_file from helpers.http_gate import ( + attr_into_str_header, attr_into_str_header_curl, get_object_by_attr_and_verify_hashes, try_to_get_object_and_expect_error, try_to_get_object_via_passed_request_and_expect_error, + upload_via_http_gate, upload_via_http_gate_curl, ) from helpers.neofs_verbs import delete_object @@ -65,11 +67,11 @@ def storage_objects_with_attributes( ) file_path = generate_file(request.param) for attributes in self.OBJECT_ATTRIBUTES: - storage_object_id = upload_via_http_gate_curl( + storage_object_id = upload_via_http_gate( cid=cid, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, - headers=attr_into_str_header_curl(attributes), + headers=attr_into_str_header(attributes), ) storage_object = StorageObjectInfo(cid, storage_object_id) storage_object.size = os.path.getsize(file_path) @@ -167,9 +169,7 @@ def test_object2_can_be_get_by_attr( ) @allure.title("[Negative] Try to put object and get right after container is deleted") - def test_negative_put_and_get_object3( - self, storage_objects_with_attributes: list[StorageObjectInfo], gw_endpoint - ): + def test_negative_put_and_get_object3(self, simple_object_size, gw_endpoint): """ Test to attempt to put object and try to download it right after the container has been deleted @@ -180,18 +180,31 @@ def test_negative_put_and_get_object3( 3. [Negative] Try to download object with attributes [peace=peace] Expected: "HTTP request sent, awaiting response... 404 Not Found" """ - storage_object_1 = storage_objects_with_attributes[0] + cid = create_container( + wallet=self.wallet.path, + shell=self.shell, + endpoint=self.neofs_env.sn_rpc, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(simple_object_size) + upload_via_http_gate( + cid=cid, + path=file_path, + endpoint=gw_endpoint, + headers=attr_into_str_header(self.OBJECT_ATTRIBUTES[0]), + ) with allure.step( "[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]" ): - file_path_3 = generate_file(storage_object_1.size) + file_path_3 = generate_file(simple_object_size) attrs_obj3 = {"Writer": "Leo Tolstoy", "peace": "peace"} headers = attr_into_str_header_curl(attrs_obj3) headers.append(" ".join(attr_into_str_header_curl({"Writer": "peace"}))) error_pattern = f"key duplication error: X-Attribute-Writer" upload_via_http_gate_curl( - cid=storage_object_1.cid, + cid=cid, filepath=file_path_3, endpoint=gw_endpoint, headers=headers, @@ -200,18 +213,18 @@ def test_negative_put_and_get_object3( with allure.step("Delete container and verify container deletion"): delete_container( wallet=self.wallet.path, - cid=storage_object_1.cid, + cid=cid, shell=self.shell, endpoint=self.neofs_env.sn_rpc, ) self.tick_epochs_and_wait(1) wait_for_container_deletion( self.wallet.path, - storage_object_1.cid, + cid, shell=self.shell, endpoint=self.neofs_env.sn_rpc, ) - assert storage_object_1.cid not in list_containers( + assert cid not in list_containers( self.wallet.path, shell=self.shell, endpoint=self.neofs_env.sn_rpc ) with allure.step( @@ -219,7 +232,7 @@ def test_negative_put_and_get_object3( ): error_pattern = "404 Not Found" try_to_get_object_via_passed_request_and_expect_error( - cid=storage_object_1.cid, + cid=cid, oid="", error_pattern=error_pattern, attrs=attrs_obj3, diff --git a/pytest_tests/tests/services/rest_gate/test_rest_streaming.py b/pytest_tests/tests/services/rest_gate/test_rest_streaming.py index ff90b157d..2c9f33843 100644 --- a/pytest_tests/tests/services/rest_gate/test_rest_streaming.py +++ b/pytest_tests/tests/services/rest_gate/test_rest_streaming.py @@ -4,7 +4,7 @@ import pytest from helpers.container import create_container from helpers.file_helper import generate_file -from helpers.http_gate import upload_via_http_gate_curl +from helpers.http_gate import upload_via_http_gate from helpers.wellknown_acl import PUBLIC_ACL from http_gw.http_utils import get_object_and_verify_hashes from neofs_env.neofs_env_test_base import NeofsEnvTestBase @@ -53,10 +53,8 @@ def test_object_can_be_put_get_by_streaming(self, object_size: int, gw_endpoint) # Generate file file_path = generate_file(object_size) - with allure.step( - "Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]" - ): - oid = upload_via_http_gate_curl(cid=cid, filepath=file_path, endpoint=gw_endpoint) + with allure.step("Put objects and Get object and verify hashes [ get/$CID/$OID ]"): + oid = upload_via_http_gate(cid=cid, path=file_path, endpoint=gw_endpoint) get_object_and_verify_hashes( oid=oid, file_name=file_path, diff --git a/pytest_tests/tests/services/rest_gate/test_rest_system_header.py b/pytest_tests/tests/services/rest_gate/test_rest_system_header.py index 2c3ad4844..0080e94b1 100644 --- a/pytest_tests/tests/services/rest_gate/test_rest_system_header.py +++ b/pytest_tests/tests/services/rest_gate/test_rest_system_header.py @@ -10,9 +10,9 @@ from helpers.file_helper import generate_file from helpers.grpc_responses import OBJECT_NOT_FOUND from helpers.http_gate import ( - attr_into_str_header_curl, + attr_into_str_header, try_to_get_object_and_expect_error, - upload_via_http_gate_curl, + upload_via_http_gate, ) from helpers.neofs_verbs import get_netmap_netinfo, get_object_from_random_node, head_object from helpers.wellknown_acl import PUBLIC_ACL @@ -113,11 +113,11 @@ def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) def oid_header_info_for_object( self, file_path: str, attributes: dict, user_container: str, gw_endpoint: str ): - oid = upload_via_http_gate_curl( + oid = upload_via_http_gate( cid=user_container, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, - headers=attr_into_str_header_curl(attributes), + headers=attr_into_str_header(attributes), ) get_object_and_verify_hashes( oid=oid, @@ -141,16 +141,16 @@ def oid_header_info_for_object( def test_unable_put_expired_epoch( self, user_container: str, simple_object_size: int, gw_endpoint ): - headers = attr_into_str_header_curl( + headers = attr_into_str_header( {"Neofs-Expiration-Epoch": str(neofs_epoch.get_epoch(self.neofs_env) - 1)} ) file_path = generate_file(simple_object_size) with allure.step( "Put object using HTTP with attribute Expiration-Epoch where epoch is expired" ): - upload_via_http_gate_curl( + upload_via_http_gate( cid=user_container, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, headers=headers, error_pattern="object has expired", @@ -160,14 +160,14 @@ def test_unable_put_expired_epoch( def test_unable_put_negative_duration( self, user_container: str, simple_object_size: int, gw_endpoint ): - headers = attr_into_str_header_curl({"Neofs-Expiration-Duration": "-1h"}) + headers = attr_into_str_header({"Neofs-Expiration-Duration": "-1h"}) file_path = generate_file(simple_object_size) with allure.step( "Put object using HTTP with attribute Neofs-Expiration-Duration where duration is negative" ): - upload_via_http_gate_curl( + upload_via_http_gate( cid=user_container, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, headers=headers, error_pattern=f"{EXPIRATION_DURATION_HEADER} must be positive", @@ -179,14 +179,14 @@ def test_unable_put_negative_duration( def test_unable_put_expired_timestamp( self, user_container: str, simple_object_size: int, gw_endpoint ): - headers = attr_into_str_header_curl({"Neofs-Expiration-Timestamp": "1635075727"}) + headers = attr_into_str_header({"Neofs-Expiration-Timestamp": "1635075727"}) file_path = generate_file(simple_object_size) with allure.step( "Put object using HTTP with attribute Neofs-Expiration-Timestamp where duration is in the past" ): - upload_via_http_gate_curl( + upload_via_http_gate( cid=user_container, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, headers=headers, error_pattern=f"{EXPIRATION_TIMESTAMP_HEADER} must be in the future", @@ -198,11 +198,11 @@ def test_unable_put_expired_timestamp( def test_unable_put_expired_rfc( self, user_container: str, simple_object_size: int, gw_endpoint ): - headers = attr_into_str_header_curl({"Neofs-Expiration-RFC3339": "2021-11-22T09:55:49Z"}) + headers = attr_into_str_header({"Neofs-Expiration-RFC3339": "2021-11-22T09:55:49Z"}) file_path = generate_file(simple_object_size) - upload_via_http_gate_curl( + upload_via_http_gate( cid=user_container, - filepath=file_path, + path=file_path, endpoint=gw_endpoint, headers=headers, error_pattern=f"{EXPIRATION_EXPIRATION_RFC} must be in the future",