tests: minimize curl utility usage inside tests

Signed-off-by: Evgeniy Zayats <[email protected]>

pytest_tests/lib/helpers/http_gate.py

@@ -143,7 +143,13 @@ def get_via_http_gate_by_attribute(
 
 @allure.step("Upload via HTTP Gate")
 def upload_via_http_gate(
-    cid: str, path: str, endpoint: str, headers: dict = None, file_content_type: str = None
+    cid: str,
+    path: str,
+    endpoint: str,
+    headers: dict = None,
+    cookies: dict = None,
+    file_content_type: str = None,
+    error_pattern: Optional[str] = None,
 ) -> str:
     """
     This function upload given object through HTTP gate
@@ -159,9 +165,13 @@ def upload_via_http_gate(
     else:
         files = {"upload_file": (path, open(path, "rb"), file_content_type)}
     body = {"filename": path}
-    resp = requests.post(request, files=files, data=body, headers=headers)
+    resp = requests.post(request, files=files, data=body, headers=headers, cookies=cookies)
 
     if not resp.ok:
+        if error_pattern:
+            match = error_pattern.casefold() in str(resp.text).casefold()
+            assert match, f"Expected {resp.text} to match {error_pattern}"
+            return ""
         raise Exception(
             f"""Failed to get object via HTTP gate:
                 request: {resp.request.path_url},
@@ -198,7 +208,6 @@ def upload_via_http_gate_curl(
     endpoint: str,
     headers: list = None,
     error_pattern: Optional[str] = None,
-    cookies: dict = None,
 ) -> str:
     """
     This function upload given object through HTTP gate using curl utility.
@@ -214,24 +223,9 @@ def upload_via_http_gate_curl(
         # parse attributes
         attributes = " ".join(headers)
 
-    cookies_attr = ""
-    if cookies:
-        for k, v in cookies.items():
-            cookies_attr += f" -b '{k}={v}'"
-
-    large_object = is_object_large(filepath)
-    if large_object:
-        # pre-clean
-        _cmd_run("rm pipe -f || true")
-        files = f"file=@pipe;filename={os.path.basename(filepath)}"
-        cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --silent --no-buffer -F '{files}' {attributes}{cookies_attr} {request}"
-        output = _cmd_run(cmd, LONG_TIMEOUT)
-        # clean up pipe
-        _cmd_run("rm pipe")
-    else:
-        files = f"file=@{filepath};filename={os.path.basename(filepath)}"
-        cmd = f"curl --silent -F '{files}' {attributes}{cookies_attr} {request}"
-        output = _cmd_run(cmd)
+    files = f"file=@{filepath};filename={os.path.basename(filepath)}"
+    cmd = f"curl --silent -F '{files}' {attributes} {request}"
+    output = _cmd_run(cmd)
 
     if error_pattern:
         match = error_pattern.casefold() in str(output).casefold()
@@ -247,23 +241,6 @@ def upload_via_http_gate_curl(
     return response_json["object_id"]
 
 
-@allure.step("Get via HTTP Gate using Curl")
-def get_via_http_curl(cid: str, oid: str, endpoint: str) -> str:
-    """
-    This function gets given object from HTTP gate using curl utility.
-    cid:      CID to get object from
-    oid:      object OID
-    endpoint: http gate endpoint
-    """
-    request = f"{endpoint}/get/{cid}/{oid}"
-    file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
-
-    cmd = f"curl {request} > {file_path}"
-    _cmd_run(cmd)
-
-    return file_path
-
-
 def _attach_allure_step(request: str, status_code: int, req_type="GET"):
     command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n"
     with allure.step(f"{req_type} Request"):
@@ -344,6 +321,11 @@ def attr_into_header(attrs: dict) -> dict:
     return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}
 
 
+@allure.step("Convert each attribute (Key=Value) to the following format: 'X-Attribute-Key: Value'")
+def attr_into_str_header(attrs: dict) -> list:
+    return {f"X-Attribute-{k}": {v} for k, v in attrs.items()}
+
+
 @allure.step(
     "Convert each attribute (Key=Value) to the following format: -H 'X-Attribute-Key: Value'"
 )

pytest_tests/tests/services/rest_gate/test_rest_bearer.py

@@ -16,7 +16,7 @@
 )
 from helpers.container import create_container
 from helpers.file_helper import generate_file
-from helpers.http_gate import upload_via_http_gate_curl
+from helpers.http_gate import upload_via_http_gate
 from helpers.wellknown_acl import PUBLIC_ACL
 from http_gw.http_utils import get_object_and_verify_hashes
 from neofs_env.neofs_env_test_base import NeofsEnvTestBase
@@ -88,9 +88,9 @@ def test_unable_put_without_bearer_token(
         self, simple_object_size: int, user_container: str, eacl_deny_for_others, gw_endpoint
     ):
         eacl_deny_for_others
-        upload_via_http_gate_curl(
+        upload_via_http_gate(
             cid=user_container,
-            filepath=generate_file(simple_object_size),
+            path=generate_file(simple_object_size),
             endpoint=gw_endpoint,
             error_pattern="access to object operation denied",
         )
@@ -119,13 +119,13 @@ def test_put_with_bearer_when_eacl_restrict(
             headers = None
             cookies = None
             if bearer_type == "header":
-                headers = [f" -H 'Authorization: Bearer {bearer}'"]
+                headers = {"Authorization": f"Bearer {bearer}"}
             if bearer_type == "cookie":
                 cookies = {"Bearer": bearer}
 
-            oid = upload_via_http_gate_curl(
+            oid = upload_via_http_gate(
                 cid=user_container,
-                filepath=file_path,
+                path=file_path,
                 endpoint=gw_endpoint,
                 headers=headers,
                 cookies=cookies,

pytest_tests/tests/services/rest_gate/test_rest_gate.py

@@ -6,16 +6,13 @@
 import neofs_env.neofs_epoch as neofs_epoch
 import pytest
 from helpers.container import create_container
-from helpers.file_helper import generate_file, generate_file_with_content, get_file_hash
+from helpers.file_helper import generate_file, generate_file_with_content
 from helpers.http_gate import (
     attr_into_header,
     get_object_by_attr_and_verify_hashes,
-    get_via_http_curl,
     get_via_http_gate,
-    get_via_zip_http_gate,
     try_to_get_object_and_expect_error,
     upload_via_http_gate,
-    upload_via_http_gate_curl,
 )
 from helpers.neofs_verbs import put_object_to_random_node
 from helpers.utility import wait_for_gc_pass_on_storage_nodes
@@ -443,7 +440,7 @@ def test_expiration_epoch_in_http(self, simple_object_size, gw_params):
                     get_via_http_gate(cid=cid, oid=oid, endpoint=gw_params["endpoint"])
 
     @pytest.mark.long
-    @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object")
+    @allure.title("Test Put over HTTP, Get over HTTP for large object")
     def test_put_http_get_http_large_file(self, complex_object_size, gw_params):
         """
         This test checks upload and download using curl with 'large' object.
@@ -462,11 +459,6 @@ def test_put_http_get_http_large_file(self, complex_object_size, gw_params):
 
         with allure.step("Put objects using HTTP"):
             oid_gate = upload_via_http_gate(cid=cid, path=file_path, endpoint=gw_params["endpoint"])
-            oid_curl = upload_via_http_gate_curl(
-                cid=cid,
-                filepath=file_path,
-                endpoint=gw_params["endpoint"],
-            )
 
         get_object_and_verify_hashes(
             oid=oid_gate,
@@ -477,53 +469,3 @@ def test_put_http_get_http_large_file(self, complex_object_size, gw_params):
             nodes=self.neofs_env.storage_nodes,
             endpoint=gw_params["endpoint"],
         )
-        get_object_and_verify_hashes(
-            oid=oid_curl,
-            file_name=file_path,
-            wallet=self.wallet.path,
-            cid=cid,
-            shell=self.shell,
-            nodes=self.neofs_env.storage_nodes,
-            endpoint=gw_params["endpoint"],
-            object_getter=get_via_http_curl,
-        )
-
-    @allure.title("Test Put/Get over HTTP using Curl utility")
-    def test_put_http_get_http_curl(self, complex_object_size, simple_object_size, gw_params):
-        """
-        Test checks upload and download over HTTP using curl utility.
-        """
-        cid = create_container(
-            self.wallet.path,
-            shell=self.shell,
-            endpoint=self.neofs_env.sn_rpc,
-            rule=self.PLACEMENT_RULE_2,
-            basic_acl=PUBLIC_ACL,
-        )
-        file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
-            complex_object_size
-        )
-
-        with allure.step("Put objects using curl utility"):
-            oid_simple = upload_via_http_gate_curl(
-                cid=cid,
-                filepath=file_path_simple,
-                endpoint=gw_params["endpoint"],
-            )
-            oid_large = upload_via_http_gate_curl(
-                cid=cid,
-                filepath=file_path_large,
-                endpoint=gw_params["endpoint"],
-            )
-
-        for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
-            get_object_and_verify_hashes(
-                oid=oid,
-                file_name=file_path,
-                wallet=self.wallet.path,
-                cid=cid,
-                shell=self.shell,
-                nodes=self.neofs_env.storage_nodes,
-                endpoint=gw_params["endpoint"],
-                object_getter=get_via_http_curl,
-            )

pytest_tests/tests/services/rest_gate/test_rest_headers.py

@@ -11,10 +11,12 @@
 )
 from helpers.file_helper import generate_file
 from helpers.http_gate import (
+    attr_into_str_header,
     attr_into_str_header_curl,
     get_object_by_attr_and_verify_hashes,
     try_to_get_object_and_expect_error,
     try_to_get_object_via_passed_request_and_expect_error,
+    upload_via_http_gate,
     upload_via_http_gate_curl,
 )
 from helpers.neofs_verbs import delete_object
@@ -65,11 +67,11 @@ def storage_objects_with_attributes(
         )
         file_path = generate_file(request.param)
         for attributes in self.OBJECT_ATTRIBUTES:
-            storage_object_id = upload_via_http_gate_curl(
+            storage_object_id = upload_via_http_gate(
                 cid=cid,
-                filepath=file_path,
+                path=file_path,
                 endpoint=gw_endpoint,
-                headers=attr_into_str_header_curl(attributes),
+                headers=attr_into_str_header(attributes),
             )
             storage_object = StorageObjectInfo(cid, storage_object_id)
             storage_object.size = os.path.getsize(file_path)
@@ -167,9 +169,7 @@ def test_object2_can_be_get_by_attr(
             )
 
     @allure.title("[Negative] Try to put object and get right after container is deleted")
-    def test_negative_put_and_get_object3(
-        self, storage_objects_with_attributes: list[StorageObjectInfo], gw_endpoint
-    ):
+    def test_negative_put_and_get_object3(self, simple_object_size, gw_endpoint):
         """
         Test to attempt to put object and try to download it right after the container has been deleted
 
@@ -180,18 +180,31 @@ def test_negative_put_and_get_object3(
         3. [Negative] Try to download object with attributes [peace=peace]
             Expected: "HTTP request sent, awaiting response... 404 Not Found"
         """
-        storage_object_1 = storage_objects_with_attributes[0]
+        cid = create_container(
+            wallet=self.wallet.path,
+            shell=self.shell,
+            endpoint=self.neofs_env.sn_rpc,
+            rule=self.PLACEMENT_RULE,
+            basic_acl=PUBLIC_ACL,
+        )
+        file_path = generate_file(simple_object_size)
+        upload_via_http_gate(
+            cid=cid,
+            path=file_path,
+            endpoint=gw_endpoint,
+            headers=attr_into_str_header(self.OBJECT_ATTRIBUTES[0]),
+        )
 
         with allure.step(
             "[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]"
         ):
-            file_path_3 = generate_file(storage_object_1.size)
+            file_path_3 = generate_file(simple_object_size)
             attrs_obj3 = {"Writer": "Leo Tolstoy", "peace": "peace"}
             headers = attr_into_str_header_curl(attrs_obj3)
             headers.append(" ".join(attr_into_str_header_curl({"Writer": "peace"})))
             error_pattern = f"key duplication error: X-Attribute-Writer"
             upload_via_http_gate_curl(
-                cid=storage_object_1.cid,
+                cid=cid,
                 filepath=file_path_3,
                 endpoint=gw_endpoint,
                 headers=headers,
@@ -200,26 +213,26 @@ def test_negative_put_and_get_object3(
         with allure.step("Delete container and verify container deletion"):
             delete_container(
                 wallet=self.wallet.path,
-                cid=storage_object_1.cid,
+                cid=cid,
                 shell=self.shell,
                 endpoint=self.neofs_env.sn_rpc,
             )
             self.tick_epochs_and_wait(1)
             wait_for_container_deletion(
                 self.wallet.path,
-                storage_object_1.cid,
+                cid,
                 shell=self.shell,
                 endpoint=self.neofs_env.sn_rpc,
             )
-            assert storage_object_1.cid not in list_containers(
+            assert cid not in list_containers(
                 self.wallet.path, shell=self.shell, endpoint=self.neofs_env.sn_rpc
             )
         with allure.step(
             "[Negative] Try to download (wget) object via wget with attributes [peace=peace]"
         ):
             error_pattern = "404 Not Found"
             try_to_get_object_via_passed_request_and_expect_error(
-                cid=storage_object_1.cid,
+                cid=cid,
                 oid="",
                 error_pattern=error_pattern,
                 attrs=attrs_obj3,

pytest_tests/tests/services/rest_gate/test_rest_streaming.py

@@ -4,7 +4,7 @@
 import pytest
 from helpers.container import create_container
 from helpers.file_helper import generate_file
-from helpers.http_gate import upload_via_http_gate_curl
+from helpers.http_gate import upload_via_http_gate
 from helpers.wellknown_acl import PUBLIC_ACL
 from http_gw.http_utils import get_object_and_verify_hashes
 from neofs_env.neofs_env_test_base import NeofsEnvTestBase
@@ -53,10 +53,8 @@ def test_object_can_be_put_get_by_streaming(self, object_size: int, gw_endpoint)
             # Generate file
             file_path = generate_file(object_size)
 
-        with allure.step(
-            "Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"
-        ):
-            oid = upload_via_http_gate_curl(cid=cid, filepath=file_path, endpoint=gw_endpoint)
+        with allure.step("Put objects and Get object and verify hashes [ get/$CID/$OID ]"):
+            oid = upload_via_http_gate(cid=cid, path=file_path, endpoint=gw_endpoint)
             get_object_and_verify_hashes(
                 oid=oid,
                 file_name=file_path,