env enhancements (#802)

neofs-testlib/neofs_testlib/env/env.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 import logging
 import os
@@ -9,6 +10,7 @@
 import subprocess
 import threading
 import time
+from collections import namedtuple
 from dataclasses import dataclass
 from enum import Enum
 from importlib.resources import files
@@ -41,6 +43,7 @@ class WalletType(Enum):
 
 
 class NeoFSEnv:
+    _id = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
     _busy_ports = []
 
     def __init__(self, neofs_env_config: dict = None):
@@ -364,19 +367,19 @@ def download_binary(repo: str, version: str, file: str, target: str):
         current_perm = os.stat(target)
         os.chmod(target, current_perm.st_mode | stat.S_IEXEC)
 
-    @staticmethod
-    def _generate_temp_file(extension: str = "", prefix: str = "tmp_file") -> str:
-        file_path = f"env_files/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}"
+    @classmethod
+    def _generate_temp_file(cls, extension: str = "", prefix: str = "tmp_file") -> str:
+        file_path = f"env_files/neofs-env-{cls._id}/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}"
         if extension:
             file_path += f".{extension}"
         file_path = Path(file_path)
         file_path.parent.mkdir(parents=True, exist_ok=True)
         file_path.touch()
         return file_path
 
-    @staticmethod
-    def _generate_temp_dir(prefix: str = "tmp_dir") -> str:
-        dir_path = f"env_files/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}"
+    @classmethod
+    def _generate_temp_dir(cls, prefix: str = "tmp_dir") -> str:
+        dir_path = f"env_files/neofs-env-{cls._id}/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}"
         Path(dir_path).mkdir(parents=True, exist_ok=True)
         return dir_path
 
@@ -695,6 +698,12 @@ def _generate_config(self):
 
         s3_config_template = "s3.yaml"
 
+        S3peer = namedtuple("S3peer", ["address", "priority", "weight"])
+
+        peers = []
+        for sn in self.neofs_env.storage_nodes:
+            peers.append(S3peer(sn.endpoint, 1, 1))
+
         NeoFSEnv.generate_config_file(
             config_template=s3_config_template,
             config_path=self.config_path,
@@ -704,27 +713,21 @@ def _generate_config(self):
             key_file_path=self.tls_key_path,
             wallet=self.wallet,
             morph_endpoint=self.neofs_env.morph_rpc,
+            peers=peers,
+            tree_service_endpoint=self.neofs_env.storage_nodes[0].endpoint,
+            listen_domain=self.neofs_env.domain,
         )
 
     def _launch_process(self):
         self.stdout = NeoFSEnv._generate_temp_file(prefix="s3gw_stdout")
         self.stderr = NeoFSEnv._generate_temp_file(prefix="s3gw_stderr")
         stdout_fp = open(self.stdout, "w")
         stderr_fp = open(self.stderr, "w")
-        s3_gw_env = {
-            "S3_GW_LISTEN_DOMAINS": self.neofs_env.domain,
-            "S3_GW_TREE_SERVICE": self.neofs_env.storage_nodes[0].endpoint,
-        }
-
-        for index, sn in enumerate(self.neofs_env.storage_nodes):
-            s3_gw_env[f"S3_GW_PEERS_{index}_ADDRESS"] = sn.endpoint
-            s3_gw_env[f"S3_GW_PEERS_{index}_WEIGHT"] = "0.2"
 
         self.process = subprocess.Popen(
             [self.neofs_env.neofs_s3_gw_path, "--config", self.config_path],
             stdout=stdout_fp,
             stderr=stderr_fp,
-            env=s3_gw_env,
         )
 
 

neofs-testlib/neofs_testlib/env/templates/neofs_env_config.yaml

@@ -1,50 +1,45 @@
 binaries:
     neofs_cli:
         repo: 'nspcc-dev/neofs-node'
-        version: 'v0.39.2'
-        file: 'neofs-cli-amd64'
+        version: 'v0.42.0'
+        file: 'neofs-cli-linux-amd64'
 
     neofs_adm:
         repo: 'nspcc-dev/neofs-node'
-        version: 'v0.39.2'
-        file: 'neofs-adm-amd64'
+        version: 'v0.42.0'
+        file: 'neofs-adm-linux-amd64'
 
     neofs_ir:
         repo: 'nspcc-dev/neofs-node'
-        version: 'v0.39.2'
-        file: 'neofs-ir-amd64'
+        version: 'v0.42.0'
+        file: 'neofs-ir-linux-amd64'
 
     neofs_lens:
         repo: 'nspcc-dev/neofs-node'
-        version: 'v0.39.2'
-        file: 'neofs-lens-amd64'
+        version: 'v0.42.0'
+        file: 'neofs-lens-linux-amd64'
 
     neofs_node:
         repo: 'nspcc-dev/neofs-node'
-        version: 'v0.39.2'
-        file: 'neofs-node-amd64'
+        version: 'v0.42.0'
+        file: 'neofs-node-linux-amd64'
 
     neofs_s3_gw:
         repo: 'nspcc-dev/neofs-s3-gw'
-        version: 'v0.29.0'
+        version: 'v0.30.0'
         file: 'neofs-s3-gw-linux-amd64'
 
     neofs_s3_authmate:
         repo: 'nspcc-dev/neofs-s3-gw'
-        version: 'v0.29.0'
+        version: 'v0.30.0'
         file: 'neofs-s3-authmate-linux-amd64'
 
     neofs_rest_gw:
         repo: 'nspcc-dev/neofs-rest-gw'
-        version: 'v0.6.0'
+        version: 'v0.8.3'
         file: 'neofs-rest-gw-linux-amd64'
 
-    neofs_http_gw:
-        repo: 'nspcc-dev/neofs-http-gw'
-        version: 'v0.28.0'
-        file: 'neofs-http-gw-linux-amd64'
-
     neo_go:
         repo: 'nspcc-dev/neo-go'
-        version: 'v0.104.0'
+        version: 'v0.106.0'
         file: 'neo-go-linux-amd64'

neofs-testlib/neofs_testlib/env/templates/s3.yaml

@@ -29,4 +29,18 @@ wallet:
   passphrase: {{ wallet.password }} # Passphrase to decrypt wallet
 
 # RPC endpoint
-rpc_endpoint: http://{{ morph_endpoint }}
+rpc_endpoint: http://{{ morph_endpoint }}
+
+peers:
+  {%- for peer in peers %}
+  {{ loop.index0 }}:
+    address: {{ peer.address }}
+    priority: {{ peer.priority }}
+    weight: {{ peer.weight }}
+  {%- endfor %}
+
+tree:
+  service: {{ tree_service_endpoint }}
+
+listen_domains:
+  - {{ listen_domain }}