From 4eb6629dc1f8554675516ebf9a8f73241f05d115 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Tue, 28 May 2024 20:13:11 -0400 Subject: [PATCH 1/3] env: move s3 gw env variables to config file Signed-off-by: Evgeniy Zayats --- neofs-testlib/neofs_testlib/env/env.py | 19 ++++++++++--------- .../neofs_testlib/env/templates/s3.yaml | 16 +++++++++++++++- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/neofs-testlib/neofs_testlib/env/env.py b/neofs-testlib/neofs_testlib/env/env.py index cfebd84fa..cc1c32c2c 100644 --- a/neofs-testlib/neofs_testlib/env/env.py +++ b/neofs-testlib/neofs_testlib/env/env.py @@ -9,6 +9,7 @@ import subprocess import threading import time +from collections import namedtuple from dataclasses import dataclass from enum import Enum from importlib.resources import files @@ -695,6 +696,12 @@ def _generate_config(self): s3_config_template = "s3.yaml" + S3peer = namedtuple("S3peer", ["address", "priority", "weight"]) + + peers = [] + for sn in self.neofs_env.storage_nodes: + peers.append(S3peer(sn.endpoint, 1, 1)) + NeoFSEnv.generate_config_file( config_template=s3_config_template, config_path=self.config_path, @@ -704,6 +711,9 @@ def _generate_config(self): key_file_path=self.tls_key_path, wallet=self.wallet, morph_endpoint=self.neofs_env.morph_rpc, + peers=peers, + tree_service_endpoint=self.neofs_env.storage_nodes[0].endpoint, + listen_domain=self.neofs_env.domain, ) def _launch_process(self): @@ -711,20 +721,11 @@ def _launch_process(self): self.stderr = NeoFSEnv._generate_temp_file(prefix="s3gw_stderr") stdout_fp = open(self.stdout, "w") stderr_fp = open(self.stderr, "w") - s3_gw_env = { - "S3_GW_LISTEN_DOMAINS": self.neofs_env.domain, - "S3_GW_TREE_SERVICE": self.neofs_env.storage_nodes[0].endpoint, - } - - for index, sn in enumerate(self.neofs_env.storage_nodes): - s3_gw_env[f"S3_GW_PEERS_{index}_ADDRESS"] = sn.endpoint - s3_gw_env[f"S3_GW_PEERS_{index}_WEIGHT"] = "0.2" self.process = subprocess.Popen( [self.neofs_env.neofs_s3_gw_path, "--config", self.config_path], stdout=stdout_fp, stderr=stderr_fp, - env=s3_gw_env, ) diff --git a/neofs-testlib/neofs_testlib/env/templates/s3.yaml b/neofs-testlib/neofs_testlib/env/templates/s3.yaml index 7113fdfa8..352787c59 100644 --- a/neofs-testlib/neofs_testlib/env/templates/s3.yaml +++ b/neofs-testlib/neofs_testlib/env/templates/s3.yaml @@ -29,4 +29,18 @@ wallet: passphrase: {{ wallet.password }} # Passphrase to decrypt wallet # RPC endpoint -rpc_endpoint: http://{{ morph_endpoint }} \ No newline at end of file +rpc_endpoint: http://{{ morph_endpoint }} + +peers: + {%- for peer in peers %} + {{ loop.index0 }}: + address: {{ peer.address }} + priority: {{ peer.priority }} + weight: {{ peer.weight }} + {%- endfor %} + +tree: + service: {{ tree_service_endpoint }} + +listen_domains: + - {{ listen_domain }} From a5cfca7ca48196c31e09a056191faa26839e4120 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Tue, 28 May 2024 20:16:55 -0400 Subject: [PATCH 2/3] env: bump binary versions for local runs Signed-off-by: Evgeniy Zayats --- .../env/templates/neofs_env_config.yaml | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/neofs-testlib/neofs_testlib/env/templates/neofs_env_config.yaml b/neofs-testlib/neofs_testlib/env/templates/neofs_env_config.yaml index da5ff7d7a..5a6b2e9a9 100644 --- a/neofs-testlib/neofs_testlib/env/templates/neofs_env_config.yaml +++ b/neofs-testlib/neofs_testlib/env/templates/neofs_env_config.yaml @@ -1,50 +1,45 @@ binaries: neofs_cli: repo: 'nspcc-dev/neofs-node' - version: 'v0.39.2' - file: 'neofs-cli-amd64' + version: 'v0.42.0' + file: 'neofs-cli-linux-amd64' neofs_adm: repo: 'nspcc-dev/neofs-node' - version: 'v0.39.2' - file: 'neofs-adm-amd64' + version: 'v0.42.0' + file: 'neofs-adm-linux-amd64' neofs_ir: repo: 'nspcc-dev/neofs-node' - version: 'v0.39.2' - file: 'neofs-ir-amd64' + version: 'v0.42.0' + file: 'neofs-ir-linux-amd64' neofs_lens: repo: 'nspcc-dev/neofs-node' - version: 'v0.39.2' - file: 'neofs-lens-amd64' + version: 'v0.42.0' + file: 'neofs-lens-linux-amd64' neofs_node: repo: 'nspcc-dev/neofs-node' - version: 'v0.39.2' - file: 'neofs-node-amd64' + version: 'v0.42.0' + file: 'neofs-node-linux-amd64' neofs_s3_gw: repo: 'nspcc-dev/neofs-s3-gw' - version: 'v0.29.0' + version: 'v0.30.0' file: 'neofs-s3-gw-linux-amd64' neofs_s3_authmate: repo: 'nspcc-dev/neofs-s3-gw' - version: 'v0.29.0' + version: 'v0.30.0' file: 'neofs-s3-authmate-linux-amd64' neofs_rest_gw: repo: 'nspcc-dev/neofs-rest-gw' - version: 'v0.6.0' + version: 'v0.8.3' file: 'neofs-rest-gw-linux-amd64' - neofs_http_gw: - repo: 'nspcc-dev/neofs-http-gw' - version: 'v0.28.0' - file: 'neofs-http-gw-linux-amd64' - neo_go: repo: 'nspcc-dev/neo-go' - version: 'v0.104.0' + version: 'v0.106.0' file: 'neo-go-linux-amd64' From 496ecb1f19a672807d9b61b7b8a7631ed362d735 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Tue, 28 May 2024 20:37:08 -0400 Subject: [PATCH 3/3] env: add folder for env logs to handle multi envs Signed-off-by: Evgeniy Zayats --- neofs-testlib/neofs_testlib/env/env.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/neofs-testlib/neofs_testlib/env/env.py b/neofs-testlib/neofs_testlib/env/env.py index cc1c32c2c..06278a877 100644 --- a/neofs-testlib/neofs_testlib/env/env.py +++ b/neofs-testlib/neofs_testlib/env/env.py @@ -1,3 +1,4 @@ +import datetime import json import logging import os @@ -42,6 +43,7 @@ class WalletType(Enum): class NeoFSEnv: + _id = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") _busy_ports = [] def __init__(self, neofs_env_config: dict = None): @@ -365,9 +367,9 @@ def download_binary(repo: str, version: str, file: str, target: str): current_perm = os.stat(target) os.chmod(target, current_perm.st_mode | stat.S_IEXEC) - @staticmethod - def _generate_temp_file(extension: str = "", prefix: str = "tmp_file") -> str: - file_path = f"env_files/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}" + @classmethod + def _generate_temp_file(cls, extension: str = "", prefix: str = "tmp_file") -> str: + file_path = f"env_files/neofs-env-{cls._id}/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}" if extension: file_path += f".{extension}" file_path = Path(file_path) @@ -375,9 +377,9 @@ def _generate_temp_file(extension: str = "", prefix: str = "tmp_file") -> str: file_path.touch() return file_path - @staticmethod - def _generate_temp_dir(prefix: str = "tmp_dir") -> str: - dir_path = f"env_files/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}" + @classmethod + def _generate_temp_dir(cls, prefix: str = "tmp_dir") -> str: + dir_path = f"env_files/neofs-env-{cls._id}/{prefix}_{''.join(random.choices(string.ascii_lowercase, k=10))}" Path(dir_path).mkdir(parents=True, exist_ok=True) return dir_path