Update some s3 tests to use dynamic env (#680)

This is an example of how we can move tests to use dynamic environment
instead of a static dev-env. In parallel we can refactor existing non
optimal/legacy code.

1. Separate directory for new tests and library code -
dynamic_env_pytest_tests
2. In the meantime we reuse some existing code located in pytest_tests/*
robot/*
3. During tests moving to a new dynamic env - we refactor existing
legacy code and move it to dynamic_env_pytest_tests/lib as well.
4. For new tests at dynamic_env_pytest_tests we create a separate
validation inside this repo

.github/workflows/run-tests.yml

@@ -0,0 +1,250 @@
+name: Run automated system tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+      - support/**
+    types: [opened, synchronize]
+    paths-ignore:
+      - '**/*.md'
+  release:
+    types:
+      - published
+  workflow_dispatch:
+    inputs:
+      neofs_testcases_ref:
+        description: 'neofs-testcases ref. Default ref - latest master. Examples: v0.36.0, 8fdcc6d7e798e6511be8806b81894622e72d7fdc, branch_name'
+        required: false
+        default: ''
+
+permissions: write-all
+
+jobs:
+  run_system_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 500
+    steps:
+      - name: Get the current date
+        id: date
+        run: echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
+
+      - name: Set RUN_ID
+        env:
+          TIMESTAMP: ${{ steps.date.outputs.timestamp }}
+        run: echo "RUN_ID=${{ github.run_number }}-$TIMESTAMP" >> $GITHUB_ENV
+
+      - name: Checkout neofs-testcases repository
+        uses: actions/checkout@v4
+        with:
+          path: neofs-testcases
+
+      - name: Download latest stable neofs-cli
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-node'
+          version: 'tags/v0.39.2'
+          file: 'neofs-cli-amd64'
+          target: 'neofs-testcases/neofs-cli'
+
+      - name: Download latest stable neofs-adm
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-node'
+          version: 'tags/v0.39.2'
+          file: 'neofs-adm-amd64'
+          target: 'neofs-testcases/neofs-adm'
+
+      - name: Download latest stable neofs-ir
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-node'
+          version: 'tags/v0.39.2'
+          file: 'neofs-ir-amd64'
+          target: 'neofs-testcases/neofs-ir'
+
+      - name: Download latest stable neofs-lens
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-node'
+          version: 'tags/v0.39.2'
+          file: 'neofs-lens-amd64'
+          target: 'neofs-testcases/neofs-lens'
+
+      - name: Download latest stable neofs-node
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-node'
+          version: 'tags/v0.39.2'
+          file: 'neofs-node-amd64'
+          target: 'neofs-testcases/neofs-node'
+
+      - name: Checkout neofs-s3-gw repository
+        uses: actions/checkout@v4
+        with:
+          repository: nspcc-dev/neofs-s3-gw
+          ref: '014a5bf493d00b8cce478e2c96c84ad9de8fe617'
+          path: neofs-s3-gw
+
+      - name: Download latest stable neofs-rest-gw
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-rest-gw'
+          version: 'tags/v0.6.0'
+          file: 'neofs-rest-gw-linux-amd64'
+          target: 'neofs-testcases/neofs-rest-gw'
+
+      - name: Download latest stable neofs-http-gw
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neofs-http-gw'
+          version: 'tags/v0.28.0'
+          file: 'neofs-http-gw-linux-amd64'
+          target: 'neofs-testcases/neofs-http-gw'
+
+      - name: Download latest stable neo-go
+        uses: dsaltares/[email protected]
+        with:
+          repo: 'nspcc-dev/neo-go'
+          version: 'tags/v0.104.0'
+          file: 'neo-go-linux-amd64'
+          target: 'neofs-testcases/neo-go'
+
+      - name: Chmod latest stable binaries
+        run: |
+          sudo chmod a+x neofs-cli
+          sudo chmod a+x neofs-adm
+          sudo chmod a+x neofs-ir
+          sudo chmod a+x neofs-lens
+          sudo chmod a+x neofs-node
+          sudo chmod a+x neofs-rest-gw
+          sudo chmod a+x neofs-http-gw
+          sudo chmod a+x neo-go
+        working-directory: neofs-testcases
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          cache: true
+          go-version: '1.20'
+      - run: go version
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10.11'
+      - run: python --version
+
+# Hashlib uses OpenSSL for ripemd160 and apparently OpenSSL disabled some older crypto algos around version 3.0
+# in November 2021. All the functions are still there but require manual enabling.
+# See https://github.com/openssl/openssl/issues/16994
+# But we use ripemd160 for tests.
+# For ripemd160 to be supported, we need the openssl configuration file to contain the following lines:
+# openssl_conf = openssl_init
+#
+# [openssl_init]
+# providers = provider_sect
+#
+# [provider_sect]
+# default = default_sect
+# legacy = legacy_sect
+#
+# [default_sect]
+# activate = 1
+#
+# [legacy_sect]
+# activate = 1
+      - name: Fix OpenSSL ripemd160
+        run: |
+          sudo python ./tools/src/openssl_config_fix.py
+        working-directory: neofs-testcases
+
+      - name: Build neofs-s3-gw
+        timeout-minutes: 5
+        run: |
+          make all
+          echo "$(pwd)/bin" >> $GITHUB_PATH
+        working-directory: neofs-s3-gw
+
+      - name: Copy binaries to testcases directory
+        timeout-minutes: 30
+        run: |
+          cp ${GITHUB_WORKSPACE}/neofs-s3-gw/bin/* .
+          echo "$(pwd)" >> $GITHUB_PATH
+        working-directory: neofs-testcases
+
+      - name: Prepare venv
+        timeout-minutes: 30
+        run: |
+          make venv.no-dev-env-pytest
+        working-directory: neofs-testcases
+
+      - name: Log environment
+        run: |
+          echo "Check free space"
+          df -h
+          echo "=========================================="
+
+          echo "Check neo-go version"
+          neo-go --version
+          echo "=========================================="
+
+          echo "Check neofs-s3-authmate version"
+          neofs-s3-authmate --version
+          echo "=========================================="
+
+          echo "Check neofs-s3-gw version"
+          echo "=========================================="
+          neofs-s3-gw --version
+          echo "=========================================="
+
+          echo "Check neofs-adm version"
+          neofs-adm --version
+          echo "=========================================="
+
+          echo "Check neofs-ir version"
+          neofs-ir --version
+          echo "=========================================="
+
+          echo "Check neofs-lens version"
+          neofs-lens --version
+          echo "=========================================="
+
+          echo "Check neofs-cli version"
+          neofs-cli --version
+          echo "=========================================="
+
+          echo "Check current dir"
+          ls -lah
+          echo "=========================================="
+        working-directory: neofs-testcases
+
+################################################################
+      - name: Run Sanity tests for pull requests
+        timeout-minutes: 120
+        if: github.event_name == 'pull_request'
+        run: |
+          source venv.no-dev-env-pytest/bin/activate && pytest -s --alluredir=${GITHUB_WORKSPACE}/allure-results dynamic_env_pytest_tests/tests
+        working-directory: neofs-testcases
+
+################################################################
+      - name: Generate Allure report
+        timeout-minutes: 60
+        uses: simple-elf/[email protected]
+        if: always()
+        id: allure-report
+        with:
+          keep_reports: 100000
+          allure_results: allure-results
+          allure_report: allure-report
+          allure_history: allure-history
+
+      - name: Archive allure report raw results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: allure-report-raw-results
+          path: allure-results

dynamic_env_pytest_tests/lib/neofs_env/neofs_env_test_base.py

@@ -0,0 +1,32 @@
+import allure
+import neofs_env.neofs_epoch as neofs_epoch
+import pytest
+from neofs_testlib.env.env import NeoFSEnv
+from neofs_testlib.shell import Shell
+
+
+class NeofsEnvTestBase:
+    shell: Shell
+    neofs_env: NeoFSEnv
+
+    @pytest.fixture(scope="session", autouse=True)
+    def fill_mandatory_dependencies(self, neofs_env: NeoFSEnv):
+        NeofsEnvTestBase.shell = neofs_env.shell
+        NeofsEnvTestBase.neofs_env = neofs_env
+        yield
+
+    def tick_epoch(self):
+        neofs_epoch.tick_epoch(self.neofs_env)
+
+    def get_epoch(self):
+        return neofs_epoch.get_epoch(self.neofs_env)
+
+    def ensure_fresh_epoch(self):
+        return neofs_epoch.ensure_fresh_epoch(self.neofs_env)
+
+    @allure.step("Tick epochs and wait for epoch alignment")
+    def tick_epochs_and_wait(self, epochs_to_tick: int):
+        current_epoch = self.get_epoch()
+        for _ in range(epochs_to_tick):
+            self.tick_epoch()
+            neofs_epoch.wait_for_epochs_align(self.neofs_env, current_epoch)

dynamic_env_pytest_tests/lib/neofs_env/neofs_epoch.py

@@ -0,0 +1,70 @@
+import logging
+from typing import Optional
+
+import allure
+from neofs_testlib.env.env import NeoFSEnv, StorageNode
+from test_control import wait_for_success
+
+logger = logging.getLogger("NeoLogger")
+
+
+@allure.step("Ensure fresh epoch")
+def ensure_fresh_epoch(neofs_env: NeoFSEnv, alive_node: Optional[StorageNode] = None) -> int:
+    # ensure new fresh epoch to avoid epoch switch during test session
+    alive_node = alive_node if alive_node else neofs_env.storage_nodes[0]
+    current_epoch = get_epoch(neofs_env, alive_node)
+    tick_epoch_and_wait(neofs_env, current_epoch, alive_node)
+    epoch = get_epoch(neofs_env, alive_node)
+    assert epoch > current_epoch, "Epoch wasn't ticked"
+    return epoch
+
+
+@allure.step("Wait for epochs align in whole cluster")
+@wait_for_success(60, 5)
+def wait_for_epochs_align(neofs_env: NeoFSEnv, epoch_number: Optional[int] = None) -> bool:
+    epochs = []
+    for node in neofs_env.storage_nodes:
+        current_epoch = get_epoch(neofs_env, node)
+        assert (
+            epoch_number is None or current_epoch > epoch_number
+        ), f"Epoch {current_epoch} wasn't ticked yet. Expected epoch > {epoch_number}"
+        epochs.append(current_epoch)
+    unique_epochs = list(set(epochs))
+    assert (
+        len(unique_epochs) == 1
+    ), f"unaligned epochs found,  {epochs}, count of unique epochs {len(unique_epochs)}"
+
+
+@allure.step("Get Epoch")
+def get_epoch(neofs_env: NeoFSEnv, alive_node: Optional[StorageNode] = None):
+    alive_node = alive_node if alive_node else neofs_env.storage_nodes[0]
+    cli = neofs_env.neofs_cli(alive_node.cli_config)
+    epoch = cli.netmap.epoch(alive_node.endpoint, alive_node.wallet.path)
+    return int(epoch.stdout)
+
+
+@allure.step("Tick Epoch")
+def tick_epoch(neofs_env: NeoFSEnv, alive_node: Optional[StorageNode] = None):
+    """
+    Tick epoch using neofs-adm or NeoGo if neofs-adm is not available (DevEnv)
+    Args:
+        neofs_env: neofs env instance under test
+        alive_node: node to send requests to (first node in cluster by default)
+    """
+
+    alive_node = alive_node if alive_node else neofs_env.storage_nodes[0]
+    neofs_env.neofs_adm().morph.force_new_epoch(
+        rpc_endpoint=f"http://{neofs_env.morph_rpc}",
+        alphabet_wallets=neofs_env.alphabet_wallets_dir,
+    )
+
+
+@allure.step("Tick Epoch and wait for epochs align")
+def tick_epoch_and_wait(
+    neofs_env: NeoFSEnv,
+    current_epoch: Optional[int] = None,
+    node: Optional[StorageNode] = None,
+):
+    current_epoch = current_epoch if current_epoch else get_epoch(neofs_env, node)
+    tick_epoch(neofs_env, node)
+    wait_for_epochs_align(neofs_env, current_epoch)