tests: fix issue with out of space during logs attachment

Signed-off-by: Evgeniy Zayats <[email protected]>

neofs-testlib/neofs_testlib/env/env.py

@@ -1,4 +1,5 @@
 import datetime
+import fcntl
 import json
 import logging
 import os
@@ -26,14 +27,14 @@
 import jinja2
 import requests
 import yaml
-from helpers.common import get_assets_dir_path
-
+from helpers.common import ALLOCATED_PORTS_FILE, ALLOCATED_PORTS_LOCK_FILE, get_assets_dir_path
 from neofs_testlib.cli import NeofsAdm, NeofsCli, NeofsLens, NeoGo
 from neofs_testlib.shell import LocalShell
 from neofs_testlib.utils import wallet as wallet_utils
 from tenacity import retry, stop_after_attempt, wait_fixed
 
 logger = logging.getLogger("neofs.testlib.env")
+_thread_lock = threading.Lock()
 
 
 @dataclass
@@ -51,8 +52,6 @@ class WalletType(Enum):
 
 
 class NeoFSEnv:
-    _busy_ports = []
-
     def __init__(self, neofs_env_config: dict = None):
         self._id = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d-%H-%M-%S")
         self._env_dir = f"{get_assets_dir_path()}/env_files/neofs-env-{self._id}"
@@ -155,12 +154,13 @@ def deploy_inner_ring_nodes(self, count=1, with_main_chain=False):
             ir_node.start(wait_until_ready=False, with_main_chain=with_main_chain)
 
         with allure.step("Wait until all IR nodes are READY"):
-            for ir_node in self.inner_ring_nodes:
+            for ir_node_idx, ir_node in enumerate(self.inner_ring_nodes):
                 logger.info(f"Wait until IR: {ir_node} is READY")
                 try:
                     ir_node._wait_until_ready()
                 except Exception as e:
-                    allure.attach.file(ir_node.stderr, name="ir node logs", extension="txt")
+                    allure.attach.file(ir_node.stderr, name=f"ir{ir_node_idx} node stderr", extension="txt")
+                    allure.attach.file(ir_node.stdout, name=f"ir{ir_node_idx} node stdout", extension="txt")
                     raise e
 
     @allure.step("Deploy storage node")
@@ -177,7 +177,13 @@ def deploy_storage_nodes(self, count=1, node_attrs: Optional[dict] = None):
         for t in deploy_threads:
             t.start()
         logger.info("Wait until storage nodes are deployed")
-        self._wait_until_all_storage_nodes_are_ready()
+        try:
+            self._wait_until_all_storage_nodes_are_ready()
+        except Exception as e:
+            for sn in self.storage_nodes:
+                allure.attach.file(sn.stderr, name=f"sn{sn.sn_number} stderr", extension="txt")
+                allure.attach.file(sn.stdout, name=f"sn{sn.sn_number} stdout", extension="txt")
+            raise e
         # tick epoch to speed up storage nodes bootstrap
         self.neofs_adm().morph.force_new_epoch(
             rpc_endpoint=f"http://{self.morph_rpc}",
@@ -327,7 +333,7 @@ def kill(self):
             ir.process.kill()
 
     def persist(self) -> str:
-        persisted_path = self._generate_temp_file(self._env_dir, prefix="persisted_env")
+        persisted_path = self._generate_temp_file(os.path.dirname(self._env_dir), prefix="persisted_env")
         with open(persisted_path, "wb") as fp:
             pickle.dump(self, fp)
         logger.info(f"Persist env at: {persisted_path}")
@@ -501,17 +507,69 @@ def _run_single_command(binary: str, command: str) -> str:
         result = subprocess.run([binary, command], capture_output=True, text=True)
         return f"{result.stdout}\n{result.stderr}\n"
 
-    @classmethod
-    def get_available_port(cls) -> str:
-        for _ in range(len(cls._busy_ports) + 2):
-            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            s.bind(("", 0))
-            addr = s.getsockname()
+    @staticmethod
+    def get_available_port() -> str:
+        with _thread_lock:
+            with open(ALLOCATED_PORTS_LOCK_FILE, "w") as lock_file:
+                fcntl.flock(lock_file, fcntl.LOCK_EX)
+                try:
+                    if os.path.exists(ALLOCATED_PORTS_FILE):
+                        with open(ALLOCATED_PORTS_FILE, "r") as f:
+                            reserved_ports = set(map(int, f.read().splitlines()))
+                    else:
+                        reserved_ports = set()
+
+                    while True:
+                        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                        s.bind(("", 0))
+                        port = s.getsockname()[1]
+                        s.close()
+
+                        if port not in reserved_ports:
+                            reserved_ports.add(port)
+                            break
+
+                    with open(ALLOCATED_PORTS_FILE, "w") as f:
+                        for p in reserved_ports:
+                            f.write(f"{p}\n")
+                finally:
+                    fcntl.flock(lock_file, fcntl.LOCK_UN)
+        return port
+
+    @staticmethod
+    def cleanup_unused_ports():
+        with open(ALLOCATED_PORTS_LOCK_FILE, "w") as lock_file:
+            fcntl.flock(lock_file, fcntl.LOCK_EX)
+
+            try:
+                if os.path.exists(ALLOCATED_PORTS_FILE):
+                    with open(ALLOCATED_PORTS_FILE, "r") as f:
+                        reserved_ports = set(map(int, f.read().splitlines()))
+                else:
+                    reserved_ports = set()
+
+                still_used_ports = set()
+
+                for port in reserved_ports:
+                    if NeoFSEnv.is_port_in_use(port):
+                        still_used_ports.add(port)
+
+                with open(ALLOCATED_PORTS_FILE, "w") as f:
+                    for port in still_used_ports:
+                        f.write(f"{port}\n")
+            finally:
+                fcntl.flock(lock_file, fcntl.LOCK_UN)
+
+    @staticmethod
+    def is_port_in_use(port: str):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.bind(("", port))
+        except OSError:
+            return True
+        finally:
             s.close()
-            if addr[1] not in cls._busy_ports:
-                cls._busy_ports.append(addr[1])
-                return addr[1]
-        raise AssertionError("Can not find an available port")
+        return False
 
     @staticmethod
     def download_binary(repo: str, version: str, file: str, target: str):

pytest_tests/lib/helpers/common.py

@@ -25,7 +25,6 @@
 NEOFS_CONTRACT = os.getenv("NEOFS_IR_CONTRACTS_NEOFS")
 
 TEST_RUN_DIR = f"test-run-{datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d-%H-%M-%S-%f")}"
-ASSETS_DIR = f"TemporaryDir-{datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d-%H-%M-%S-%f")}"
 TEST_FILES_DIR = os.getenv("TEST_FILES_DIR", "TestFilesDir")
 TEST_OBJECTS_DIR = os.getenv("TEST_OBJECTS_DIR", "TestObjectsDir")
 DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "neofs-dev-env"))
@@ -60,6 +59,9 @@
 
 ENDPOINT_INTERNAL0 = "endpoint_internal0"
 
+ALLOCATED_PORTS_LOCK_FILE = "/tmp/allocated_ports.lock"
+ALLOCATED_PORTS_FILE = "/tmp/allocated_ports.txt"
+
 # Generate wallet configs
 # TODO: we should move all info about wallet configs to fixtures
 WALLET_CONFIG = os.path.join(os.getcwd(), "wallet_config.yml")
@@ -68,4 +70,4 @@
 
 
 def get_assets_dir_path() -> str:
-    return os.path.join(os.getcwd(), TEST_RUN_DIR, ASSETS_DIR)
+    return os.path.join(os.getcwd(), TEST_RUN_DIR)

pytest_tests/tests/conftest.py

@@ -33,6 +33,7 @@ def pytest_addoption(parser):
 
 @pytest.fixture(scope="session")
 def neofs_env(temp_directory, artifacts_directory, request):
+    NeoFSEnv.cleanup_unused_ports()
     if request.config.getoption("--load-env"):
         neofs_env = NeoFSEnv.load(request.config.getoption("--load-env"))
     else:
@@ -46,18 +47,27 @@ def neofs_env(temp_directory, artifacts_directory, request):
         if not request.config.getoption("--load-env"):
             neofs_env.kill()
 
-    if request.session.testsfailed:
-        env_files_path = os.path.join(os.getcwd(), neofs_env._env_dir)
-        env_files_archived = shutil.make_archive(
-            os.path.join(get_assets_dir_path(), f"neofs_env_{neofs_env._id}"), "zip", env_files_path
+    if request.session.testsfailed and not request.config.getoption("--persist-env"):
+        for ir in neofs_env.inner_ring_nodes:
+            os.remove(ir.ir_storage_path)
+        for sn in neofs_env.storage_nodes:
+            for shard in sn.shards:
+                os.remove(shard.metabase_path)
+                os.remove(shard.blobovnicza_path)
+                shutil.rmtree(shard.fstree_path, ignore_errors=True)
+                os.remove(shard.pilorama_path)
+                os.remove(shard.wc_path)
+
+        shutil.make_archive(
+            os.path.join(get_assets_dir_path(), f"neofs_env_{neofs_env._id}"), "zip", neofs_env._env_dir
         )
-        allure.attach.file(env_files_archived, name="neofs env files", extension="zip")
-
-        temp_files_path = os.path.join(get_assets_dir_path())
-        temp_files_archived = shutil.make_archive(
-            os.path.join(get_assets_dir_path(), "temp_files"), "zip", temp_files_path
+        shutil.rmtree(neofs_env._env_dir, ignore_errors=True)
+        allure.attach.file(
+            os.path.join(get_assets_dir_path(), f"neofs_env_{neofs_env._id}.zip"),
+            name="neofs env files",
+            extension="zip",
         )
-        allure.attach.file(temp_files_archived, name="tests temp files", extension="zip")
+    NeoFSEnv.cleanup_unused_ports()
 
 
 @pytest.fixture(scope="session")
@@ -203,6 +213,7 @@ def datadir(tmpdir, request):
 
 @pytest.fixture
 def neofs_env_with_mainchain(request):
+    NeoFSEnv.cleanup_unused_ports()
     if request.config.getoption("--load-env"):
         neofs_env = NeoFSEnv.load(request.config.getoption("--load-env"))
     else:
@@ -235,6 +246,7 @@ def neofs_env_with_mainchain(request):
         allure.attach.file(logs_zip_file_path, name="neofs env with main chain files", extension="zip")
     logger.info(f"Cleaning up dir {neofs_env}")
     shutil.rmtree(os.path.join(get_assets_dir_path(), neofs_env._env_dir), ignore_errors=True)
+    NeoFSEnv.cleanup_unused_ports()
 
 
 @pytest.fixture(scope="module", autouse=True)

pytest_tests/tests/failovers/test_failover_network.py

@@ -47,6 +47,7 @@ def test_block_storage_node_traffic(self, default_wallet, simple_object_size, re
         """
         Block storage nodes traffic using iptables and wait for replication for objects.
         """
+        raise AssertionError("Invoke allure attachments")
         wallet = default_wallet
         placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
         wakeup_node_timeout = 10  # timeout to let nodes detect that traffic has blocked

pytest_tests/tests/s3/test_s3_locking.py

@@ -160,6 +160,7 @@ def test_s3_mode_governance(self, version_id, simple_object_size):
 
     @allure.title("Test S3: Checking if an Object Cannot Be Locked")
     def test_s3_legal_hold(self, version_id, simple_object_size):
+        raise AssertionError()
         file_path = generate_file(simple_object_size)
         file_name = object_key_from_file_path(file_path)
 

pytest_tests/tests/session_token/test_static_session_token_container.py

@@ -473,6 +473,7 @@ def test_static_session_token_container_set_eacl_only_trusted_party_proved_by_th
         temp_directory: str,
         not_owner_wallet,
     ):
+        raise AssertionError()
         with allure.step("Create container"):
             cid = create_container(
                 owner_wallet.path,