🧪 Add timeout to cleanup fcn (#129)

* update workflow before publishing python package

* fix dependency issue and bump version

* point to website in project description

* fix broken dependency

* improve doc

* add github token to download artifacts

* ensure only read-access @wvangeit

* yet another attempt at downloading artifacts

* make sure to use repo that ran the trigger wf

* another attempt at fixing

* change owner

* allow publishing to testpypi also when pr

* minor change

* revert minor (but breaking) change

* minor fix

* add debug messages

* another debug message

* hopefully the final version

* final fix

* minor fix

* move master and tag to individual jobs

* add debug messages

* dev->post

* add python script for determining semantic version

* minor changes

* minor changes

* improve error handling and add version file to artifacts

* check if release

* minor fix

* ensure to enter venv

* also when tagging

* source venv in publishin workflow

* ensure only master

* add script for testing 'pure' semver

* adapt workflows to new python script

* minor change

* attempt to evaluate expressions correctly

* several fixes to fix tests

* ensure repo is checked out in publish workflow

* several small fixes

* cleanup

* debug

* minor cleanup

* mionr changes

* add debug message

* minor change

* minor change

* yet another try

* minor change

* minor change

* minor change

* mionr change

* minor changes

* correct workflow run id

* cosmetic change

* avoid using gh

* change to a single job for publishing

* minor cleanup

* add retry to clean up

* improve clean up fcn

clients/python/test/e2e/ci/e2e/e2e/postprocess.py

@@ -1,5 +1,6 @@
 import configparser
 import warnings
+from datetime import timedelta
 from pathlib import Path
 from typing import Optional
 from urllib.parse import urlparse
@@ -8,6 +9,13 @@
 import pandas as pd
 import pytest
 import typer
+from pydantic import PositiveInt
+from tenacity import (
+    Retrying,
+    retry_if_exception_type,
+    stop_after_attempt,
+    stop_after_delay,
+)
 
 from ._models import Artifacts, ClientSettings, PytestIniFile, ServerSettings
 from ._utils import E2eExitCodes, E2eScriptFailure, handle_validation_error
@@ -188,7 +196,7 @@ def log_dir(pytest_ini: Optional[Path] = None):
 
 
 @cli.command()
-def clean_up_jobs(artifacts_dir: Path):
+def clean_up_jobs(artifacts_dir: Path, retry_minutes: Optional[PositiveInt] = None):
     """Loop through all users defined in pytest.ini files
     in artifacts_dir and stop+delete all jobs.
     """
@@ -202,22 +210,31 @@ def clean_up_jobs(artifacts_dir: Path):
         cfg = {s: dict(obj.items(s)) for s in obj.sections()}
         server_config = ServerSettings.model_validate(cfg.get("server"))
         servers.add(server_config)
-    for server_config in servers:
-        config = osparc.Configuration(
-            host=server_config.host,
-            username=server_config.key,
-            password=server_config.secret,
-        )
-        typer.echo(
-            f"Cleaning up jobs for user:\n{server_config.model_dump_json(indent=1)}"
-        )
-        with osparc.ApiClient(config) as api_client:
-            solvers_api = osparc.SolversApi(api_client)
-            assert isinstance(solvers := solvers_api.list_solvers_releases(), list)
-            for solver in solvers:
-                assert isinstance(solver, osparc.Solver)
-                assert (id_ := solver.id) is not None
-                assert (version := solver.version) is not None
-                for job in solvers_api.jobs(id_, version):
-                    assert isinstance(job, osparc.Job)
-                    solvers_api.delete_job(id_, version, job.id)
+    for attempt in Retrying(
+        retry=retry_if_exception_type(osparc.ApiException),
+        stop=stop_after_delay(timedelta(minutes=retry_minutes))
+        if retry_minutes
+        else stop_after_attempt(1),
+    ):
+        with attempt:
+            for server_config in servers:
+                config = osparc.Configuration(
+                    host=server_config.host,
+                    username=server_config.key,
+                    password=server_config.secret,
+                )
+                msg = "Cleaning up jobs for user: "
+                msg += f"\n{server_config.model_dump_json(indent=1)}"
+                typer.echo(msg)
+                with osparc.ApiClient(config) as api_client:
+                    solvers_api = osparc.SolversApi(api_client)
+                    assert isinstance(
+                        solvers := solvers_api.list_solvers_releases(), list
+                    )
+                    for solver in solvers:
+                        assert isinstance(solver, osparc.Solver)
+                        assert (id_ := solver.id) is not None
+                        assert (version := solver.version) is not None
+                        for job in solvers_api.jobs(id_, version):
+                            assert isinstance(job, osparc.Job)
+                            solvers_api.delete_job(id_, version, job.id)

clients/python/test/e2e/ci/e2e/pyproject.toml

@@ -14,6 +14,7 @@ pydantic-settings = "^2.1"
 pytest = "^7.4"
 python = "^3.9"
 typer = {extras = ["all"], version = "^0.9.0"}
+tenacity = "^8.2.3"
 
 [tool.poetry.scripts]
 e2e = "e2e.cli:cli"