Add retry for testrunner validation (#65)

python/requirements.txt

@@ -24,7 +24,7 @@ kubernetes~=26.1
 sse-starlette~=1.6
 opentelemetry-api~=1.21
 opentelemetry-exporter-otlp~=1.21
-opentelemetry-instrumentation-fastapi~=0.45b0
+opentelemetry-instrumentation-fastapi==0.46b0
 opentelemetry-sdk~=1.21
 jsontas~=1.4
 packageurl-python~=0.11

python/setup.cfg

@@ -36,7 +36,7 @@ install_requires =
     sse-starlette~=1.6
     opentelemetry-api~=1.21
     opentelemetry-exporter-otlp~=1.21
-    opentelemetry-instrumentation-fastapi~=0.45b0
+    opentelemetry-instrumentation-fastapi==0.46b0
     opentelemetry-sdk~=1.21
     jsontas~=1.4
     packageurl-python~=0.11

python/src/etos_api/library/validator.py

@@ -15,6 +15,8 @@
 # limitations under the License.
 """ETOS API suite validator module."""
 import logging
+import asyncio
+import random
 from typing import List, Union
 from uuid import UUID
 
@@ -24,6 +26,7 @@
 from pydantic import BaseModel  # pylint:disable=no-name-in-module
 from pydantic import ValidationError, conlist, constr, field_validator
 from pydantic.fields import PrivateAttr
+from opentelemetry import trace
 
 from etos_api.library.docker import Docker
 
@@ -179,6 +182,7 @@ async def validate(self, test_suite_url):
         :type test_suite_url: str
         :raises ValidationError: If the suite did not validate.
         """
+        span = trace.get_current_span()
         downloaded_suite = await self._download_suite(test_suite_url)
         for suite_json in downloaded_suite:
             test_runners = set()
@@ -191,6 +195,18 @@ async def validate(self, test_suite_url):
                         test_runners.add(constraint.value)
             docker = Docker()
             for test_runner in test_runners:
-                assert (
-                    await docker.digest(test_runner) is not None
-                ), f"Test runner {test_runner} not found"
+                for attempt in range(3):
+                    result = await docker.digest(test_runner)
+                    if result:
+                        break
+                    span.add_event(
+                        f"Test runner validation unsuccessful, retrying {3 - attempt} more times"
+                    )
+                    self.logger.warning(
+                        "Test runner %s validation unsuccessful, retrying %d more times",
+                        test_runner,
+                        3 - attempt,
+                    )
+                    await asyncio.sleep(random.randint(1, 3))
+
+                assert result is not None, f"Test runner {test_runner} not found"