From cd4d8b5c83526aa747ad95036451f6c20df8304f Mon Sep 17 00:00:00 2001 From: Fredrik Fristedt Date: Mon, 3 Jun 2024 13:16:43 +0200 Subject: [PATCH 1/6] Add retry for testrunner validation --- python/src/etos_api/library/validator.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/src/etos_api/library/validator.py b/python/src/etos_api/library/validator.py index 5aad911..30c6a9e 100644 --- a/python/src/etos_api/library/validator.py +++ b/python/src/etos_api/library/validator.py @@ -15,6 +15,8 @@ # limitations under the License. """ETOS API suite validator module.""" import logging +import asyncio +import random from typing import List, Union from uuid import UUID @@ -191,6 +193,13 @@ async def validate(self, test_suite_url): test_runners.add(constraint.value) docker = Docker() for test_runner in test_runners: + for _ in range(3): + result = await docker.pull(test_runner) + if result: + break + + await asyncio.sleep(random.randint(1, 3 )) + assert ( - await docker.digest(test_runner) is not None + result is not None ), f"Test runner {test_runner} not found" From 846b88cc4adc64276b4efaffa3d3be7ea4ee9d29 Mon Sep 17 00:00:00 2001 From: Fredrik Fristedt Date: Tue, 4 Jun 2024 11:20:25 +0200 Subject: [PATCH 2/6] Fix CoPilot hallucination bug. --- python/src/etos_api/library/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/etos_api/library/validator.py b/python/src/etos_api/library/validator.py index 30c6a9e..b9e8d94 100644 --- a/python/src/etos_api/library/validator.py +++ b/python/src/etos_api/library/validator.py @@ -194,7 +194,7 @@ async def validate(self, test_suite_url): docker = Docker() for test_runner in test_runners: for _ in range(3): - result = await docker.pull(test_runner) + result = await docker.digest(test_runner) if result: break From c316daa09fe03924410ff88843e8cd21b20a9604 Mon Sep 17 00:00:00 2001 From: Fredrik Fristedt Date: Tue, 4 Jun 2024 11:24:50 +0200 Subject: [PATCH 3/6] Add retry log message --- python/src/etos_api/library/validator.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/src/etos_api/library/validator.py b/python/src/etos_api/library/validator.py index b9e8d94..e8e8185 100644 --- a/python/src/etos_api/library/validator.py +++ b/python/src/etos_api/library/validator.py @@ -193,11 +193,16 @@ async def validate(self, test_suite_url): test_runners.add(constraint.value) docker = Docker() for test_runner in test_runners: - for _ in range(3): + for attempt in range(3): result = await docker.digest(test_runner) if result: break - + else: + self.logger.warning( + "Test runner %s validation unsuccessful, retrying %d more times", + test_runner, + 3 - attempt, + ) await asyncio.sleep(random.randint(1, 3 )) assert ( From 7c0685f9fe17e12f52fc6bf73cd5a1a820a50bbf Mon Sep 17 00:00:00 2001 From: Fredrik Fristedt Date: Tue, 4 Jun 2024 15:30:50 +0200 Subject: [PATCH 4/6] Fix Open Telemetry dependancies problems --- python/requirements.txt | 2 +- python/setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index c3ea0f2..4ef8864 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -24,7 +24,7 @@ kubernetes~=26.1 sse-starlette~=1.6 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 -opentelemetry-instrumentation-fastapi~=0.45b0 +opentelemetry-instrumentation-fastapi==0.46b0 opentelemetry-sdk~=1.21 jsontas~=1.4 packageurl-python~=0.11 diff --git a/python/setup.cfg b/python/setup.cfg index 97e04e6..7dab435 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -36,7 +36,7 @@ install_requires = sse-starlette~=1.6 opentelemetry-api~=1.21 opentelemetry-exporter-otlp~=1.21 - opentelemetry-instrumentation-fastapi~=0.45b0 + opentelemetry-instrumentation-fastapi==0.46b0 opentelemetry-sdk~=1.21 jsontas~=1.4 packageurl-python~=0.11 From ebc5990fa9c27e1fd0e0a49d24577f0b6fd0f10c Mon Sep 17 00:00:00 2001 From: Fredrik Fristedt Date: Tue, 4 Jun 2024 15:41:40 +0200 Subject: [PATCH 5/6] Make Black and Pylint happy --- python/src/etos_api/library/validator.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/python/src/etos_api/library/validator.py b/python/src/etos_api/library/validator.py index e8e8185..70a9d4d 100644 --- a/python/src/etos_api/library/validator.py +++ b/python/src/etos_api/library/validator.py @@ -197,14 +197,12 @@ async def validate(self, test_suite_url): result = await docker.digest(test_runner) if result: break - else: - self.logger.warning( - "Test runner %s validation unsuccessful, retrying %d more times", - test_runner, - 3 - attempt, - ) - await asyncio.sleep(random.randint(1, 3 )) - - assert ( - result is not None - ), f"Test runner {test_runner} not found" + + self.logger.warning( + "Test runner %s validation unsuccessful, retrying %d more times", + test_runner, + 3 - attempt, + ) + await asyncio.sleep(random.randint(1, 3)) + + assert result is not None, f"Test runner {test_runner} not found" From 2bc0d0fd0bdb0831088f414243c715a27349d284 Mon Sep 17 00:00:00 2001 From: Fredrik Fristedt Date: Tue, 11 Jun 2024 11:38:26 +0200 Subject: [PATCH 6/6] Add Open telemetry span event on test runner validation retries --- python/src/etos_api/library/validator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/src/etos_api/library/validator.py b/python/src/etos_api/library/validator.py index 70a9d4d..8afa986 100644 --- a/python/src/etos_api/library/validator.py +++ b/python/src/etos_api/library/validator.py @@ -26,6 +26,7 @@ from pydantic import BaseModel # pylint:disable=no-name-in-module from pydantic import ValidationError, conlist, constr, field_validator from pydantic.fields import PrivateAttr +from opentelemetry import trace from etos_api.library.docker import Docker @@ -181,6 +182,7 @@ async def validate(self, test_suite_url): :type test_suite_url: str :raises ValidationError: If the suite did not validate. """ + span = trace.get_current_span() downloaded_suite = await self._download_suite(test_suite_url) for suite_json in downloaded_suite: test_runners = set() @@ -197,7 +199,9 @@ async def validate(self, test_suite_url): result = await docker.digest(test_runner) if result: break - + span.add_event( + f"Test runner validation unsuccessful, retrying {3 - attempt} more times" + ) self.logger.warning( "Test runner %s validation unsuccessful, retrying %d more times", test_runner,