Retry cosign

src/pubtools/sign/signers/cosignsigner.py

@@ -142,6 +142,11 @@ class CosignSigner(Signer):
         metadata={"description": "Number of retries for http requests", "sample": 5},
         default=5,
     )
+    wait_time_increase: int = field(
+        init=False,
+        metadata={"description": "Time increase to wait before retrying cosign", "sample": 10},
+        default=10,
+    )
 
     SUPPORTED_OPERATIONS: ClassVar[List[Type[SignOperation]]] = [
         ContainerSignOperation,
@@ -179,6 +184,7 @@ def load_config(self: CosignSigner, config_data: Dict[str, Any]) -> None:
             "registry_password", self.registry_password
         )
         self.retries = config_data["cosign_signer"].get("retries", self.retries)
+        self.wait_time_increase = config_data["cosign_signer"].get("wait_time_increase", self.wait_time_increase)
         self.container_registry_client = ContainerRegistryClient(
             username=self.registry_user,
             password=self.registry_password,
@@ -232,7 +238,7 @@ def container_sign(self: CosignSigner, operation: ContainerSignOperation) -> Sig
         )
 
         outputs = {}
-        processes = {}
+        ref_args = {}
         common_args = [
             self.cosign_bin,
             "-t",
@@ -256,16 +262,17 @@ def container_sign(self: CosignSigner, operation: ContainerSignOperation) -> Sig
         if operation.references:
             for ref, digest in zip(operation.references, operation.digests):
                 repo, tag = ref.rsplit(":", 1)
-                processes[f"{repo}:{digest}"] = run_command(
-                    common_args + ["-a", f"tag={tag}", f"{repo}@{digest}"],
-                    env=env_vars,
-                )
+                ref_args[f"{repo}@{digest}"] = ["-a", f"tag={tag}", f"{repo}@{digest}"]
         else:
             for ref_digest in operation.digests:
-                processes[f"{ref_digest}"] = run_command(common_args + [ref_digest], env=env_vars)
-        for ref, process in processes.items():
-            stdout, stderr = process.communicate()
-            outputs[ref] = (stdout, stderr, process.returncode)
+                ref_args[ref_digest] = [ref_digest]
+        for ref, args in ref_args.items():
+            outputs[ref] = run_command(
+                common_args + args,
+                env=env_vars,
+                tries=self.retries,
+                wait_time_increase=self.wait_time_increase,
+            )
 
         for ref, (stdout, stderr, returncode) in outputs.items():
             if returncode != 0:
@@ -300,12 +307,11 @@ def existing_signatures(self, reference: str) -> Tuple[bool, str]:
             common_args += ["--registry-password", self.registry_password]
         env_vars = os.environ.copy()
         env_vars.update(self.env_variables)
-        process = run_command(
+        stdout, stderr, returncode = run_command(
             common_args + [reference],
             env=env_vars,
         )
-        stdout, stderr = process.communicate()
-        if process.returncode != 0:
+        if returncode != 0:
             return False, stderr
         else:
             ret, err_msg = self.container_registry_client.check_container_image_exists(

src/pubtools/sign/utils.py

@@ -5,13 +5,15 @@
 import subprocess
 import os
 import logging
+import time
 from typing import Any, Dict, List, Union, Callable, cast, Iterable
 
 from .conf.conf import CONFIG_PATHS
 
 from pubtools.tracing import get_trace_wrapper
 
 tw = get_trace_wrapper()
+LOG = logging.getLogger("pubtools.sign.utils")
 
 
 def set_log_level(logger: logging.Logger, level: str) -> None:
@@ -50,12 +52,26 @@ def isodate_now() -> str:
 
 
 @tw.instrument_func(args_to_attr=True)
-def run_command(cmd: List[str], env: Union[Dict[str, Any], None] = None) -> Any:
-    """Run external command and return Process instance."""
-    process = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env
-    )
-    return process
+def run_command(cmd: List[str], env: Union[Dict[str, Any], None] = None, tries=4, wait_time_increase=10) -> Any:
+    """Run external command and return stdout, stderr and returncode."""
+    def _run_command(cmd, env):
+        process = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env
+        )
+        stdout, stderr = process.communicate()
+        return (stdout, stderr, process.returncode)
+    for i in range(tries):
+        stdout, stderr, returncode = _run_command(cmd, env)
+        if returncode != 0:
+            wait_time = i * wait_time_increase
+            LOG.warning("Run command failed. Will retry in %d seconds [try %s/%s]: %s"
+                % (wait_time, i + 1, tries, stderr)
+            )
+            time.sleep(wait_time)
+            stdout, stderr, returncode = _run_command(cmd, env)
+        else:
+            break
+    return (stdout, stderr, returncode)
 
 
 def _get_config_file(config_candidate: str) -> str: