retry

SpareCores · Sep 25, 2024 · c0c8f5e · c0c8f5e
1 parent 5188d2d
commit c0c8f5e
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/inspector/lib.py b/inspector/lib.py
@@ -424,7 +424,12 @@ def run_tasks(vendor, data_dir: str | os.PathLike, instance: str, gpu_count: int
         q.join()
         # do a push at the end of each round
         if os.environ.get("GITHUB_TOKEN"):
-            repo.push_path(data_dir, f"Inspecting server from {repo.gha_url()}")
+            for i in range(3):
+                try:
+                    repo.push_path(data_dir, f"Inspecting server from {repo.gha_url()}")
+                except Exception:
+                    logging.exception("push failed")
+                    time.sleep(random.randint(1, 10))
     q.join()