Merge pull request #57 from MinaFoundation/stop-after-all-retries2

PM-1726 retry and exit using statuses
MinaFoundation · Jun 7, 2024 · 404a886 · 404a886
2 parents 0b44774 + fb59933
commit 404a886
Showing 1 changed file with 11 additions and 3 deletions.
diff --git a/uptime_service_validation/coordinator/server.py b/uptime_service_validation/coordinator/server.py
@@ -272,9 +272,17 @@ def setUpValidatorPods(time_intervals, logging, worker_image, worker_tag):
                 if job_status.status.succeeded:
                     logging.info(f"Job {job_name} succeeded.")
                     jobs.remove(job_name)
-                elif job_status.status.failed:
-                    logging.error(f"Job {job_name} failed.")
-                    jobs.remove(job_name)
+                elif job_status.status.failed is not None:
+                    if job_status.status.failed < Config.RETRY_COUNT:
+                        logging.warning(
+                            f"Job {job_name} failed. Retrying attempt {job_status.status.failed}/{Config.RETRY_COUNT}..."
+                        )
+                    else:
+                        logging.error(
+                            f"Job {job_name} failed. Maximum retries ({Config.RETRY_COUNT}) reached. Exiting the program..."
+                        )
+                        jobs.remove(job_name)
+                        exit(1)
             except Exception as e:
                 logging.error(f"Error reading job status for {job_name}: {e}")