abdelkouddous/EB_Dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

On Retry Exhausted Policy

Browse Source
This commit is contained in:
Abdelkouddous LHACHIMI
2026-02-20 13:54:34 +00:00
parent af57a1e187
commit cc709200a0
1 changed files with 55 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
eb_dashboard.py
View File

@@ -118,6 +118,7 @@ access_token = ""
refresh_token = "" refresh_token = ""
threads_list = [] threads_list = []
_token_refresh_lock = threading.Lock() _token_refresh_lock = threading.Lock()
on_retry_exhausted = "ask" # "ask" | "ignore" | "abort" — set at startup
_threads_list_lock = threading.Lock() _threads_list_lock = threading.Lock()
global_pbar = None global_pbar = None
_global_pbar_lock = threading.Lock() _global_pbar_lock = threading.Lock()
@@ -225,32 +226,43 @@ def api_call_with_retry(func):
sleep(WAIT_BEFORE_NEW_BATCH_OF_RETRIES) sleep(WAIT_BEFORE_NEW_BATCH_OF_RETRIES)
break # Exit for loop to restart batch in while True break # Exit for loop to restart batch in while True
else: else:
# All automatic batches exhausted, ask the user # All automatic batches exhausted — apply on_retry_exhausted policy
with _user_interaction_lock: with _user_interaction_lock:
console.print(f"\n[bold red]Persistent error in {func_name} after {batch_count} batches ({total_attempts} attempts).[/bold red]") console.print(f"\n[bold red]Persistent error in {func_name} after {batch_count} batches ({total_attempts} attempts).[/bold red]")
console.print(f"[red]Exception: {exc}[/red]") console.print(f"[red]Exception: {exc}[/red]")
choice = questionary.select( if on_retry_exhausted == "ignore":
f"What would you like to do for {func_name}?",
choices=[
"Retry (try another batch of retries)",
"Ignore (return None and continue)",
"Stop script (critical error)"
]
).ask()
if choice == "Retry (try another batch of retries)":
logging.info(f"User chose to retry {func_name}. Restarting batch sequence.")
batch_count = 1 # Reset batch counter for the next interactive round
break # Exit for loop to restart batch in while True
elif choice == "Ignore (return None and continue)":
# Retrieve context if available
ctx = getattr(thread_local_storage, "current_patient_context", {"id": "Unknown", "pseudo": "Unknown"}) ctx = getattr(thread_local_storage, "current_patient_context", {"id": "Unknown", "pseudo": "Unknown"})
logging.warning(f"[IGNORE] User opted to skip {func_name} for Patient {ctx['id']} ({ctx['pseudo']}). Error: {exc}") logging.warning(f"[AUTO-IGNORE] Skipping {func_name} for Patient {ctx['id']} ({ctx['pseudo']}). Error: {exc}")
console.print(f"[yellow]⚠ Auto-ignore: skipping {func_name}.[/yellow]")
return None return None
else:
logging.critical(f"User chose to stop script after persistent error in {func_name}.") elif on_retry_exhausted == "abort":
raise httpx.RequestError(message=f"Persistent error in {func_name} (stopped by user)") logging.critical(f"[AUTO-ABORT] Stopping script after persistent error in {func_name}. Error: {exc}")
console.print(f"[bold red]Auto-abort: stopping script.[/bold red]")
raise httpx.RequestError(message=f"Persistent error in {func_name} (auto-aborted)")
else: # "ask" — interactive prompt (original behaviour)
choice = questionary.select(
f"What would you like to do for {func_name}?",
choices=[
"Retry (try another batch of retries)",
"Ignore (return None and continue)",
"Stop script (critical error)"
]
).ask()
if choice == "Retry (try another batch of retries)":
logging.info(f"User chose to retry {func_name}. Restarting batch sequence.")
batch_count = 1 # Reset batch counter for the next interactive round
break # Exit for loop to restart batch in while True
elif choice == "Ignore (return None and continue)":
ctx = getattr(thread_local_storage, "current_patient_context", {"id": "Unknown", "pseudo": "Unknown"})
logging.warning(f"[IGNORE] User opted to skip {func_name} for Patient {ctx['id']} ({ctx['pseudo']}). Error: {exc}")
return None
else:
logging.critical(f"User chose to stop script after persistent error in {func_name}.")
raise httpx.RequestError(message=f"Persistent error in {func_name} (stopped by user)")
return wrapper return wrapper
@@ -313,6 +325,25 @@ def login():
# BLOCK 3B: FILE UTILITIES # BLOCK 3B: FILE UTILITIES
# ============================================================================ # ============================================================================
def ask_on_retry_exhausted():
"""Asks the user what to do when all API retry batches are exhausted."""
global on_retry_exhausted
choice = questionary.select(
"On retry exhausted :",
choices=[
"Ask (interactive prompt)",
"Ignore (return None and continue)",
"Abort (stop script)"
]
).ask()
if choice is None or choice == "Ask (interactive prompt)":
on_retry_exhausted = "ask"
elif choice == "Ignore (return None and continue)":
on_retry_exhausted = "ignore"
else:
on_retry_exhausted = "abort"
def wait_for_scheduled_launch(): def wait_for_scheduled_launch():
"""Asks the user when to start the processing and waits if needed. """Asks the user when to start the processing and waits if needed.
Options: Immediately / In X minutes / At HH:MM Options: Immediately / In X minutes / At HH:MM
@@ -1295,6 +1326,9 @@ def main():
number_of_threads = int((questionary.text("Number of threads :", default="12", number_of_threads = int((questionary.text("Number of threads :", default="12",
validate=lambda x: x.isdigit() and 0 < int(x) <= MAX_THREADS).ask())) validate=lambda x: x.isdigit() and 0 < int(x) <= MAX_THREADS).ask()))
print()
ask_on_retry_exhausted()
print() print()
wait_for_scheduled_launch() wait_for_scheduled_launch()