""" Example Usage of eb_script_template.py This file demonstrates how to use the template for a real-world task: Fetching all organizations and their inclusion counts. Copy this pattern to create your own scripts. """ import json import logging import os import sys import threading import traceback from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import timedelta from time import perf_counter, sleep import functools import httpx import questionary from tqdm import tqdm from rich.console import Console # ============================================================================ # CONFIGURATION - CREDENTIALS # ============================================================================ DEFAULT_USER_NAME = "ziwig-invest2@yopmail.com" DEFAULT_PASSWORD = "pbrrA765$bP3beiuyuiyhiuy!agx" # ============================================================================ # CONFIGURATION - MICROSERVICES # ============================================================================ MICROSERVICES = { "IAM": { "app_id": None, "base_url": "https://api-auth.ziwig-connect.com", "endpoints": { "login": "/api/auth/ziwig-pro/login", "refresh": "/api/auth/refreshToken", } }, "RC": { "app_id": "602aea51-cdb2-4f73-ac99-fd84050dc393", "base_url": "https://api-hcp.ziwig-connect.com", "endpoints": { "config_token": "/api/auth/config-token", "refresh": "/api/auth/refreshToken", "organizations": "/api/inclusions/getAllOrganizations", "statistics": "/api/inclusions/inclusion-statistics", } }, # GDD not needed for this example } # ============================================================================ # CONFIGURATION - THREADING # ============================================================================ MAX_THREADS = 20 SUBTASKS_POOL_SIZE = 40 # ============================================================================ # CONFIGURATION - RETRY & TIMEOUTS # ============================================================================ ERROR_MAX_RETRY = 10 WAIT_BEFORE_RETRY = 0.5 API_TIMEOUT = 60 # ============================================================================ # CONFIGURATION - LOGGING # ============================================================================ LOG_LEVEL = logging.INFO LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s' # ============================================================================ # CONFIGURATION - PROGRESS BARS # ============================================================================ BAR_N_FMT_WIDTH = 4 BAR_TOTAL_FMT_WIDTH = 4 BAR_TIME_WIDTH = 8 BAR_RATE_WIDTH = 10 custom_bar_format = ("{l_bar}{bar}" f" {{n_fmt:>{BAR_N_FMT_WIDTH}}}/{{total_fmt:<{BAR_TOTAL_FMT_WIDTH}}} " f"[{{elapsed:<{BAR_TIME_WIDTH}}}<{{remaining:>{BAR_TIME_WIDTH}}}, " f"{{rate_fmt:>{BAR_RATE_WIDTH}}}]{{postfix}}") # ============================================================================ # GLOBAL VARIABLES # ============================================================================ tokens = {} httpx_clients = {} threads_list = [] _threads_list_lock = threading.Lock() _token_refresh_lock = threading.Lock() main_thread_pool = None subtasks_thread_pool = None console = Console() # ============================================================================ # UTILITIES (copied from template) # ============================================================================ def get_nested_value(data_structure, path, default=None): """Extract value from nested dict/list structures with wildcard support.""" if data_structure is None: return "$$$$ No Data" if not path: return default if "*" in path: wildcard_index = path.index("*") path_before = path[:wildcard_index] path_after = path[wildcard_index+1:] def _get_simple_nested_value(ds, p, d): cl = ds for k in p: if isinstance(cl, dict): cl = cl.get(k) elif isinstance(cl, list): try: if isinstance(k, int) and -len(cl) <= k < len(cl): cl = cl[k] else: return d except (IndexError, TypeError): return d else: return d if cl is None: return d return cl base_level = _get_simple_nested_value(data_structure, path_before, default) if not isinstance(base_level, list): return default results = [] for item in base_level: value = get_nested_value(item, path_after, default) if value is not default and value != "$$$$ No Data": results.append(value) final_results = [] for res in results: if isinstance(res, list): final_results.extend(res) else: final_results.append(res) return final_results current_level = data_structure for key_or_index in path: if isinstance(current_level, dict): current_level = current_level.get(key_or_index) if current_level is None: return default elif isinstance(current_level, list): try: if isinstance(key_or_index, int) and -len(current_level) <= key_or_index < len(current_level): current_level = current_level[key_or_index] else: return default except (IndexError, TypeError): return default else: return default return current_level def get_httpx_client() -> httpx.Client: """Get or create thread-local HTTP client with keep-alive enabled.""" global httpx_clients thread_id = threading.get_ident() if thread_id not in httpx_clients: httpx_clients[thread_id] = httpx.Client( headers={"Connection": "keep-alive"}, limits=httpx.Limits(max_keepalive_connections=20, max_connections=100) ) return httpx_clients[thread_id] def get_thread_position(): """Get position of current thread in threads list.""" global threads_list thread_id = threading.get_ident() with _threads_list_lock: if thread_id not in threads_list: threads_list.append(thread_id) return len(threads_list) - 1 else: return threads_list.index(thread_id) # ============================================================================ # AUTHENTICATION (copied from template) # ============================================================================ def login(): """Authenticate with IAM and configure tokens for all microservices.""" global tokens user_name = questionary.text("login:", default=DEFAULT_USER_NAME).ask() password = questionary.password("password:", default=DEFAULT_PASSWORD).ask() if not (user_name and password): return "Exit" try: client = get_httpx_client() client.base_url = MICROSERVICES["IAM"]["base_url"] response = client.post( MICROSERVICES["IAM"]["endpoints"]["login"], json={"username": user_name, "password": password}, timeout=20 ) response.raise_for_status() master_token = response.json()["access_token"] user_id = response.json()["userId"] except (httpx.RequestError, httpx.HTTPStatusError) as exc: print(f"Login Error: {exc}") logging.warning(f"Login Error: {exc}") return "Error" for app_name, app_config in MICROSERVICES.items(): if app_name == "IAM": continue try: client = get_httpx_client() client.base_url = app_config["base_url"] response = client.post( app_config["endpoints"]["config_token"], headers={"Authorization": f"Bearer {master_token}"}, json={ "userId": user_id, "clientId": app_config["app_id"], "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" }, timeout=20 ) response.raise_for_status() tokens[app_name] = { "access_token": response.json()["access_token"], "refresh_token": response.json()["refresh_token"] } except (httpx.RequestError, httpx.HTTPStatusError) as exc: print(f"Config-token Error for {app_name}: {exc}") logging.warning(f"Config-token Error for {app_name}: {exc}") return "Error" print("\nLogin Success") return "Success" def new_token(app): """Refresh access token for a specific microservice.""" global tokens with _token_refresh_lock: for attempt in range(ERROR_MAX_RETRY): try: client = get_httpx_client() client.base_url = MICROSERVICES[app]["base_url"] response = client.post( MICROSERVICES[app]["endpoints"]["refresh"], headers={"Authorization": f"Bearer {tokens[app]['access_token']}"}, json={"refresh_token": tokens[app]["refresh_token"]}, timeout=20 ) response.raise_for_status() tokens[app]["access_token"] = response.json()["access_token"] tokens[app]["refresh_token"] = response.json()["refresh_token"] return except (httpx.RequestError, httpx.HTTPStatusError) as exc: logging.warning(f"Refresh Token Error for {app} (Attempt {attempt + 1}): {exc}") if attempt < ERROR_MAX_RETRY - 1: sleep(WAIT_BEFORE_RETRY) logging.critical(f"Persistent error in refresh_token for {app}") raise httpx.RequestError(message=f"Persistent error in refresh_token for {app}") # ============================================================================ # DECORATORS (copied from template) # ============================================================================ def api_call_with_retry(app): """Decorator for API calls with automatic retry and token refresh on 401.""" def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ for attempt in range(ERROR_MAX_RETRY): try: return func(*args, **kwargs) except (httpx.RequestError, httpx.HTTPStatusError) as exc: logging.warning(f"Error in {func_name} (Attempt {attempt + 1}/{ERROR_MAX_RETRY}): {exc}") if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401: logging.info(f"Token expired for {func_name}. Refreshing token for {app}.") new_token(app) if attempt < ERROR_MAX_RETRY - 1: sleep(WAIT_BEFORE_RETRY) logging.critical(f"Persistent error in {func_name} after {ERROR_MAX_RETRY} attempts.") raise httpx.RequestError(message=f"Persistent error in {func_name}") return wrapper return decorator # ============================================================================ # API CALLS - CUSTOM IMPLEMENTATION # ============================================================================ @api_call_with_retry("RC") def get_all_organizations(): """Fetch all organizations from RC API.""" client = get_httpx_client() client.base_url = MICROSERVICES["RC"]["base_url"] response = client.get( MICROSERVICES["RC"]["endpoints"]["organizations"], headers={"Authorization": f"Bearer {tokens['RC']['access_token']}"}, timeout=API_TIMEOUT ) response.raise_for_status() return response.json() @api_call_with_retry("RC") def get_organization_statistics(organization_id, protocol_id): """Fetch statistics for a specific organization.""" client = get_httpx_client() client.base_url = MICROSERVICES["RC"]["base_url"] response = client.post( MICROSERVICES["RC"]["endpoints"]["statistics"], headers={"Authorization": f"Bearer {tokens['RC']['access_token']}"}, json={ "protocolId": protocol_id, "center": organization_id, "excludedCenters": [] }, timeout=API_TIMEOUT ) response.raise_for_status() return response.json()["statistic"] # ============================================================================ # PROCESSING FUNCTIONS - CUSTOM IMPLEMENTATION # ============================================================================ def process_organization(organization, protocol_id): """ Process a single organization: fetch statistics and enrich data. Args: organization: Organization dict from API protocol_id: Protocol UUID Returns: Dict with organization data and statistics """ org_id = organization["id"] org_name = organization["name"] # Fetch statistics using subtasks pool stats_future = subtasks_thread_pool.submit(get_organization_statistics, org_id, protocol_id) stats = stats_future.result() return { "id": org_id, "name": org_name, "total_inclusions": stats.get("totalInclusions", 0), "pre_included": stats.get("preIncluded", 0), "included": stats.get("included", 0), "terminated": stats.get("prematurelyTerminated", 0) } # ============================================================================ # MAIN PROCESSING - CUSTOM IMPLEMENTATION # ============================================================================ def main(): """Main processing: fetch organizations and their statistics.""" global main_thread_pool, subtasks_thread_pool # ========== AUTHENTICATION ========== print() login_status = login() while login_status == "Error": login_status = login() if login_status == "Exit": return # ========== CONFIGURATION ========== print() number_of_threads = int( questionary.text( "Number of threads:", default="8", validate=lambda x: x.isdigit() and 0 < int(x) <= MAX_THREADS ).ask() ) # Protocol ID for Endobest protocol_id = "3c7bcb4d-91ed-4e9f-b93f-99d8447a276e" # ========== INITIALIZATION ========== start_time = perf_counter() main_thread_pool = ThreadPoolExecutor(max_workers=number_of_threads) subtasks_thread_pool = ThreadPoolExecutor(max_workers=SUBTASKS_POOL_SIZE) # ========== MAIN PROCESSING ========== print() console.print("[bold cyan]Fetching organizations...[/bold cyan]") organizations = get_all_organizations() console.print(f"[green]Found {len(organizations)} organizations[/green]") print() console.print("[bold cyan]Processing organizations in parallel...[/bold cyan]") results = [] with tqdm(total=len(organizations), desc="Processing organizations", bar_format=custom_bar_format) as pbar: with main_thread_pool as executor: futures = [executor.submit(process_organization, org, protocol_id) for org in organizations] for future in as_completed(futures): try: result = future.result() results.append(result) pbar.update(1) except Exception as exc: logging.critical(f"Error processing organization: {exc}", exc_info=True) print(f"\nCRITICAL ERROR: {exc}") executor.shutdown(wait=False, cancel_futures=True) raise # ========== RESULTS ========== print() console.print("[bold cyan]Results Summary:[/bold cyan]") # Sort by total inclusions (descending) results.sort(key=lambda x: x["total_inclusions"], reverse=True) # Display top 10 for i, org in enumerate(results[:10], 1): console.print( f"{i:2}. {org['name'][:40]:<40} | " f"Total: {org['total_inclusions']:3} | " f"Pre: {org['pre_included']:3} | " f"Inc: {org['included']:3} | " f"Term: {org['terminated']:2}" ) # Save to JSON output_file = "organizations_summary.json" with open(output_file, 'w', encoding='utf-8') as f: json.dump(results, f, indent=4, ensure_ascii=False) console.print(f"\n[green]✓ Results saved to {output_file}[/green]") # ========== FINALIZATION ========== print() print(f"Elapsed time: {str(timedelta(seconds=perf_counter() - start_time))}") # ============================================================================ # ENTRY POINT # ============================================================================ if __name__ == '__main__': script_name = os.path.splitext(os.path.basename(__file__))[0] log_file_name = f"{script_name}.log" logging.basicConfig( level=LOG_LEVEL, format=LOG_FORMAT, filename=log_file_name, filemode='w' ) try: main() except Exception as e: logging.critical(f"Script terminated with exception: {e}", exc_info=True) print(f"\nScript stopped due to error: {e}") print(traceback.format_exc()) finally: if 'main_thread_pool' in globals() and main_thread_pool: main_thread_pool.shutdown(wait=False, cancel_futures=True) if 'subtasks_thread_pool' in globals() and subtasks_thread_pool: subtasks_thread_pool.shutdown(wait=False, cancel_futures=True) print('\n') input("Press Enter to exit...")