523 lines
18 KiB
Python
523 lines
18 KiB
Python
"""
|
|
Example Usage of eb_script_template.py
|
|
|
|
This file demonstrates how to use the template for a real-world task:
|
|
Fetching all organizations and their inclusion counts.
|
|
|
|
Copy this pattern to create your own scripts.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import threading
|
|
import traceback
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import timedelta
|
|
from time import perf_counter, sleep
|
|
import functools
|
|
|
|
import httpx
|
|
import questionary
|
|
from tqdm import tqdm
|
|
from rich.console import Console
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION - CREDENTIALS
|
|
# ============================================================================
|
|
|
|
DEFAULT_USER_NAME = "ziwig-invest2@yopmail.com"
|
|
DEFAULT_PASSWORD = "pbrrA765$bP3beiuyuiyhiuy!agx"
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION - MICROSERVICES
|
|
# ============================================================================
|
|
|
|
MICROSERVICES = {
|
|
"IAM": {
|
|
"app_id": None,
|
|
"base_url": "https://api-auth.ziwig-connect.com",
|
|
"endpoints": {
|
|
"login": "/api/auth/ziwig-pro/login",
|
|
"refresh": "/api/auth/refreshToken",
|
|
}
|
|
},
|
|
"RC": {
|
|
"app_id": "602aea51-cdb2-4f73-ac99-fd84050dc393",
|
|
"base_url": "https://api-hcp.ziwig-connect.com",
|
|
"endpoints": {
|
|
"config_token": "/api/auth/config-token",
|
|
"refresh": "/api/auth/refreshToken",
|
|
"organizations": "/api/inclusions/getAllOrganizations",
|
|
"statistics": "/api/inclusions/inclusion-statistics",
|
|
}
|
|
},
|
|
# GDD not needed for this example
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION - THREADING
|
|
# ============================================================================
|
|
|
|
MAX_THREADS = 20
|
|
SUBTASKS_POOL_SIZE = 40
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION - RETRY & TIMEOUTS
|
|
# ============================================================================
|
|
|
|
ERROR_MAX_RETRY = 10
|
|
WAIT_BEFORE_RETRY = 0.5
|
|
API_TIMEOUT = 60
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION - LOGGING
|
|
# ============================================================================
|
|
|
|
LOG_LEVEL = logging.INFO
|
|
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
|
|
|
|
|
|
# ============================================================================
|
|
# CONFIGURATION - PROGRESS BARS
|
|
# ============================================================================
|
|
|
|
BAR_N_FMT_WIDTH = 4
|
|
BAR_TOTAL_FMT_WIDTH = 4
|
|
BAR_TIME_WIDTH = 8
|
|
BAR_RATE_WIDTH = 10
|
|
|
|
custom_bar_format = ("{l_bar}{bar}"
|
|
f" {{n_fmt:>{BAR_N_FMT_WIDTH}}}/{{total_fmt:<{BAR_TOTAL_FMT_WIDTH}}} "
|
|
f"[{{elapsed:<{BAR_TIME_WIDTH}}}<{{remaining:>{BAR_TIME_WIDTH}}}, "
|
|
f"{{rate_fmt:>{BAR_RATE_WIDTH}}}]{{postfix}}")
|
|
|
|
|
|
# ============================================================================
|
|
# GLOBAL VARIABLES
|
|
# ============================================================================
|
|
|
|
tokens = {}
|
|
httpx_clients = {}
|
|
threads_list = []
|
|
_threads_list_lock = threading.Lock()
|
|
_token_refresh_lock = threading.Lock()
|
|
main_thread_pool = None
|
|
subtasks_thread_pool = None
|
|
console = Console()
|
|
|
|
|
|
# ============================================================================
|
|
# UTILITIES (copied from template)
|
|
# ============================================================================
|
|
|
|
def get_nested_value(data_structure, path, default=None):
|
|
"""Extract value from nested dict/list structures with wildcard support."""
|
|
if data_structure is None:
|
|
return "$$$$ No Data"
|
|
if not path:
|
|
return default
|
|
|
|
if "*" in path:
|
|
wildcard_index = path.index("*")
|
|
path_before = path[:wildcard_index]
|
|
path_after = path[wildcard_index+1:]
|
|
|
|
def _get_simple_nested_value(ds, p, d):
|
|
cl = ds
|
|
for k in p:
|
|
if isinstance(cl, dict):
|
|
cl = cl.get(k)
|
|
elif isinstance(cl, list):
|
|
try:
|
|
if isinstance(k, int) and -len(cl) <= k < len(cl):
|
|
cl = cl[k]
|
|
else:
|
|
return d
|
|
except (IndexError, TypeError):
|
|
return d
|
|
else:
|
|
return d
|
|
if cl is None:
|
|
return d
|
|
return cl
|
|
|
|
base_level = _get_simple_nested_value(data_structure, path_before, default)
|
|
|
|
if not isinstance(base_level, list):
|
|
return default
|
|
|
|
results = []
|
|
for item in base_level:
|
|
value = get_nested_value(item, path_after, default)
|
|
if value is not default and value != "$$$$ No Data":
|
|
results.append(value)
|
|
|
|
final_results = []
|
|
for res in results:
|
|
if isinstance(res, list):
|
|
final_results.extend(res)
|
|
else:
|
|
final_results.append(res)
|
|
|
|
return final_results
|
|
|
|
current_level = data_structure
|
|
for key_or_index in path:
|
|
if isinstance(current_level, dict):
|
|
current_level = current_level.get(key_or_index)
|
|
if current_level is None:
|
|
return default
|
|
elif isinstance(current_level, list):
|
|
try:
|
|
if isinstance(key_or_index, int) and -len(current_level) <= key_or_index < len(current_level):
|
|
current_level = current_level[key_or_index]
|
|
else:
|
|
return default
|
|
except (IndexError, TypeError):
|
|
return default
|
|
else:
|
|
return default
|
|
return current_level
|
|
|
|
|
|
def get_httpx_client() -> httpx.Client:
|
|
"""Get or create thread-local HTTP client with keep-alive enabled."""
|
|
global httpx_clients
|
|
thread_id = threading.get_ident()
|
|
if thread_id not in httpx_clients:
|
|
httpx_clients[thread_id] = httpx.Client(
|
|
headers={"Connection": "keep-alive"},
|
|
limits=httpx.Limits(max_keepalive_connections=20, max_connections=100)
|
|
)
|
|
return httpx_clients[thread_id]
|
|
|
|
|
|
def get_thread_position():
|
|
"""Get position of current thread in threads list."""
|
|
global threads_list
|
|
thread_id = threading.get_ident()
|
|
with _threads_list_lock:
|
|
if thread_id not in threads_list:
|
|
threads_list.append(thread_id)
|
|
return len(threads_list) - 1
|
|
else:
|
|
return threads_list.index(thread_id)
|
|
|
|
|
|
# ============================================================================
|
|
# AUTHENTICATION (copied from template)
|
|
# ============================================================================
|
|
|
|
def login():
|
|
"""Authenticate with IAM and configure tokens for all microservices."""
|
|
global tokens
|
|
|
|
user_name = questionary.text("login:", default=DEFAULT_USER_NAME).ask()
|
|
password = questionary.password("password:", default=DEFAULT_PASSWORD).ask()
|
|
|
|
if not (user_name and password):
|
|
return "Exit"
|
|
|
|
try:
|
|
client = get_httpx_client()
|
|
client.base_url = MICROSERVICES["IAM"]["base_url"]
|
|
response = client.post(
|
|
MICROSERVICES["IAM"]["endpoints"]["login"],
|
|
json={"username": user_name, "password": password},
|
|
timeout=20
|
|
)
|
|
response.raise_for_status()
|
|
master_token = response.json()["access_token"]
|
|
user_id = response.json()["userId"]
|
|
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
|
|
print(f"Login Error: {exc}")
|
|
logging.warning(f"Login Error: {exc}")
|
|
return "Error"
|
|
|
|
for app_name, app_config in MICROSERVICES.items():
|
|
if app_name == "IAM":
|
|
continue
|
|
|
|
try:
|
|
client = get_httpx_client()
|
|
client.base_url = app_config["base_url"]
|
|
response = client.post(
|
|
app_config["endpoints"]["config_token"],
|
|
headers={"Authorization": f"Bearer {master_token}"},
|
|
json={
|
|
"userId": user_id,
|
|
"clientId": app_config["app_id"],
|
|
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
|
},
|
|
timeout=20
|
|
)
|
|
response.raise_for_status()
|
|
tokens[app_name] = {
|
|
"access_token": response.json()["access_token"],
|
|
"refresh_token": response.json()["refresh_token"]
|
|
}
|
|
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
|
|
print(f"Config-token Error for {app_name}: {exc}")
|
|
logging.warning(f"Config-token Error for {app_name}: {exc}")
|
|
return "Error"
|
|
|
|
print("\nLogin Success")
|
|
return "Success"
|
|
|
|
|
|
def new_token(app):
|
|
"""Refresh access token for a specific microservice."""
|
|
global tokens
|
|
|
|
with _token_refresh_lock:
|
|
for attempt in range(ERROR_MAX_RETRY):
|
|
try:
|
|
client = get_httpx_client()
|
|
client.base_url = MICROSERVICES[app]["base_url"]
|
|
response = client.post(
|
|
MICROSERVICES[app]["endpoints"]["refresh"],
|
|
headers={"Authorization": f"Bearer {tokens[app]['access_token']}"},
|
|
json={"refresh_token": tokens[app]["refresh_token"]},
|
|
timeout=20
|
|
)
|
|
response.raise_for_status()
|
|
tokens[app]["access_token"] = response.json()["access_token"]
|
|
tokens[app]["refresh_token"] = response.json()["refresh_token"]
|
|
return
|
|
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
|
|
logging.warning(f"Refresh Token Error for {app} (Attempt {attempt + 1}): {exc}")
|
|
if attempt < ERROR_MAX_RETRY - 1:
|
|
sleep(WAIT_BEFORE_RETRY)
|
|
|
|
logging.critical(f"Persistent error in refresh_token for {app}")
|
|
raise httpx.RequestError(message=f"Persistent error in refresh_token for {app}")
|
|
|
|
|
|
# ============================================================================
|
|
# DECORATORS (copied from template)
|
|
# ============================================================================
|
|
|
|
def api_call_with_retry(app):
|
|
"""Decorator for API calls with automatic retry and token refresh on 401."""
|
|
def decorator(func):
|
|
@functools.wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
func_name = func.__name__
|
|
for attempt in range(ERROR_MAX_RETRY):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
|
|
logging.warning(f"Error in {func_name} (Attempt {attempt + 1}/{ERROR_MAX_RETRY}): {exc}")
|
|
|
|
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401:
|
|
logging.info(f"Token expired for {func_name}. Refreshing token for {app}.")
|
|
new_token(app)
|
|
|
|
if attempt < ERROR_MAX_RETRY - 1:
|
|
sleep(WAIT_BEFORE_RETRY)
|
|
|
|
logging.critical(f"Persistent error in {func_name} after {ERROR_MAX_RETRY} attempts.")
|
|
raise httpx.RequestError(message=f"Persistent error in {func_name}")
|
|
|
|
return wrapper
|
|
return decorator
|
|
|
|
|
|
# ============================================================================
|
|
# API CALLS - CUSTOM IMPLEMENTATION
|
|
# ============================================================================
|
|
|
|
@api_call_with_retry("RC")
|
|
def get_all_organizations():
|
|
"""Fetch all organizations from RC API."""
|
|
client = get_httpx_client()
|
|
client.base_url = MICROSERVICES["RC"]["base_url"]
|
|
response = client.get(
|
|
MICROSERVICES["RC"]["endpoints"]["organizations"],
|
|
headers={"Authorization": f"Bearer {tokens['RC']['access_token']}"},
|
|
timeout=API_TIMEOUT
|
|
)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
|
|
@api_call_with_retry("RC")
|
|
def get_organization_statistics(organization_id, protocol_id):
|
|
"""Fetch statistics for a specific organization."""
|
|
client = get_httpx_client()
|
|
client.base_url = MICROSERVICES["RC"]["base_url"]
|
|
response = client.post(
|
|
MICROSERVICES["RC"]["endpoints"]["statistics"],
|
|
headers={"Authorization": f"Bearer {tokens['RC']['access_token']}"},
|
|
json={
|
|
"protocolId": protocol_id,
|
|
"center": organization_id,
|
|
"excludedCenters": []
|
|
},
|
|
timeout=API_TIMEOUT
|
|
)
|
|
response.raise_for_status()
|
|
return response.json()["statistic"]
|
|
|
|
|
|
# ============================================================================
|
|
# PROCESSING FUNCTIONS - CUSTOM IMPLEMENTATION
|
|
# ============================================================================
|
|
|
|
def process_organization(organization, protocol_id):
|
|
"""
|
|
Process a single organization: fetch statistics and enrich data.
|
|
|
|
Args:
|
|
organization: Organization dict from API
|
|
protocol_id: Protocol UUID
|
|
|
|
Returns:
|
|
Dict with organization data and statistics
|
|
"""
|
|
org_id = organization["id"]
|
|
org_name = organization["name"]
|
|
|
|
# Fetch statistics using subtasks pool
|
|
stats_future = subtasks_thread_pool.submit(get_organization_statistics, org_id, protocol_id)
|
|
stats = stats_future.result()
|
|
|
|
return {
|
|
"id": org_id,
|
|
"name": org_name,
|
|
"total_inclusions": stats.get("totalInclusions", 0),
|
|
"pre_included": stats.get("preIncluded", 0),
|
|
"included": stats.get("included", 0),
|
|
"terminated": stats.get("prematurelyTerminated", 0)
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# MAIN PROCESSING - CUSTOM IMPLEMENTATION
|
|
# ============================================================================
|
|
|
|
def main():
|
|
"""Main processing: fetch organizations and their statistics."""
|
|
global main_thread_pool, subtasks_thread_pool
|
|
|
|
# ========== AUTHENTICATION ==========
|
|
print()
|
|
login_status = login()
|
|
while login_status == "Error":
|
|
login_status = login()
|
|
if login_status == "Exit":
|
|
return
|
|
|
|
# ========== CONFIGURATION ==========
|
|
print()
|
|
number_of_threads = int(
|
|
questionary.text(
|
|
"Number of threads:",
|
|
default="8",
|
|
validate=lambda x: x.isdigit() and 0 < int(x) <= MAX_THREADS
|
|
).ask()
|
|
)
|
|
|
|
# Protocol ID for Endobest
|
|
protocol_id = "3c7bcb4d-91ed-4e9f-b93f-99d8447a276e"
|
|
|
|
# ========== INITIALIZATION ==========
|
|
start_time = perf_counter()
|
|
|
|
main_thread_pool = ThreadPoolExecutor(max_workers=number_of_threads)
|
|
subtasks_thread_pool = ThreadPoolExecutor(max_workers=SUBTASKS_POOL_SIZE)
|
|
|
|
# ========== MAIN PROCESSING ==========
|
|
print()
|
|
console.print("[bold cyan]Fetching organizations...[/bold cyan]")
|
|
|
|
organizations = get_all_organizations()
|
|
console.print(f"[green]Found {len(organizations)} organizations[/green]")
|
|
|
|
print()
|
|
console.print("[bold cyan]Processing organizations in parallel...[/bold cyan]")
|
|
|
|
results = []
|
|
with tqdm(total=len(organizations), desc="Processing organizations",
|
|
bar_format=custom_bar_format) as pbar:
|
|
with main_thread_pool as executor:
|
|
futures = [executor.submit(process_organization, org, protocol_id)
|
|
for org in organizations]
|
|
|
|
for future in as_completed(futures):
|
|
try:
|
|
result = future.result()
|
|
results.append(result)
|
|
pbar.update(1)
|
|
except Exception as exc:
|
|
logging.critical(f"Error processing organization: {exc}", exc_info=True)
|
|
print(f"\nCRITICAL ERROR: {exc}")
|
|
executor.shutdown(wait=False, cancel_futures=True)
|
|
raise
|
|
|
|
# ========== RESULTS ==========
|
|
print()
|
|
console.print("[bold cyan]Results Summary:[/bold cyan]")
|
|
|
|
# Sort by total inclusions (descending)
|
|
results.sort(key=lambda x: x["total_inclusions"], reverse=True)
|
|
|
|
# Display top 10
|
|
for i, org in enumerate(results[:10], 1):
|
|
console.print(
|
|
f"{i:2}. {org['name'][:40]:<40} | "
|
|
f"Total: {org['total_inclusions']:3} | "
|
|
f"Pre: {org['pre_included']:3} | "
|
|
f"Inc: {org['included']:3} | "
|
|
f"Term: {org['terminated']:2}"
|
|
)
|
|
|
|
# Save to JSON
|
|
output_file = "organizations_summary.json"
|
|
with open(output_file, 'w', encoding='utf-8') as f:
|
|
json.dump(results, f, indent=4, ensure_ascii=False)
|
|
|
|
console.print(f"\n[green]✓ Results saved to {output_file}[/green]")
|
|
|
|
# ========== FINALIZATION ==========
|
|
print()
|
|
print(f"Elapsed time: {str(timedelta(seconds=perf_counter() - start_time))}")
|
|
|
|
|
|
# ============================================================================
|
|
# ENTRY POINT
|
|
# ============================================================================
|
|
|
|
if __name__ == '__main__':
|
|
script_name = os.path.splitext(os.path.basename(__file__))[0]
|
|
log_file_name = f"{script_name}.log"
|
|
|
|
logging.basicConfig(
|
|
level=LOG_LEVEL,
|
|
format=LOG_FORMAT,
|
|
filename=log_file_name,
|
|
filemode='w'
|
|
)
|
|
|
|
try:
|
|
main()
|
|
except Exception as e:
|
|
logging.critical(f"Script terminated with exception: {e}", exc_info=True)
|
|
print(f"\nScript stopped due to error: {e}")
|
|
print(traceback.format_exc())
|
|
finally:
|
|
if 'main_thread_pool' in globals() and main_thread_pool:
|
|
main_thread_pool.shutdown(wait=False, cancel_futures=True)
|
|
if 'subtasks_thread_pool' in globals() and subtasks_thread_pool:
|
|
subtasks_thread_pool.shutdown(wait=False, cancel_futures=True)
|
|
|
|
print('\n')
|
|
input("Press Enter to exit...")
|