abdelkouddous/EB_Script_Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version Initiale

Browse Source
This commit is contained in:
Abdelkouddous LHACHIMI
2025-12-13 00:06:47 +01:00
commit 599360ba34
6 changed files with 2170 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

522
example_usage.py Normal file
View File

@@ -0,0 +1,522 @@
"""
Example Usage of eb_script_template.py
This file demonstrates how to use the template for a real-world task:
Fetching all organizations and their inclusion counts.
Copy this pattern to create your own scripts.
"""
import json
import logging
import os
import sys
import threading
import traceback
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import timedelta
from time import perf_counter, sleep
import functools
import httpx
import questionary
from tqdm import tqdm
from rich.console import Console
# ============================================================================
# CONFIGURATION - CREDENTIALS
# ============================================================================
DEFAULT_USER_NAME = "ziwig-invest2@yopmail.com"
DEFAULT_PASSWORD = "pbrrA765$bP3beiuyuiyhiuy!agx"
# ============================================================================
# CONFIGURATION - MICROSERVICES
# ============================================================================
MICROSERVICES = {
"IAM": {
"app_id": None,
"base_url": "https://api-auth.ziwig-connect.com",
"endpoints": {
"login": "/api/auth/ziwig-pro/login",
"refresh": "/api/auth/refreshToken",
}
},
"RC": {
"app_id": "602aea51-cdb2-4f73-ac99-fd84050dc393",
"base_url": "https://api-hcp.ziwig-connect.com",
"endpoints": {
"config_token": "/api/auth/config-token",
"refresh": "/api/auth/refreshToken",
"organizations": "/api/inclusions/getAllOrganizations",
"statistics": "/api/inclusions/inclusion-statistics",
}
},
# GDD not needed for this example
}
# ============================================================================
# CONFIGURATION - THREADING
# ============================================================================
MAX_THREADS = 20
SUBTASKS_POOL_SIZE = 40
# ============================================================================
# CONFIGURATION - RETRY & TIMEOUTS
# ============================================================================
ERROR_MAX_RETRY = 10
WAIT_BEFORE_RETRY = 0.5
API_TIMEOUT = 60
# ============================================================================
# CONFIGURATION - LOGGING
# ============================================================================
LOG_LEVEL = logging.INFO
LOG_FORMAT = '%(asctime)s - %(levelname)s - %(message)s'
# ============================================================================
# CONFIGURATION - PROGRESS BARS
# ============================================================================
BAR_N_FMT_WIDTH = 4
BAR_TOTAL_FMT_WIDTH = 4
BAR_TIME_WIDTH = 8
BAR_RATE_WIDTH = 10
custom_bar_format = ("{l_bar}{bar}"
f" {{n_fmt:>{BAR_N_FMT_WIDTH}}}/{{total_fmt:<{BAR_TOTAL_FMT_WIDTH}}} "
f"[{{elapsed:<{BAR_TIME_WIDTH}}}<{{remaining:>{BAR_TIME_WIDTH}}}, "
f"{{rate_fmt:>{BAR_RATE_WIDTH}}}]{{postfix}}")
# ============================================================================
# GLOBAL VARIABLES
# ============================================================================
tokens = {}
httpx_clients = {}
threads_list = []
_threads_list_lock = threading.Lock()
_token_refresh_lock = threading.Lock()
main_thread_pool = None
subtasks_thread_pool = None
console = Console()
# ============================================================================
# UTILITIES (copied from template)
# ============================================================================
def get_nested_value(data_structure, path, default=None):
"""Extract value from nested dict/list structures with wildcard support."""
if data_structure is None:
return "$$$$ No Data"
if not path:
return default
if "*" in path:
wildcard_index = path.index("*")
path_before = path[:wildcard_index]
path_after = path[wildcard_index+1:]
def _get_simple_nested_value(ds, p, d):
cl = ds
for k in p:
if isinstance(cl, dict):
cl = cl.get(k)
elif isinstance(cl, list):
try:
if isinstance(k, int) and -len(cl) <= k < len(cl):
cl = cl[k]
else:
return d
except (IndexError, TypeError):
return d
else:
return d
if cl is None:
return d
return cl
base_level = _get_simple_nested_value(data_structure, path_before, default)
if not isinstance(base_level, list):
return default
results = []
for item in base_level:
value = get_nested_value(item, path_after, default)
if value is not default and value != "$$$$ No Data":
results.append(value)
final_results = []
for res in results:
if isinstance(res, list):
final_results.extend(res)
else:
final_results.append(res)
return final_results
current_level = data_structure
for key_or_index in path:
if isinstance(current_level, dict):
current_level = current_level.get(key_or_index)
if current_level is None:
return default
elif isinstance(current_level, list):
try:
if isinstance(key_or_index, int) and -len(current_level) <= key_or_index < len(current_level):
current_level = current_level[key_or_index]
else:
return default
except (IndexError, TypeError):
return default
else:
return default
return current_level
def get_httpx_client() -> httpx.Client:
"""Get or create thread-local HTTP client with keep-alive enabled."""
global httpx_clients
thread_id = threading.get_ident()
if thread_id not in httpx_clients:
httpx_clients[thread_id] = httpx.Client(
headers={"Connection": "keep-alive"},
limits=httpx.Limits(max_keepalive_connections=20, max_connections=100)
)
return httpx_clients[thread_id]
def get_thread_position():
"""Get position of current thread in threads list."""
global threads_list
thread_id = threading.get_ident()
with _threads_list_lock:
if thread_id not in threads_list:
threads_list.append(thread_id)
return len(threads_list) - 1
else:
return threads_list.index(thread_id)
# ============================================================================
# AUTHENTICATION (copied from template)
# ============================================================================
def login():
"""Authenticate with IAM and configure tokens for all microservices."""
global tokens
user_name = questionary.text("login:", default=DEFAULT_USER_NAME).ask()
password = questionary.password("password:", default=DEFAULT_PASSWORD).ask()
if not (user_name and password):
return "Exit"
try:
client = get_httpx_client()
client.base_url = MICROSERVICES["IAM"]["base_url"]
response = client.post(
MICROSERVICES["IAM"]["endpoints"]["login"],
json={"username": user_name, "password": password},
timeout=20
)
response.raise_for_status()
master_token = response.json()["access_token"]
user_id = response.json()["userId"]
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
print(f"Login Error: {exc}")
logging.warning(f"Login Error: {exc}")
return "Error"
for app_name, app_config in MICROSERVICES.items():
if app_name == "IAM":
continue
try:
client = get_httpx_client()
client.base_url = app_config["base_url"]
response = client.post(
app_config["endpoints"]["config_token"],
headers={"Authorization": f"Bearer {master_token}"},
json={
"userId": user_id,
"clientId": app_config["app_id"],
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
},
timeout=20
)
response.raise_for_status()
tokens[app_name] = {
"access_token": response.json()["access_token"],
"refresh_token": response.json()["refresh_token"]
}
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
print(f"Config-token Error for {app_name}: {exc}")
logging.warning(f"Config-token Error for {app_name}: {exc}")
return "Error"
print("\nLogin Success")
return "Success"
def new_token(app):
"""Refresh access token for a specific microservice."""
global tokens
with _token_refresh_lock:
for attempt in range(ERROR_MAX_RETRY):
try:
client = get_httpx_client()
client.base_url = MICROSERVICES[app]["base_url"]
response = client.post(
MICROSERVICES[app]["endpoints"]["refresh"],
headers={"Authorization": f"Bearer {tokens[app]['access_token']}"},
json={"refresh_token": tokens[app]["refresh_token"]},
timeout=20
)
response.raise_for_status()
tokens[app]["access_token"] = response.json()["access_token"]
tokens[app]["refresh_token"] = response.json()["refresh_token"]
return
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
logging.warning(f"Refresh Token Error for {app} (Attempt {attempt + 1}): {exc}")
if attempt < ERROR_MAX_RETRY - 1:
sleep(WAIT_BEFORE_RETRY)
logging.critical(f"Persistent error in refresh_token for {app}")
raise httpx.RequestError(message=f"Persistent error in refresh_token for {app}")
# ============================================================================
# DECORATORS (copied from template)
# ============================================================================
def api_call_with_retry(app):
"""Decorator for API calls with automatic retry and token refresh on 401."""
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
func_name = func.__name__
for attempt in range(ERROR_MAX_RETRY):
try:
return func(*args, **kwargs)
except (httpx.RequestError, httpx.HTTPStatusError) as exc:
logging.warning(f"Error in {func_name} (Attempt {attempt + 1}/{ERROR_MAX_RETRY}): {exc}")
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401:
logging.info(f"Token expired for {func_name}. Refreshing token for {app}.")
new_token(app)
if attempt < ERROR_MAX_RETRY - 1:
sleep(WAIT_BEFORE_RETRY)
logging.critical(f"Persistent error in {func_name} after {ERROR_MAX_RETRY} attempts.")
raise httpx.RequestError(message=f"Persistent error in {func_name}")
return wrapper
return decorator
# ============================================================================
# API CALLS - CUSTOM IMPLEMENTATION
# ============================================================================
@api_call_with_retry("RC")
def get_all_organizations():
"""Fetch all organizations from RC API."""
client = get_httpx_client()
client.base_url = MICROSERVICES["RC"]["base_url"]
response = client.get(
MICROSERVICES["RC"]["endpoints"]["organizations"],
headers={"Authorization": f"Bearer {tokens['RC']['access_token']}"},
timeout=API_TIMEOUT
)
response.raise_for_status()
return response.json()
@api_call_with_retry("RC")
def get_organization_statistics(organization_id, protocol_id):
"""Fetch statistics for a specific organization."""
client = get_httpx_client()
client.base_url = MICROSERVICES["RC"]["base_url"]
response = client.post(
MICROSERVICES["RC"]["endpoints"]["statistics"],
headers={"Authorization": f"Bearer {tokens['RC']['access_token']}"},
json={
"protocolId": protocol_id,
"center": organization_id,
"excludedCenters": []
},
timeout=API_TIMEOUT
)
response.raise_for_status()
return response.json()["statistic"]
# ============================================================================
# PROCESSING FUNCTIONS - CUSTOM IMPLEMENTATION
# ============================================================================
def process_organization(organization, protocol_id):
"""
Process a single organization: fetch statistics and enrich data.
Args:
organization: Organization dict from API
protocol_id: Protocol UUID
Returns:
Dict with organization data and statistics
"""
org_id = organization["id"]
org_name = organization["name"]
# Fetch statistics using subtasks pool
stats_future = subtasks_thread_pool.submit(get_organization_statistics, org_id, protocol_id)
stats = stats_future.result()
return {
"id": org_id,
"name": org_name,
"total_inclusions": stats.get("totalInclusions", 0),
"pre_included": stats.get("preIncluded", 0),
"included": stats.get("included", 0),
"terminated": stats.get("prematurelyTerminated", 0)
}
# ============================================================================
# MAIN PROCESSING - CUSTOM IMPLEMENTATION
# ============================================================================
def main():
"""Main processing: fetch organizations and their statistics."""
global main_thread_pool, subtasks_thread_pool
# ========== AUTHENTICATION ==========
print()
login_status = login()
while login_status == "Error":
login_status = login()
if login_status == "Exit":
return
# ========== CONFIGURATION ==========
print()
number_of_threads = int(
questionary.text(
"Number of threads:",
default="8",
validate=lambda x: x.isdigit() and 0 < int(x) <= MAX_THREADS
).ask()
)
# Protocol ID for Endobest
protocol_id = "3c7bcb4d-91ed-4e9f-b93f-99d8447a276e"
# ========== INITIALIZATION ==========
start_time = perf_counter()
main_thread_pool = ThreadPoolExecutor(max_workers=number_of_threads)
subtasks_thread_pool = ThreadPoolExecutor(max_workers=SUBTASKS_POOL_SIZE)
# ========== MAIN PROCESSING ==========
print()
console.print("[bold cyan]Fetching organizations...[/bold cyan]")
organizations = get_all_organizations()
console.print(f"[green]Found {len(organizations)} organizations[/green]")
print()
console.print("[bold cyan]Processing organizations in parallel...[/bold cyan]")
results = []
with tqdm(total=len(organizations), desc="Processing organizations",
bar_format=custom_bar_format) as pbar:
with main_thread_pool as executor:
futures = [executor.submit(process_organization, org, protocol_id)
for org in organizations]
for future in as_completed(futures):
try:
result = future.result()
results.append(result)
pbar.update(1)
except Exception as exc:
logging.critical(f"Error processing organization: {exc}", exc_info=True)
print(f"\nCRITICAL ERROR: {exc}")
executor.shutdown(wait=False, cancel_futures=True)
raise
# ========== RESULTS ==========
print()
console.print("[bold cyan]Results Summary:[/bold cyan]")
# Sort by total inclusions (descending)
results.sort(key=lambda x: x["total_inclusions"], reverse=True)
# Display top 10
for i, org in enumerate(results[:10], 1):
console.print(
f"{i:2}. {org['name'][:40]:<40} | "
f"Total: {org['total_inclusions']:3} | "
f"Pre: {org['pre_included']:3} | "
f"Inc: {org['included']:3} | "
f"Term: {org['terminated']:2}"
)
# Save to JSON
output_file = "organizations_summary.json"
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(results, f, indent=4, ensure_ascii=False)
console.print(f"\n[green]✓ Results saved to {output_file}[/green]")
# ========== FINALIZATION ==========
print()
print(f"Elapsed time: {str(timedelta(seconds=perf_counter() - start_time))}")
# ============================================================================
# ENTRY POINT
# ============================================================================
if __name__ == '__main__':
script_name = os.path.splitext(os.path.basename(__file__))[0]
log_file_name = f"{script_name}.log"
logging.basicConfig(
level=LOG_LEVEL,
format=LOG_FORMAT,
filename=log_file_name,
filemode='w'
)
try:
main()
except Exception as e:
logging.critical(f"Script terminated with exception: {e}", exc_info=True)
print(f"\nScript stopped due to error: {e}")
print(traceback.format_exc())
finally:
if 'main_thread_pool' in globals() and main_thread_pool:
main_thread_pool.shutdown(wait=False, cancel_futures=True)
if 'subtasks_thread_pool' in globals() and subtasks_thread_pool:
subtasks_thread_pool.shutdown(wait=False, cancel_futures=True)
print('\n')
input("Press Enter to exit...")