bsiggel/motia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor calendar sync for parallel processing: cron emits per-employee events, event-step processes one employee with per-employee locks

Browse Source
This commit is contained in:
root
2025-10-24 00:19:34 +00:00
parent 409bea3615
commit 8e9dc87b2a
2 changed files with 131 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
bitbylaw/steps/advoware_cal_sync/calendar_sync_cron_step.py
View File

@@ -3,77 +3,70 @@ import redis
from config import Config from config import Config
from services.advoware import AdvowareAPI from services.advoware import AdvowareAPI
async def get_advoware_employees(advoware, logger): CALENDAR_SYNC_LOCK_KEY = 'calendar_sync_lock'
"""Fetch list of employees from Advoware."""
try: config = {
result = await advoware.api_call('api/v1/advonet/Mitarbeiter', method='GET', params={'aktiv': 'true'})
employees = result if isinstance(result, list) else []
logger.info(f"Fetched {len(employees)} Advoware employees")
return employees
except Exception as e:
logger.error(f"Failed to fetch Advoware employees: {e}")
raise
'type': 'cron', 'type': 'cron',
'name': 'Calendar Sync Cron Job', 'name': 'Calendar Sync Cron Job',
'description': 'Führt den Calendar Sync alle 15 Minuten automatisch aus', 'description': 'Führt den Calendar Sync alle 15 Minuten automatisch aus',
'cron': '*/15 * * * *', # Alle 15 Minuten 'cron': '*/15 * * * *', # Alle 15 Minuten
'emits': ['calendar.sync.triggered'] 'emits': ['calendar.sync.employee']
} }
async def get_advoware_employees(advoware):
"""Fetch list of employees from Advoware."""
try:
result = await advoware.api_call('api/v1/advonet/Mitarbeiter', method='GET', params={'aktiv': 'true'})
employees = result if isinstance(result, list) else []
context.logger.info(f"Fetched {len(employees)} Advoware employees")
return employees
except Exception as e:
context.logger.error(f"Failed to fetch Advoware employees: {e}")
raise
async def handler(context): async def handler(context):
try: try:
context.logger.info("Calendar Sync Cron: Starting automatic synchronization every 15 minutes") context.logger.info("Calendar Sync Cron: Starting to fetch employees and emit events")
# Initialize Advoware API # Initialize Advoware API
advoware = AdvowareAPI(context) advoware = AdvowareAPI(context)
# Fetch all employees # Fetch employees
employees = await get_advoware_employees(advoware, context.logger) employees = await get_advoware_employees(advoware)
if not employees: if not employees:
context.logger.error("Calendar Sync Cron: No employees found. Sync cancelled.") context.logger.error("Keine Mitarbeiter gefunden. Cron abgebrochen.")
return { return {'status': 500, 'body': {'error': 'Keine Mitarbeiter gefunden'}}
'status': 'error',
'reason': 'no_employees',
'triggered_by': 'cron'
}
total_emitted = 0 # Emit event for each employee (DEBUG: only for SB)
for employee in employees: for employee in employees:
kuerzel = employee.get('kuerzel') kuerzel = employee.get('kuerzel')
if not kuerzel: if not kuerzel:
context.logger.warning(f"Calendar Sync Cron: Employee without kuerzel skipped: {employee}") context.logger.warning(f"Mitarbeiter ohne Kürzel übersprungen: {employee}")
continue continue
# DEBUG: Limit to SB for debugging # DEBUG: Nur für Nutzer SB syncen
if kuerzel != 'SB': if kuerzel != 'SB':
context.logger.info(f"Calendar Sync Cron: DEBUG: Skipping {kuerzel}, only SB synced") context.logger.info(f"DEBUG: Überspringe {kuerzel}, nur SB wird gesynct")
continue continue
context.logger.info(f"Calendar Sync Cron: Emitting sync event for {kuerzel}") # Emit event for this employee
# Emit Event for the Sync with kuerzel
await context.emit({ await context.emit({
"topic": "calendar.sync.triggered", "topic": "calendar.sync.employee",
"data": { "data": {
"body": { "kuerzel": kuerzel,
"kuerzel": kuerzel, "triggered_by": "cron"
"full_content": True, # Cron uses always full details
"triggered_by": "cron"
}
} }
}) })
context.logger.info(f"Calendar Sync Cron: Emitted event for employee {kuerzel}")
total_emitted += 1 context.logger.info("Calendar Sync Cron: Completed emitting events for employees")
context.logger.info(f"Calendar Sync Cron: Emitted {total_emitted} sync events")
return { return {
'status': 'completed', 'status': 'completed',
'total_emitted': total_emitted,
'triggered_by': 'cron' 'triggered_by': 'cron'
} }
except Exception as e: except Exception as e:
context.logger.error(f"Calendar Sync Cron: Error in cron job: {e}") context.logger.error(f"Fehler beim Cron-Job: {e}")
return { return {
'status': 'error', 'status': 'error',
'error': str(e) 'error': str(e)

210
bitbylaw/steps/advoware_cal_sync/calendar_sync_event_step.py
View File

@@ -846,136 +846,126 @@ async def process_updates(state, conn, service, calendar_id, kuerzel, advoware,
log_operation('warning', f"Phase 4: Failed to update sync_id {row['sync_id']}: {e}", context=context) log_operation('warning', f"Phase 4: Failed to update sync_id {row['sync_id']}: {e}", context=context)
async with conn.transaction(): async with conn.transaction():
await conn.execute("UPDATE calendar_sync SET sync_status = 'failed' WHERE sync_id = $1;", row['sync_id']) await conn.execute("UPDATE calendar_sync SET sync_status = 'failed' WHERE sync_id = $1;", row['sync_id'])
CALENDAR_SYNC_LOCK_KEY = 'calendar_sync_lock'
async def handler(context):
"""Main event handler for calendar sync.""" """Main event handler for calendar sync."""
logger.info("Starting calendar sync for all employees") kuerzel = context.input.data.get('kuerzel')
if not kuerzel:
log_operation('error', "No kuerzel provided in event", context=context)
return {'status': 400, 'body': {'error': 'No kuerzel provided'}}
employee_lock_key = f'calendar_sync_lock_{kuerzel}'
log_operation('info', f"Starting calendar sync for employee {kuerzel}", context=context)
redis_client = redis.Redis(
host=Config.REDIS_HOST,
port=int(Config.REDIS_PORT),
db=int(Config.REDIS_DB_CALENDAR_SYNC),
socket_timeout=Config.REDIS_TIMEOUT_SECONDS
)
try: try:
logger.debug("Initializing Advoware API") if redis_client.get(employee_lock_key):
log_operation('info', f"Sync already running for {kuerzel}, skipping", context=context)
return {'status': 200, 'body': {'status': 'skipped', 'reason': 'sync_already_running', 'kuerzel': kuerzel}}
# Set lock for 30 minutes
redis_client.set(employee_lock_key, 'event', ex=1800)
log_operation('info', f"Lock set for {kuerzel}, starting sync", context=context)
log_operation('debug', "Initializing Advoware API", context=context)
advoware = AdvowareAPI(context) advoware = AdvowareAPI(context)
# Alle Mitarbeiter abrufen log_operation('debug', "Initializing Google service", context=context)
logger.info("Rufe Advoware Mitarbeiter ab...") service = await get_google_service()
employees = await get_advoware_employees(advoware) log_operation('debug', f"Ensuring Google calendar for {kuerzel}", context=context)
if not employees: calendar_id = await ensure_google_calendar(service, kuerzel)
logger.error("Keine Mitarbeiter gefunden. Sync abgebrochen.")
return {'status': 500, 'body': {'error': 'Keine Mitarbeiter gefunden'}}
total_synced = 0 conn = await connect_db()
for employee in employees: try:
kuerzel = employee.get('kuerzel') # Initialize state
if not kuerzel: state = {
logger.warning(f"Mitarbeiter ohne Kürzel übersprungen: {employee}") 'rows': [],
continue 'db_adv_index': {},
'db_google_index': {},
'adv_appointments': [],
'adv_map': {},
'google_events': [],
'google_map': {}
}
# DEBUG: Nur für Nutzer AI syncen (für Test der Travel/Prep Zeit) async def reload_db_indexes():
if kuerzel != 'SB': """Reload database indexes after DB changes in phases."""
logger.info(f"DEBUG: Überspringe {kuerzel}, nur AI wird gesynct") state['rows'] = conn.fetch(
continue """
SELECT * FROM calendar_sync
WHERE employee_kuerzel = $1 AND deleted = FALSE
""",
kuerzel
)
state['db_adv_index'] = {str(row['advoware_frnr']): row for row in state['rows'] if row['advoware_frnr']}
state['db_google_index'] = {}
for row in state['rows']:
if row['google_event_id']:
state['db_google_index'][row['google_event_id']] = row
log_operation('debug', "Reloaded indexes", context=context, rows=len(state['rows']), adv=len(state['db_adv_index']), google=len(state['db_google_index']))
logger.info(f"Starting calendar sync for {kuerzel}") async def reload_api_maps():
"""Reload API maps after creating new events in phases."""
state['adv_appointments'] = fetch_advoware_appointments(advoware, kuerzel)
state['adv_map'] = {str(app['frNr']): app for app in state['adv_appointments'] if app.get('frNr')}
state['google_events'] = fetch_google_events(service, calendar_id)
state['google_map'] = {evt['id']: evt for evt in state['google_events']}
log_operation('debug', "Reloaded API maps", context=context, adv=len(state['adv_map']), google=len(state['google_map']))
redis_client = redis.Redis( # Initial fetch
host=Config.REDIS_HOST, log_operation('info', "Fetching fresh data from APIs", context=context)
port=int(Config.REDIS_PORT), await reload_api_maps()
db=int(Config.REDIS_DB_CALENDAR_SYNC), await reload_db_indexes()
socket_timeout=Config.REDIS_TIMEOUT_SECONDS log_operation('info', "Fetched existing sync rows", context=context, count=len(state['rows']))
)
try: # Phase 1: New from Advoware => Google
logger.debug("Initializing Google service") await process_new_from_advoware(state, conn, service, calendar_id, kuerzel, advoware, context)
service = await get_google_service()
logger.debug(f"Ensuring Google calendar for {kuerzel}")
calendar_id = await ensure_google_calendar(service, kuerzel)
conn = await connect_db() # Reload indexes after Phase 1 changes
try: await reload_db_indexes()
# Initialize state # Reload API maps after Phase 1 changes
state = { await reload_api_maps()
'rows': [],
'db_adv_index': {},
'db_google_index': {},
'adv_appointments': [],
'adv_map': {},
'google_events': [],
'google_map': {}
}
async def reload_db_indexes(): # Phase 2: New from Google => Advoware
"""Reload database indexes after DB changes in phases.""" await process_new_from_google(state, conn, service, calendar_id, kuerzel, advoware, context)
state['rows'] = await conn.fetch(
"""
SELECT * FROM calendar_sync
WHERE employee_kuerzel = $1 AND deleted = FALSE
""",
kuerzel
)
state['db_adv_index'] = {str(row['advoware_frnr']): row for row in state['rows'] if row['advoware_frnr']}
state['db_google_index'] = {}
for row in state['rows']:
if row['google_event_id']:
state['db_google_index'][row['google_event_id']] = row
log_operation('debug', "Reloaded indexes", context=context, rows=len(state['rows']), adv=len(state['db_adv_index']), google=len(state['db_google_index']))
async def reload_api_maps(): # Reload indexes after Phase 2 changes
"""Reload API maps after creating new events in phases.""" await reload_db_indexes()
state['adv_appointments'] = await fetch_advoware_appointments(advoware, kuerzel) # Reload API maps after Phase 2 changes
state['adv_map'] = {str(app['frNr']): app for app in state['adv_appointments'] if app.get('frNr')} await reload_api_maps()
state['google_events'] = await fetch_google_events(service, calendar_id)
state['google_map'] = {evt['id']: evt for evt in state['google_events']}
log_operation('debug', "Reloaded API maps", context=context, adv=len(state['adv_map']), google=len(state['google_map']))
# Initial fetch # Phase 3: Identify deleted entries
log_operation('info', "Fetching fresh data from APIs", context=context) await process_deleted_entries(state, conn, service, calendar_id, kuerzel, advoware, context)
await reload_api_maps()
await reload_db_indexes()
log_operation('info', "Fetched existing sync rows", context=context, count=len(state['rows']))
# Phase 1: New from Advoware => Google # Reload indexes after Phase 3 changes
await process_new_from_advoware(state, conn, service, calendar_id, kuerzel, advoware, context) await reload_db_indexes()
# Reload API maps after Phase 3 changes
await reload_api_maps()
# Reload indexes after Phase 1 changes # Phase 4: Update existing entries if changed
await reload_db_indexes() await process_updates(state, conn, service, calendar_id, kuerzel, advoware, context)
# Reload API maps after Phase 1 changes
await reload_api_maps()
# Phase 2: New from Google => Advoware # Update last_sync timestamps
await process_new_from_google(state, conn, service, calendar_id, kuerzel, advoware, context) log_operation('debug', "Updated last_sync timestamps", context=context)
# Reload indexes after Phase 2 changes finally:
await reload_db_indexes() conn.close()
# Reload API maps after Phase 2 changes
await reload_api_maps()
# Phase 3: Identify deleted entries redis_client.delete(employee_lock_key)
await process_deleted_entries(state, conn, service, calendar_id, kuerzel, advoware, context) log_operation('info', f"Calendar sync completed for {kuerzel}", context=context)
return {'status': 200, 'body': {'status': 'completed', 'kuerzel': kuerzel}}
# Reload indexes after Phase 3 changes
await reload_db_indexes()
# Reload API maps after Phase 3 changes
await reload_api_maps()
# Phase 4: Update existing entries if changed
await process_updates(state, conn, service, calendar_id, kuerzel, advoware, context)
# Update last_sync timestamps
logger.debug("Updated last_sync timestamps")
finally:
await conn.close()
redis_client.delete(CALENDAR_SYNC_LOCK_KEY)
logger.info(f"Calendar sync completed for {kuerzel}")
total_synced += 1
except Exception as e:
logger.error(f"Sync failed for {kuerzel}: {e}")
redis_client.delete(CALENDAR_SYNC_LOCK_KEY)
logger.info(f"Calendar sync completed for all employees. Total synced: {total_synced}")
return {'status': 200, 'body': {'status': 'completed', 'total_synced': total_synced}}
except Exception as e: except Exception as e:
logger.error(f"Sync failed: {e}") log_operation('error', f"Sync failed for {kuerzel}: {e}", context=context)
redis_client.delete(employee_lock_key)
return {'status': 500, 'body': {'error': str(e)}} return {'status': 500, 'body': {'error': str(e)}}
# Motia Step Configuration # Motia Step Configuration
@@ -983,7 +973,7 @@ config = {
"type": "event", "type": "event",
"name": "Calendar Sync Event Step", "name": "Calendar Sync Event Step",
"description": "Handles bidirectional calendar sync between Advoware and Google Calendar using Postgres as hub", "description": "Handles bidirectional calendar sync between Advoware and Google Calendar using Postgres as hub",
"subscribes": ["calendar.sync.triggered"], "subscribes": ["calendar.sync.employee"],
"emits": [], "emits": [],
"flows": ["advoware"] "flows": ["advoware"]
} }