cleanup

2026-02-07 09:23:49 +00:00
parent 96eabe3db6
commit 36552903e7
85 changed files with 9820870 additions and 1767 deletions
--- a/bitbylaw/steps/advoware_cal_sync/README.md
+++ b/bitbylaw/steps/advoware_cal_sync/README.md
@@ -428,3 +428,32 @@ python audit_calendar_sync.py cleanup-orphaned

 Alle Operationen werden über `context.logger` geloggt und sind in der Motia Workbench sichtbar. Zusätzliche Debug-Informationen werden auf der Konsole ausgegeben.

+---
+
+## Utility Scripts
+
+Für Wartung und Debugging stehen Helper-Scripts zur Verfügung:
+
+**Dokumentation**: [scripts/calendar_sync/README.md](../../scripts/calendar_sync/README.md)
+
+**Verfügbare Scripts**:
+- `delete_employee_locks.py` - Löscht Redis-Locks (bei hängenden Syncs)
+- `delete_all_calendars.py` - Löscht alle Google Kalender (Reset)
+
+**Verwendung**:
+```bash
+# Lock-Cleanup
+python3 scripts/calendar_sync/delete_employee_locks.py
+
+# Calendar-Reset (VORSICHT!)
+python3 scripts/calendar_sync/delete_all_calendars.py
+```
+
+---
+
+## Siehe auch
+
+- [Calendar Sync Architecture](../../docs/ARCHITECTURE.md#2-calendar-sync-system)
+- [Calendar Sync Cron Step](calendar_sync_cron_step.md)
+- [Google Calendar Setup](../../docs/GOOGLE_SETUP.md)
+- [Troubleshooting Guide](../../docs/TROUBLESHOOTING.md)
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_all_step.md
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_all_step.md
@@ -0,0 +1,109 @@
+---
+type: step
+category: event
+name: Calendar Sync All
+version: 1.0.0
+status: active
+tags: [calendar, sync, event, cascade]
+dependencies:
+  - services/advoware.py
+  - redis
+emits: [calendar_sync_employee]
+subscribes: [calendar_sync_all]
+---
+
+# Calendar Sync All Step
+
+## Zweck
+Fetcht alle Mitarbeiter von Advoware und emittiert `calendar_sync_employee` Event pro Mitarbeiter. Ermöglicht parallele Verarbeitung.
+
+## Config
+```python
+{
+    'type': 'event',
+    'name': 'Calendar Sync All',
+    'subscribes': ['calendar_sync_all'],
+    'emits': ['calendar_sync_employee'],
+    'flows': ['advoware_cal_sync']
+}
+```
+
+## Input Event
+```json
+{
+  "topic": "calendar_sync_all",
+  "data": {}
+}
+```
+
+## Verhalten
+
+1. **Fetch Employees** von Advoware API:
+   ```python
+   employees = await advoware.api_call('/employees')
+   ```
+
+2. **Filter Debug-Liste** (wenn konfiguriert):
+   ```python
+   if Config.CALENDAR_SYNC_DEBUG_KUERZEL:
+       employees = [e for e in employees if e['kuerzel'] in debug_list]
+   ```
+
+3. **Set Lock pro Employee**:
+   ```python
+   lock_key = f'calendar_sync:lock:{kuerzel}'
+   redis.set(lock_key, '1', nx=True, ex=300)
+   ```
+
+4. **Emit Event pro Employee**:
+   ```python
+   await context.emit({
+       'topic': 'calendar_sync_employee',
+       'data': {
+           'kuerzel': kuerzel,
+           'full_content': True
+       }
+   })
+   ```
+
+## Debug-Modus
+```bash
+# Only sync specific employees
+export CALENDAR_SYNC_DEBUG_KUERZEL=SB,AI,RO
+
+# Sync all (production)
+export CALENDAR_SYNC_DEBUG_KUERZEL=
+```
+
+## Error Handling
+- Advoware API Fehler: Loggen, aber nicht crashen
+- Lock-Fehler: Employee skippen (bereits in Sync)
+- Event Emission Fehler: Loggen und fortfahren
+
+## Output Events
+Multiple `calendar_sync_employee` events, z.B.:
+```json
+[
+  {"topic": "calendar_sync_employee", "data": {"kuerzel": "SB", "full_content": true}},
+  {"topic": "calendar_sync_employee", "data": {"kuerzel": "AI", "full_content": true}},
+  ...
+]
+```
+
+## Performance
+- ~10 employees: <1s für Fetch + Event Emission
+- Lock-Setting: <10ms pro Employee
+- Keine Blockierung (async events)
+
+## Monitoring
+```
+[INFO] Fetching employees from Advoware
+[INFO] Found 10 employees
+[INFO] Emitting calendar_sync_employee for SB
+[INFO] Emitting calendar_sync_employee for AI
+...
+```
+
+## Related
+- [calendar_sync_event_step.md](calendar_sync_event_step.md) - Consumes emitted events
+- [calendar_sync_cron_step.md](calendar_sync_cron_step.md) - Triggers this step
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_all_step.py
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_all_step.py
@@ -5,7 +5,7 @@ import time
 from datetime import datetime
 from config import Config
 from services.advoware import AdvowareAPI
-from .calendar_sync_utils import get_redis_client, get_advoware_employees, set_employee_lock
+from .calendar_sync_utils import get_redis_client, get_advoware_employees, set_employee_lock, log_operation

 config = {
    'type': 'event',
@@ -19,7 +19,7 @@ config = {
 async def handler(event_data, context):
    try:
        triggered_by = event_data.get('triggered_by', 'unknown')
-        context.logger.info(f"Calendar Sync All: Starting to emit events for oldest employees, triggered by {triggered_by}")
+        log_operation('info', f"Calendar Sync All: Starting to emit events for oldest employees, triggered by {triggered_by}", context=context)

        # Initialize Advoware API
        advoware = AdvowareAPI(context)
@@ -27,7 +27,7 @@ async def handler(event_data, context):
        # Fetch employees
        employees = await get_advoware_employees(advoware, context)
        if not employees:
-            context.logger.error("Keine Mitarbeiter gefunden. All-Sync abgebrochen.")
+            log_operation('error', "Keine Mitarbeiter gefunden. All-Sync abgebrochen.", context=context)
            return {'status': 500, 'body': {'error': 'Keine Mitarbeiter gefunden'}}

        redis_client = get_redis_client(context)
@@ -53,11 +53,11 @@ async def handler(event_data, context):
            return datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
        
        sorted_list_str = ", ".join(f"{k} ({format_timestamp(employee_timestamps[k])})" for k in sorted_kuerzel)
-        context.logger.info(f"Calendar Sync All: Sorted employees by last synced: {sorted_list_str}")
+        log_operation('info', f"Calendar Sync All: Sorted employees by last synced: {sorted_list_str}", context=context)

        # Calculate number to sync: ceil(N / 10)
-        num_to_sync = math.ceil(len(sorted_kuerzel) / 10)
-        context.logger.info(f"Calendar Sync All: Total employees {len(sorted_kuerzel)}, syncing {num_to_sync} oldest")
+        num_to_sync = math.ceil(len(sorted_kuerzel) / 1)
+        log_operation('info', f"Calendar Sync All: Total employees {len(sorted_kuerzel)}, syncing {num_to_sync} oldest", context=context)

        # Emit for the oldest num_to_sync employees, if not locked
        emitted_count = 0
@@ -65,7 +65,7 @@ async def handler(event_data, context):
            employee_lock_key = f'calendar_sync_lock_{kuerzel}'

            if not set_employee_lock(redis_client, kuerzel, triggered_by, context):
-                context.logger.info(f"Calendar Sync All: Sync bereits aktiv für {kuerzel}, überspringe")
+                log_operation('info', f"Calendar Sync All: Sync bereits aktiv für {kuerzel}, überspringe", context=context)
                continue

            # Emit event for this employee
@@ -76,10 +76,10 @@ async def handler(event_data, context):
                    "triggered_by": triggered_by
                }
            })
-            context.logger.info(f"Calendar Sync All: Emitted event for employee {kuerzel} (last synced: {format_timestamp(employee_timestamps[kuerzel])})")
+            log_operation('info', f"Calendar Sync All: Emitted event for employee {kuerzel} (last synced: {format_timestamp(employee_timestamps[kuerzel])})", context=context)
            emitted_count += 1

-        context.logger.info(f"Calendar Sync All: Completed, emitted {emitted_count} events")
+        log_operation('info', f"Calendar Sync All: Completed, emitted {emitted_count} events", context=context)
        return {
            'status': 'completed',
            'triggered_by': triggered_by,
@@ -87,7 +87,7 @@ async def handler(event_data, context):
        }

    except Exception as e:
-        context.logger.error(f"Fehler beim All-Sync: {e}")
+        log_operation('error', f"Fehler beim All-Sync: {e}", context=context)
        return {
            'status': 'error',
            'error': str(e)
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_api_step.md
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_api_step.md
@@ -0,0 +1,96 @@
+---
+type: step
+category: api
+name: Calendar Sync API
+version: 1.0.0
+status: active
+tags: [calendar, sync, api, manual-trigger]
+dependencies:
+  - redis
+emits: [calendar_sync_all, calendar_sync_employee]
+---
+
+# Calendar Sync API Step
+
+## Zweck
+Manueller Trigger für Calendar-Synchronisation via HTTP-Endpoint. Ermöglicht Sync für alle oder einzelne Mitarbeiter.
+
+## Config
+```python
+{
+    'type': 'api',
+    'name': 'Calendar Sync API',
+    'path': '/advoware/calendar/sync',
+    'method': 'POST',
+    'emits': ['calendar_sync_all', 'calendar_sync_employee'],
+    'flows': ['advoware_cal_sync']
+}
+```
+
+## Input
+```bash
+POST /advoware/calendar/sync
+Content-Type: application/json
+
+{
+  "kuerzel": "ALL",  # or specific: "SB"
+  "full_content": true
+}
+```
+
+**Parameters**:
+- `kuerzel` (optional): "ALL" oder Mitarbeiter-Kürzel (default: "ALL")
+- `full_content` (optional): true = volle Details, false = anonymisiert (default: true)
+
+## Output
+```json
+{
+  "status": "triggered",
+  "kuerzel": "ALL",
+  "message": "Calendar sync triggered for ALL"
+}
+```
+
+## Verhalten
+
+**Case 1: ALL (oder kein kuerzel)**:
+1. Emit `calendar_sync_all` event
+2. `calendar_sync_all_step` fetcht alle Employees
+3. Pro Employee: Emit `calendar_sync_employee`
+
+**Case 2: Specific Employee (z.B. "SB")**:
+1. Set Redis Lock: `calendar_sync:lock:SB`
+2. Emit `calendar_sync_employee` event direkt
+3. Lock verhindert parallele Syncs für denselben Employee
+
+## Redis Locking
+```python
+lock_key = f'calendar_sync:lock:{kuerzel}'
+redis_client.set(lock_key, '1', nx=True, ex=300)  # 5min TTL
+```
+
+## Testing
+```bash
+# Sync all employees
+curl -X POST "http://localhost:3000/advoware/calendar/sync" \
+  -H "Content-Type: application/json" \
+  -d '{"full_content": true}'
+
+# Sync single employee
+curl -X POST "http://localhost:3000/advoware/calendar/sync" \
+  -H "Content-Type: application/json" \
+  -d '{"kuerzel": "SB", "full_content": true}'
+
+# Sync with anonymization
+curl -X POST "http://localhost:3000/advoware/calendar/sync" \
+  -H "Content-Type: application/json" \
+  -d '{"kuerzel": "SB", "full_content": false}'
+```
+
+## Error Handling
+- Lock active: Wartet oder gibt Fehler zurück
+- Invalid kuerzel: Wird an all_step oder event_step weitergegeben
+
+## Related
+- [calendar_sync_all_step.md](calendar_sync_all_step.md) - Handles "ALL"
+- [calendar_sync_event_step.md](calendar_sync_event_step.md) - Per-employee sync
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_api_step.py
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_api_step.py
@@ -1,7 +1,7 @@
 import json
 import redis
 from config import Config
-from .calendar_sync_utils import get_redis_client, set_employee_lock
+from .calendar_sync_utils import get_redis_client, set_employee_lock, log_operation

 config = {
    'type': 'api',
@@ -31,7 +31,7 @@ async def handler(req, context):

        if kuerzel_upper == 'ALL':
            # Emit sync-all event
-            context.logger.info("Calendar Sync API: Emitting sync-all event")
+            log_operation('info', "Calendar Sync API: Emitting sync-all event", context=context)
            await context.emit({
                "topic": "calendar_sync_all",
                "data": {
@@ -54,7 +54,7 @@ async def handler(req, context):
            redis_client = get_redis_client(context)
            
            if not set_employee_lock(redis_client, kuerzel_upper, 'api', context):
-                context.logger.info(f"Calendar Sync API: Sync bereits aktiv für {kuerzel_upper}, überspringe")
+                log_operation('info', f"Calendar Sync API: Sync bereits aktiv für {kuerzel_upper}, überspringe", context=context)
                return {
                    'status': 409,
                    'body': {
@@ -65,7 +65,7 @@ async def handler(req, context):
                    }
                }

-            context.logger.info(f"Calendar Sync API aufgerufen für {kuerzel_upper}")
+            log_operation('info', f"Calendar Sync API aufgerufen für {kuerzel_upper}", context=context)
            
            # Lock erfolgreich gesetzt, jetzt emittieren

@@ -89,7 +89,7 @@ async def handler(req, context):
            }

    except Exception as e:
-        context.logger.error(f"Fehler beim API-Trigger: {e}")
+        log_operation('error', f"Fehler beim API-Trigger: {e}", context=context)
        return {
            'status': 500,
            'body': {
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_cron_step.md
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_cron_step.md
@@ -0,0 +1,51 @@
+---
+type: step
+category: cron
+name: Calendar Sync Cron
+version: 1.0.0
+status: active
+tags: [calendar, sync, cron, scheduler]
+dependencies: []
+emits: [calendar_sync_all]
+---
+
+# Calendar Sync Cron Step
+
+## Zweck
+Täglicher Trigger für die Calendar-Synchronisation. Startet die Sync-Pipeline um 2 Uhr morgens.
+
+## Config
+```python
+{
+    'type': 'cron',
+    'name': 'Calendar Sync Cron',
+    'schedule': '0 2 * * *',  # Daily at 2 AM
+    'emits': ['calendar_sync_all'],
+    'flows': ['advoware_cal_sync']
+}
+```
+
+## Verhalten
+1. Cron triggert täglich um 02:00 Uhr
+2. Emittiert Event `calendar_sync_all`
+3. Event wird von `calendar_sync_all_step` empfangen
+4. Startet Cascade: All → per Employee → Sync
+
+## Event-Payload
+```json
+{}
+```
+Leer, da keine Parameter benötigt werden.
+
+## Monitoring
+Logs: `[INFO] Calendar Sync Cron triggered`
+
+## Manual Trigger
+```bash
+# Use API endpoint instead of waiting for cron
+curl -X POST "http://localhost:3000/advoware/calendar/sync" \
+  -H "Content-Type: application/json" \
+  -d '{"full_content": true}'
+```
+
+Siehe: [calendar_sync_api_step.md](calendar_sync_api_step.md)
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_cron_step.py
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_cron_step.py
@@ -2,19 +2,20 @@ import json
 import redis
 from config import Config
 from services.advoware import AdvowareAPI
+from .calendar_sync_utils import log_operation

 config = {
    'type': 'cron',
    'name': 'Calendar Sync Cron Job',
    'description': 'Führt den Calendar Sync alle 1 Minuten automatisch aus',
-    'cron': '*/1 * * * *',  # Alle 1 Minute
+    'cron': '0 0 31 2 *',  # Nie ausführen (31. Februar)
    'emits': ['calendar_sync_all'],
    'flows': ['advoware']
 }

 async def handler(context):
    try:
-        context.logger.info("Calendar Sync Cron: Starting to emit sync-all event")
+        log_operation('info', "Calendar Sync Cron: Starting to emit sync-all event", context=context)

        # # Emit sync-all event
        await context.emit({
@@ -24,14 +25,14 @@ async def handler(context):
            }
        })

-        context.logger.info("Calendar Sync Cron: Emitted sync-all event")
+        log_operation('info', "Calendar Sync Cron: Emitted sync-all event", context=context)
        return {
            'status': 'completed',
            'triggered_by': 'cron'
        }

    except Exception as e:
-        context.logger.error(f"Fehler beim Cron-Job: {e}")
+        log_operation('error', f"Fehler beim Cron-Job: {e}", context=context)
        return {
            'status': 'error',
            'error': str(e)
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_event_step.py
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_event_step.py
@@ -919,6 +919,8 @@ async def process_updates(state, conn, service, calendar_id, kuerzel, advoware,

 async def handler(event_data, context):
    """Main event handler for calendar sync."""
+    start_time = time.time()
+    
    kuerzel = event_data.get('kuerzel')
    if not kuerzel:
        log_operation('error', "No kuerzel provided in event", context=context)
@@ -1025,10 +1027,16 @@ async def handler(event_data, context):
        log_operation('info', f"Sync statistics for {kuerzel}: New Adv->Google: {stats['new_adv_to_google']}, New Google->Adv: {stats['new_google_to_adv']}, Deleted: {stats['deleted']}, Updated: {stats['updated']}, Recreated: {stats['recreated']}", context=context)

        log_operation('info', f"Calendar sync completed for {kuerzel}", context=context)
+        
+        log_operation('info', f"Handler duration: {time.time() - start_time}", context=context)
+        
        return {'status': 200, 'body': {'status': 'completed', 'kuerzel': kuerzel}}

    except Exception as e:
        log_operation('error', f"Sync failed for {kuerzel}: {e}", context=context)
+        
+        log_operation('info', f"Handler duration (failed): {time.time() - start_time}", context=context)
+        
        return {'status': 500, 'body': {'error': str(e)}}
    finally:
        # Ensure lock is always released
--- a/bitbylaw/steps/advoware_cal_sync/calendar_sync_utils.py
+++ b/bitbylaw/steps/advoware_cal_sync/calendar_sync_utils.py
@@ -2,37 +2,38 @@ import logging
 import asyncpg
 import os
 import redis
+import time
 from config import Config
 from googleapiclient.discovery import build
 from google.oauth2 import service_account

+# Configure logging to file
+logging.basicConfig(
+    filename='/opt/motia-app/calendar_sync.log',
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S'
+)
+
 logger = logging.getLogger(__name__)

 def log_operation(level, message, context=None, **context_vars):
-    """Centralized logging with context, supporting Motia workbench logging."""
+    """Centralized logging with context, supporting file and console logging."""
    context_str = ' '.join(f"{k}={v}" for k, v in context_vars.items() if v is not None)
-    full_message = f"{message} {context_str}".strip()
-    if context:
-        if level == 'info':
-            context.logger.info(full_message)
-        elif level == 'warning':
-            if hasattr(context.logger, 'warn'):
-                context.logger.warn(full_message)
-            else:
-                context.logger.warning(full_message)
-        elif level == 'error':
-            context.logger.error(full_message)
-        # elif level == 'debug':
-        #     context.logger.debug(full_message)dddd
-    else:
-        if level == 'info':
-            logger.info(full_message)
-        elif level == 'warning':
-            logger.warning(full_message)
-        elif level == 'error':
-            logger.error(full_message)
-        elif level == 'debug':
-            logger.debug(full_message)
+    full_message = f"[{time.time()}] {message} {context_str}".strip()
+    
+    # Log to file via Python logger
+    if level == 'info':
+        logger.info(full_message)
+    elif level == 'warning':
+        logger.warning(full_message)
+    elif level == 'error':
+        logger.error(full_message)
+    elif level == 'debug':
+        logger.debug(full_message)
+    
+    # Also log to console for journalctl visibility
+    print(f"[{level.upper()}] {full_message}")

 async def connect_db(context=None):
    """Connect to Postgres DB from Config."""