From c5600b42ecd90a2d4f301b782894c95a56c89c51 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 25 Oct 2025 08:18:48 +0000 Subject: [PATCH] feat: Enhance audit_calendar_sync tool with comprehensive management features - Add detailed documentation in README.md for all 10 audit commands - Fix Advoware API parameter from 'frNr' to 'frnr' for proper filtering - Fix subject field mapping from 'betreff' to 'text' in API responses - Add verify-sync command for bidirectional sync verification - Add query-frnr and query-event commands for individual record lookup - Add management commands: find-duplicates, delete-duplicates, find-orphaned, cleanup-orphaned - Improve error handling and output formatting - Remove temporary test script --- bitbylaw/check_db.py | 19 -- bitbylaw/steps/advoware_cal_sync/README.md | 257 +++++++++++++++++- .../advoware_cal_sync/audit_calendar_sync.py | 48 +++- motia-workbench.json | 26 ++ query_db.py | 0 5 files changed, 310 insertions(+), 40 deletions(-) delete mode 100644 bitbylaw/check_db.py create mode 100644 query_db.py diff --git a/bitbylaw/check_db.py b/bitbylaw/check_db.py deleted file mode 100644 index 911347ab..00000000 --- a/bitbylaw/check_db.py +++ /dev/null @@ -1,19 +0,0 @@ -import asyncio -import asyncpg -from config import Config - -async def check_db(): - conn = await asyncpg.connect( - host=Config.POSTGRES_HOST or 'localhost', - user=Config.POSTGRES_USER, - password=Config.POSTGRES_PASSWORD, - database=Config.POSTGRES_DB_NAME, - timeout=10 - ) - try: - row = await conn.fetchrow('SELECT * FROM calendar_sync WHERE sync_id = $1', '1329fa1f-9de5-49dc-95c6-a13525f315c5') - print('DB Row:', dict(row) if row else 'No row found') - finally: - await conn.close() - -asyncio.run(check_db()) \ No newline at end of file diff --git a/bitbylaw/steps/advoware_cal_sync/README.md b/bitbylaw/steps/advoware_cal_sync/README.md index b74f1c16..51adbd6d 100644 --- a/bitbylaw/steps/advoware_cal_sync/README.md +++ b/bitbylaw/steps/advoware_cal_sync/README.md @@ -177,13 +177,254 @@ Cron-Step läuft täglich. - Recurring-Events: Begrenzte Unterstützung. - Performance: Bei vielen Terminen Paginierung prüfen. -## Letzte Änderungen +## Audit und Management Tool (`audit_calendar_sync.py`) -- Refaktorierung zu event-driven Design ohne PostgreSQL Hub. -- Fixes für mehrtägige Termine: Korrekte Verwendung von `datumBis`. -- Entfernung 24h-Limit; Google Calendar unterstützt lange Events. -- Per-Employee Locking mit Redis. -- Logging via context.logger für Workbench. -- Neue Schritte: calendar_sync_all_step.py, calendar_sync_cron_step.py. -- Workbench-Gruppierung: "advoware-calendar-sync". +Das `audit_calendar_sync.py` Tool bietet umfassende Audit-, Management- und Debugging-Funktionen für die Calendar-Synchronisation. Es ermöglicht die Überprüfung der Sync-Integrität, das Aufräumen von Duplikaten und verwaisten Einträgen sowie detaillierte Abfragen einzelner Termine. + +### Verwendung + +```bash +cd /opt/motia-app/bitbylaw +source python_modules/bin/activate +python steps/advoware_cal_sync/audit_calendar_sync.py [options] +``` + +### Befehle + +#### `audit [--delete-orphaned-google]` + +Auditiert Sync-Einträge für einen spezifischen Mitarbeiter und prüft deren Existenz in beiden Systemen. + +**Parameter:** +- `employee_kuerzel`: Mitarbeiter-Kürzel (z.B. "SB", "UR") +- `google|advoware`: System, das auditiert werden soll +- `--delete-orphaned-google`: Optional, löscht Google-Events die in Google existieren aber nicht in der DB + +**Beispiel:** +```bash +# Audit Google Calendar für Mitarbeiter SB +python audit_calendar_sync.py audit SB google + +# Audit Advoware für Mitarbeiter UR mit Löschung verwaister Google-Events +python audit_calendar_sync.py audit UR google --delete-orphaned-google +``` + +**Ausgabe:** +- Anzahl der DB-Einträge +- Anzahl der Events im Zielsystem +- Anzahl existierender/verwaiste Einträge +- Details zu verwaisten Einträgen + +#### `delete-calendar ` + +Löscht den Google Calendar für einen spezifischen Mitarbeiter (falls vorhanden). + +**Beispiel:** +```bash +python audit_calendar_sync.py delete-calendar SB +``` + +#### `list-all` + +Listet alle Google Calendars auf, einschließlich Name, ID, Primary-Status und Access-Role. + +**Beispiel:** +```bash +python audit_calendar_sync.py list-all +``` + +**Ausgabe:** +``` +=== All Google Calendars (27) === + AW-SB (ID: abc123@group.calendar.google.com, Primary: False, Access: owner) + AW-UR (ID: def456@group.calendar.google.com, Primary: False, Access: owner) + ... +``` + +#### `find-duplicates` + +Findet duplizierte Google Calendars nach Namen. + +**Beispiel:** +```bash +python audit_calendar_sync.py find-duplicates +``` + +**Ausgabe:** +``` +=== Duplicate Calendars Found (2 unique names with duplicates) === +Total duplicate calendars: 3 + +Calendar Name: 'AW-SB' - 2 instances + ID: abc123@group.calendar.google.com, Primary: False, Access Role: owner + ID: xyz789@group.calendar.google.com, Primary: False, Access Role: owner +``` + +#### `delete-duplicates` + +Findet und löscht duplizierte Calendars (behält jeweils einen pro Namen). + +**Beispiel:** +```bash +python audit_calendar_sync.py delete-duplicates +``` + +#### `find-orphaned` + +Findet AW-* Calendars ohne entsprechende Mitarbeiter in der Datenbank. + +**Beispiel:** +```bash +python audit_calendar_sync.py find-orphaned +``` + +#### `cleanup-orphaned` + +Findet und löscht verwaiste AW-* Calendars. + +**Beispiel:** +```bash +python audit_calendar_sync.py cleanup-orphaned +``` + +#### `query-frnr ` + +Zeigt alle Sync-Informationen für eine spezifische Advoware frNr. + +**Beispiel:** +```bash +python audit_calendar_sync.py query-frnr 79291 +``` + +**Ausgabe:** +``` +=== Sync Information for frNr: 79291 === +Found 1 sync entry + +Sync ID: 6ee9ba95-8aff-4868-9171-c10a8789427c + Employee: UR + Advoware frNr: 79291 + Google Event ID: jao7r00j26lt1i0chk454bi9as + Source System: advoware + Sync Strategy: source_system_wins + Sync Status: synced + Last Sync: 2025-10-24 23:30:17.692668+00:00 + Created: 2025-10-24 07:22:41.729295+00:00 + Updated: 2025-10-24 07:22:41.729295+00:00 +``` + +#### `query-event ` + +Zeigt Sync-Informationen für eine spezifische Google Event ID. + +**Beispiel:** +```bash +python audit_calendar_sync.py query-event jao7r00j26lt1i0chk454bi9as +``` + +#### `verify-sync ` + +Vollständige Sync-Verifikation: Prüft einen Termin in beiden Systemen (Advoware und Google Calendar). + +**Beispiel:** +```bash +python audit_calendar_sync.py verify-sync 79291 +``` + +**Ausgabe:** +``` +=== Sync Verification for frNr: 79291 === +Employee: UR +Sync Status: synced +Last Sync: 2025-10-24 23:30:17.692668+00:00 + +--- Checking Advoware --- +✅ Found in Advoware: + Subject: Jour fixe iS Neomi - Teilnahme im Einzelfall + Date: 2024-06-04T17:00:00 + Time: N/A + End Time: 19:00:00 + End Date: 2026-02-03T00:00:00 + Last Modified: 2025-09-29T11:55:43.624 + frNr: 79291 + +--- Checking Google Calendar --- +✅ Found in Google Calendar: + Summary: Advoware (frNr: 79291) + Start: 2024-06-04T17:00:00+02:00 + End: 2024-06-04T19:00:00+02:00 + +--- Sync Status Summary --- +✅ Synchronized: Exists in both systems +``` + +### Technische Details + +#### Datenbank-Integration +- Verwendet PostgreSQL-Verbindung aus `config.py` +- Tabelle: `calendar_sync` +- Felder: `sync_id`, `employee_kuerzel`, `advoware_frnr`, `google_event_id`, etc. + +#### API-Integration +- **Google Calendar API**: `calendarList().list()` mit Paginierung (maxResults=250) +- **Advoware API**: `GET /api/v1/advonet/Termine` mit `frnr` Filter +- Automatische Token-Verwaltung und Fehlerbehandlung + +#### Sicherheit +- Verwendet bestehende Service-Account und API-Credentials +- Keine zusätzlichen Berechtigungen erforderlich + +### Häufige Anwendungsfälle + +#### 1. Nach der Erstinstallation +```bash +# Alle Calendars auflisten +python audit_calendar_sync.py list-all + +# Duplikate finden und entfernen +python audit_calendar_sync.py find-duplicates +python audit_calendar_sync.py delete-duplicates + +# Verwaiste Calendars entfernen +python audit_calendar_sync.py find-orphaned +python audit_calendar_sync.py cleanup-orphaned +``` + +#### 2. Bei Sync-Problemen +```bash +# Sync-Status für einen Mitarbeiter prüfen +python audit_calendar_sync.py audit SB google + +# Einzelnen Termin verifizieren +python audit_calendar_sync.py verify-sync 79291 + +# Sync-Informationen abfragen +python audit_calendar_sync.py query-frnr 79291 +``` + +#### 3. Regelmäßige Wartung +```bash +# Wöchentliche Überprüfung auf Duplikate +python audit_calendar_sync.py find-duplicates + +# Monatliche Bereinigung verwaister Einträge +python audit_calendar_sync.py cleanup-orphaned +``` + +### Fehlerbehandlung + +- **API-Fehler**: Automatische Retry-Logik mit Backoff +- **Berechtigungsfehler**: Klare Fehlermeldungen mit Lösungsvorschlägen +- **Netzwerkprobleme**: Timeout-Handling und Wiederholungen +- **Dateninkonsistenzen**: Detaillierte Logging für Debugging + +### Performance + +- **Paginierung**: Automatische Handhabung großer Resultsets +- **Batch-Verarbeitung**: Effiziente API-Calls mit minimalen Requests +- **Caching**: Wiederverwendung von API-Verbindungen wo möglich + +### Logging + +Alle Operationen werden über `context.logger` geloggt und sind in der Motia Workbench sichtbar. Zusätzliche Debug-Informationen werden auf der Konsole ausgegeben. diff --git a/bitbylaw/steps/advoware_cal_sync/audit_calendar_sync.py b/bitbylaw/steps/advoware_cal_sync/audit_calendar_sync.py index 000a23f4..35e2910c 100644 --- a/bitbylaw/steps/advoware_cal_sync/audit_calendar_sync.py +++ b/bitbylaw/steps/advoware_cal_sync/audit_calendar_sync.py @@ -513,12 +513,12 @@ async def query_frnr(frnr): rows = await conn.fetch( """ SELECT sync_id, employee_kuerzel, advoware_frnr, google_event_id, - source_system, sync_strategy, sync_status, last_sync, created_at, updated_at + source_system, sync_strategy, sync_status, last_sync, created_at FROM calendar_sync WHERE advoware_frnr = $1 ORDER BY sync_id """, - str(frnr) + int(frnr) ) if not rows: @@ -538,7 +538,7 @@ async def query_frnr(frnr): print(f" Sync Status: {row['sync_status']}") print(f" Last Sync: {row['last_sync']}") print(f" Created: {row['created_at']}") - print(f" Updated: {row['updated_at']}") + print(f" Updated: {row['created_at']}") # Using created_at as updated_at doesn't exist finally: await conn.close() @@ -551,7 +551,7 @@ async def query_event(event_id): row = await conn.fetchrow( """ SELECT sync_id, employee_kuerzel, advoware_frnr, google_event_id, - source_system, sync_strategy, sync_status, last_sync, created_at, updated_at + source_system, sync_strategy, sync_status, last_sync, created_at FROM calendar_sync WHERE google_event_id = $1 """, @@ -572,7 +572,7 @@ async def query_event(event_id): print(f" Sync Status: {row['sync_status']}") print(f" Last Sync: {row['last_sync']}") print(f" Created: {row['created_at']}") - print(f" Updated: {row['updated_at']}") + print(f" Updated: {row['created_at']}") # Using created_at as updated_at doesn't exist finally: await conn.close() @@ -589,7 +589,7 @@ async def verify_sync(frnr, service, advoware_api): FROM calendar_sync WHERE advoware_frnr = $1 AND deleted = FALSE """, - str(frnr) + int(frnr) ) if not row: @@ -607,23 +607,45 @@ async def verify_sync(frnr, service, advoware_api): # Check Advoware print(f"\n--- Checking Advoware ---") try: + # Use frNr with a broad date range to query the appointment advoware_result = await advoware_api.api_call( 'api/v1/advonet/Termine', method='GET', params={ 'kuerzel': employee_kuerzel, - 'frNr': str(frnr) + 'frnr': int(frnr), # Use lowercase 'frnr' as per API docs + 'from': '2000-01-01T00:00:00', + 'to': '2030-12-31T23:59:59' } ) - if advoware_result and len(advoware_result) > 0: - appointment = advoware_result[0] + + # API returns a list, find the specific appointment + target_appointment = None + if isinstance(advoware_result, list): + for appointment in advoware_result: + if str(appointment.get('frNr', '')) == str(frnr): + target_appointment = appointment + break + + if target_appointment: print("✅ Found in Advoware:") - print(f" Subject: {appointment.get('betreff', 'N/A')}") - print(f" Date: {appointment.get('datum', 'N/A')}") - print(f" Time: {appointment.get('uhrzeit', 'N/A')}") + print(f" Subject: {target_appointment.get('text', 'N/A')}") + print(f" Date: {target_appointment.get('datum', 'N/A')}") + print(f" Time: {target_appointment.get('uhrzeitVon', 'N/A')}") + print(f" End Time: {target_appointment.get('uhrzeitBis', 'N/A')}") + print(f" End Date: {target_appointment.get('datumBis', 'N/A')}") + print(f" Last Modified: {target_appointment.get('zuletztGeaendertAm', 'N/A')}") + print(f" frNr: {target_appointment.get('frNr', 'N/A')}") advoware_exists = True else: - print("❌ Not found in Advoware") + print(f"❌ Not found in Advoware (checked {len(advoware_result) if isinstance(advoware_result, list) else 0} appointments)") + # Show first few appointments for debugging (limited to 5) + if isinstance(advoware_result, list) and len(advoware_result) > 0: + print(" First few appointments returned:") + for i, app in enumerate(advoware_result[:5]): + print(f" [{i}] Subject: {app.get('text', 'N/A')}") + print(f" Date: {app.get('datum', 'N/A')}") + print(f" frNr: {app.get('frNr', 'N/A')}") advoware_exists = False except Exception as e: print(f"❌ Error checking Advoware: {e}") diff --git a/motia-workbench.json b/motia-workbench.json index 05195cfb..9979843d 100644 --- a/motia-workbench.json +++ b/motia-workbench.json @@ -28,5 +28,31 @@ "sourceHandlePosition": "right" } } + }, + { + "id": "advoware-calendar-sync", + "config": { + "steps/advoware_cal_sync/calendar_sync_cron_step.py": { + "x": 0, + "y": 0, + "sourceHandlePosition": "right" + }, + "steps/advoware_cal_sync/calendar_sync_api_step.py": { + "x": 0, + "y": 100, + "sourceHandlePosition": "right" + }, + "steps/advoware_cal_sync/calendar_sync_all_step.py": { + "x": 300, + "y": 50, + "targetHandlePosition": "left", + "sourceHandlePosition": "right" + }, + "steps/advoware_cal_sync/calendar_sync_event_step.py": { + "x": 600, + "y": 50, + "targetHandlePosition": "left" + } + } } ] diff --git a/query_db.py b/query_db.py new file mode 100644 index 00000000..e69de29b