Refactor and enhance logging in webhook handlers and Redis client
- Translated comments and docstrings from German to English for better clarity. - Improved logging consistency across various webhook handlers for create, delete, and update operations. - Centralized logging functionality by utilizing a dedicated logger utility. - Added new enums for file and XAI sync statuses in models. - Updated Redis client factory to use a centralized logger and improved error handling. - Enhanced API responses to include more descriptive messages and status codes.
This commit is contained in:
bsiggel
@@ -1,17 +1,18 @@
|
||||
"""
|
||||
Document Sync Utilities
|
||||
|
||||
Hilfsfunktionen für Document-Synchronisation mit xAI:
|
||||
Utility functions for document synchronization with xAI:
|
||||
- Distributed locking via Redis + syncStatus
|
||||
- Entscheidungslogik: Wann muss ein Document zu xAI?
|
||||
- Related Entities ermitteln (Many-to-Many Attachments)
|
||||
- xAI Collection Management
|
||||
- Decision logic: When does a document need xAI sync?
|
||||
- Related entities determination (Many-to-Many attachments)
|
||||
- xAI Collection management
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from services.sync_utils_base import BaseSyncUtils
|
||||
from services.models import FileStatus, XAISyncStatus
|
||||
|
||||
# Max retry before permanent failure
|
||||
MAX_SYNC_RETRIES = 5
|
||||
@@ -21,10 +22,10 @@ RETRY_BACKOFF_MINUTES = [1, 5, 15, 60, 240] # 1min, 5min, 15min, 1h, 4h
|
||||
|
||||
|
||||
class DocumentSync(BaseSyncUtils):
|
||||
"""Utility-Klasse für Document-Synchronisation mit xAI"""
|
||||
"""Utility class for document synchronization with xAI"""
|
||||
|
||||
def _get_lock_key(self, entity_id: str) -> str:
|
||||
"""Redis Lock-Key für Documents"""
|
||||
"""Redis lock key for documents"""
|
||||
return f"sync_lock:document:{entity_id}"
|
||||
|
||||
async def acquire_sync_lock(self, entity_id: str, entity_type: str = 'CDokumente') -> bool:
|
||||
@@ -45,13 +46,13 @@ class DocumentSync(BaseSyncUtils):
|
||||
self._log(f"Redis lock bereits aktiv für {entity_type} {entity_id}", level='warn')
|
||||
return False
|
||||
|
||||
# STEP 2: Update xaiSyncStatus auf pending_sync
|
||||
# STEP 2: Update xaiSyncStatus to pending_sync
|
||||
try:
|
||||
await self.espocrm.update_entity(entity_type, entity_id, {
|
||||
'xaiSyncStatus': 'pending_sync'
|
||||
'xaiSyncStatus': XAISyncStatus.PENDING_SYNC.value
|
||||
})
|
||||
except Exception as e:
|
||||
self._log(f"Konnte xaiSyncStatus nicht setzen: {e}", level='debug')
|
||||
self._log(f"Could not set xaiSyncStatus: {e}", level='debug')
|
||||
|
||||
self._log(f"Sync-Lock für {entity_type} {entity_id} erworben")
|
||||
return True
|
||||
@@ -84,16 +85,16 @@ class DocumentSync(BaseSyncUtils):
|
||||
try:
|
||||
update_data = {}
|
||||
|
||||
# xaiSyncStatus setzen: clean bei Erfolg, failed bei Fehler
|
||||
# Set xaiSyncStatus: clean on success, failed on error
|
||||
try:
|
||||
update_data['xaiSyncStatus'] = 'clean' if success else 'failed'
|
||||
update_data['xaiSyncStatus'] = XAISyncStatus.CLEAN.value if success else XAISyncStatus.FAILED.value
|
||||
|
||||
if error_message:
|
||||
update_data['xaiSyncError'] = error_message[:2000]
|
||||
else:
|
||||
update_data['xaiSyncError'] = None
|
||||
except:
|
||||
pass # Felder existieren evtl. nicht
|
||||
pass # Fields may not exist
|
||||
|
||||
# Merge extra fields (z.B. xaiFileId, xaiCollections)
|
||||
if extra_fields:
|
||||
@@ -120,37 +121,37 @@ class DocumentSync(BaseSyncUtils):
|
||||
entity_type: str = 'CDokumente'
|
||||
) -> Tuple[bool, List[str], str]:
|
||||
"""
|
||||
Entscheidet ob ein Document zu xAI synchronisiert werden muss
|
||||
Decide if a document needs to be synchronized to xAI.
|
||||
|
||||
Prüft:
|
||||
1. Datei-Status Feld ("Neu", "Geändert")
|
||||
2. Hash-Werte für Change Detection
|
||||
3. Related Entities mit xAI Collections
|
||||
Checks:
|
||||
1. File status field ("new", "changed")
|
||||
2. Hash values for change detection
|
||||
3. Related entities with xAI collections
|
||||
|
||||
Args:
|
||||
document: Vollständiges Document Entity von EspoCRM
|
||||
document: Complete document entity from EspoCRM
|
||||
|
||||
Returns:
|
||||
Tuple[bool, List[str], str]:
|
||||
- bool: Ob Sync nötig ist
|
||||
- List[str]: Liste der Collection-IDs in die das Document soll
|
||||
- str: Grund/Beschreibung der Entscheidung
|
||||
- bool: Whether sync is needed
|
||||
- List[str]: List of collection IDs where the document should go
|
||||
- str: Reason/description of the decision
|
||||
"""
|
||||
doc_id = document.get('id')
|
||||
doc_name = document.get('name', 'Unbenannt')
|
||||
|
||||
# xAI-relevante Felder
|
||||
# xAI-relevant fields
|
||||
xai_file_id = document.get('xaiFileId')
|
||||
xai_collections = document.get('xaiCollections') or []
|
||||
xai_sync_status = document.get('xaiSyncStatus')
|
||||
|
||||
# Datei-Status und Hash-Felder
|
||||
# File status and hash fields
|
||||
datei_status = document.get('dateiStatus') or document.get('fileStatus')
|
||||
file_md5 = document.get('md5') or document.get('fileMd5')
|
||||
file_sha = document.get('sha') or document.get('fileSha')
|
||||
xai_synced_hash = document.get('xaiSyncedHash') # Hash beim letzten xAI-Sync
|
||||
xai_synced_hash = document.get('xaiSyncedHash') # Hash at last xAI sync
|
||||
|
||||
self._log(f"📋 Document Analysis: {doc_name} (ID: {doc_id})")
|
||||
self._log(f"📋 Document analysis: {doc_name} (ID: {doc_id})")
|
||||
self._log(f" xaiFileId: {xai_file_id or 'N/A'}")
|
||||
self._log(f" xaiCollections: {xai_collections}")
|
||||
self._log(f" xaiSyncStatus: {xai_sync_status or 'N/A'}")
|
||||
@@ -165,65 +166,74 @@ class DocumentSync(BaseSyncUtils):
|
||||
entity_type=entity_type
|
||||
)
|
||||
|
||||
# Prüfe xaiSyncStatus="no_sync" → kein Sync für dieses Dokument
|
||||
if xai_sync_status == 'no_sync':
|
||||
self._log("⏭️ Kein xAI-Sync nötig: xaiSyncStatus='no_sync'")
|
||||
return (False, [], "xaiSyncStatus ist 'no_sync'")
|
||||
# Check xaiSyncStatus="no_sync" -> no sync for this document
|
||||
if xai_sync_status == XAISyncStatus.NO_SYNC.value:
|
||||
self._log("⏭️ No xAI sync needed: xaiSyncStatus='no_sync'")
|
||||
return (False, [], "xaiSyncStatus is 'no_sync'")
|
||||
|
||||
if not target_collections:
|
||||
self._log("⏭️ Kein xAI-Sync nötig: Keine Related Entities mit xAI Collections")
|
||||
return (False, [], "Keine verknüpften Entities mit xAI Collections")
|
||||
self._log("⏭️ No xAI sync needed: No related entities with xAI collections")
|
||||
return (False, [], "No linked entities with xAI collections")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# PRIORITY CHECK 1: xaiSyncStatus="unclean" → Dokument wurde geändert
|
||||
# PRIORITY CHECK 1: xaiSyncStatus="unclean" -> document was changed
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
if xai_sync_status == 'unclean':
|
||||
self._log(f"🆕 xaiSyncStatus='unclean' → xAI-Sync ERFORDERLICH")
|
||||
if xai_sync_status == XAISyncStatus.UNCLEAN.value:
|
||||
self._log(f"🆕 xaiSyncStatus='unclean' → xAI sync REQUIRED")
|
||||
return (True, target_collections, "xaiSyncStatus='unclean'")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# PRIORITY CHECK 2: fileStatus "new" oder "changed"
|
||||
# PRIORITY CHECK 2: fileStatus "new" or "changed"
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
if datei_status in ['new', 'changed', 'neu', 'geändert', 'New', 'Changed', 'Neu', 'Geändert']:
|
||||
self._log(f"🆕 fileStatus: '{datei_status}' → xAI-Sync ERFORDERLICH")
|
||||
if datei_status in [
|
||||
FileStatus.NEW.value,
|
||||
FileStatus.CHANGED.value,
|
||||
'neu', # Legacy German values
|
||||
'geändert', # Legacy German values
|
||||
'Neu', # Case variations
|
||||
'Geändert',
|
||||
'New',
|
||||
'Changed'
|
||||
]:
|
||||
self._log(f"🆕 fileStatus: '{datei_status}' → xAI sync REQUIRED")
|
||||
|
||||
if target_collections:
|
||||
return (True, target_collections, f"fileStatus: {datei_status}")
|
||||
else:
|
||||
# Datei ist neu/geändert aber keine Collections gefunden
|
||||
self._log(f"⚠️ fileStatus '{datei_status}' aber keine Collections gefunden - überspringe Sync")
|
||||
return (False, [], f"fileStatus: {datei_status}, aber keine Collections")
|
||||
# File is new/changed but no collections found
|
||||
self._log(f"⚠️ fileStatus '{datei_status}' but no collections found - skipping sync")
|
||||
return (False, [], f"fileStatus: {datei_status}, but no collections")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# FALL 1: Document ist bereits in xAI UND Collections sind gesetzt
|
||||
# CASE 1: Document is already in xAI AND collections are set
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
if xai_file_id:
|
||||
self._log(f"✅ Document bereits in xAI gesynct mit {len(target_collections)} Collection(s)")
|
||||
self._log(f"✅ Document already synced to xAI with {len(target_collections)} collection(s)")
|
||||
|
||||
# Prüfe ob File-Inhalt geändert wurde (Hash-Vergleich)
|
||||
# Check if file content was changed (hash comparison)
|
||||
current_hash = file_md5 or file_sha
|
||||
|
||||
if current_hash and xai_synced_hash:
|
||||
if current_hash != xai_synced_hash:
|
||||
self._log(f"🔄 Hash-Änderung erkannt! RESYNC erforderlich")
|
||||
self._log(f" Alt: {xai_synced_hash[:16]}...")
|
||||
self._log(f" Neu: {current_hash[:16]}...")
|
||||
return (True, target_collections, "File-Inhalt geändert (Hash-Mismatch)")
|
||||
self._log(f"🔄 Hash change detected! RESYNC required")
|
||||
self._log(f" Old: {xai_synced_hash[:16]}...")
|
||||
self._log(f" New: {current_hash[:16]}...")
|
||||
return (True, target_collections, "File content changed (hash mismatch)")
|
||||
else:
|
||||
self._log(f"✅ Hash identisch - keine Änderung")
|
||||
self._log(f"✅ Hash identical - no change")
|
||||
else:
|
||||
self._log(f"⚠️ Keine Hash-Werte verfügbar für Vergleich")
|
||||
self._log(f"⚠️ No hash values available for comparison")
|
||||
|
||||
return (False, target_collections, "Bereits gesynct, keine Änderung erkannt")
|
||||
return (False, target_collections, "Already synced, no change detected")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# FALL 2: Document hat xaiFileId aber Collections ist leer/None
|
||||
# CASE 2: Document has xaiFileId but collections is empty/None
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# FALL 3: Collections vorhanden aber kein Status/Hash-Trigger
|
||||
# CASE 3: Collections present but no status/hash trigger
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
self._log(f"✅ Document ist mit {len(target_collections)} Entity/ies verknüpft die Collections haben")
|
||||
return (True, target_collections, "Verknüpft mit Entities die Collections benötigen")
|
||||
self._log(f"✅ Document is linked to {len(target_collections)} entity/ies with collections")
|
||||
return (True, target_collections, "Linked to entities that require collections")
|
||||
|
||||
async def _get_required_collections_from_relations(
|
||||
self,
|
||||
@@ -231,20 +241,20 @@ class DocumentSync(BaseSyncUtils):
|
||||
entity_type: str = 'Document'
|
||||
) -> List[str]:
|
||||
"""
|
||||
Ermittelt alle xAI Collection-IDs von Entities die mit diesem Document verknüpft sind
|
||||
Determine all xAI collection IDs of entities linked to this document.
|
||||
|
||||
EspoCRM Many-to-Many: Document kann mit beliebigen Entities verknüpft sein
|
||||
EspoCRM Many-to-Many: Document can be linked to arbitrary entities
|
||||
(CBeteiligte, Account, CVmhErstgespraech, etc.)
|
||||
|
||||
Args:
|
||||
document_id: Document ID
|
||||
|
||||
Returns:
|
||||
Liste von xAI Collection-IDs (dedupliziert)
|
||||
List of xAI collection IDs (deduplicated)
|
||||
"""
|
||||
collections = set()
|
||||
|
||||
self._log(f"🔍 Prüfe Relations von {entity_type} {document_id}...")
|
||||
self._log(f"🔍 Checking relations of {entity_type} {document_id}...")
|
||||
|
||||
try:
|
||||
entity_def = await self.espocrm.get_entity_def(entity_type)
|
||||
|
||||
Reference in New Issue
Block a user