Add comprehensive test scripts for thumbnail generation and xAI collections API
- Implemented `test_thumbnail_generation.py` to validate the complete flow of document thumbnail generation in EspoCRM, including document creation, file upload, webhook triggering, and preview verification. - Created `test_xai_collections_api.py` to test critical operations of the xAI Collections API, covering file uploads, collection CRUD operations, document management, and response validation. - Both scripts include detailed logging for success and error states, ensuring robust testing and easier debugging.
This commit is contained in:
150
services/sync_utils_base.py
Normal file
150
services/sync_utils_base.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
Base Sync Utilities
|
||||
|
||||
Gemeinsame Funktionalität für alle Sync-Operationen:
|
||||
- Redis Distributed Locking
|
||||
- Context-aware Logging
|
||||
- EspoCRM API Helpers
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import redis
|
||||
import os
|
||||
import pytz
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Lock TTL in seconds (prevents deadlocks)
|
||||
LOCK_TTL_SECONDS = 900 # 15 minutes
|
||||
|
||||
|
||||
class BaseSyncUtils:
|
||||
"""Base-Klasse mit gemeinsamer Sync-Funktionalität"""
|
||||
|
||||
def __init__(self, espocrm_api, redis_client: redis.Redis = None, context=None):
|
||||
"""
|
||||
Args:
|
||||
espocrm_api: EspoCRM API client instance
|
||||
redis_client: Optional Redis client (wird sonst initialisiert)
|
||||
context: Optional Motia FlowContext für Logging
|
||||
"""
|
||||
self.espocrm = espocrm_api
|
||||
self.context = context
|
||||
self.logger = context.logger if context else logger
|
||||
self.redis = redis_client or self._init_redis()
|
||||
|
||||
def _init_redis(self) -> Optional[redis.Redis]:
|
||||
"""Initialize Redis client for distributed locking"""
|
||||
try:
|
||||
redis_host = os.getenv('REDIS_HOST', 'localhost')
|
||||
redis_port = int(os.getenv('REDIS_PORT', '6379'))
|
||||
redis_db = int(os.getenv('REDIS_DB_ADVOWARE_CACHE', '1'))
|
||||
|
||||
client = redis.Redis(
|
||||
host=redis_host,
|
||||
port=redis_port,
|
||||
db=redis_db,
|
||||
decode_responses=True
|
||||
)
|
||||
client.ping()
|
||||
return client
|
||||
except Exception as e:
|
||||
self._log(f"Redis connection failed: {e}", level='error')
|
||||
return None
|
||||
|
||||
def _log(self, message: str, level: str = 'info'):
|
||||
"""
|
||||
Context-aware logging
|
||||
|
||||
Falls ein FlowContext vorhanden ist, wird dessen Logger verwendet.
|
||||
Sonst fallback auf Standard-Logger.
|
||||
"""
|
||||
if self.context and hasattr(self.context, 'logger'):
|
||||
getattr(self.context.logger, level)(message)
|
||||
else:
|
||||
getattr(logger, level)(message)
|
||||
|
||||
def _get_lock_key(self, entity_id: str) -> str:
|
||||
"""
|
||||
Erzeugt Redis Lock-Key für eine Entity
|
||||
|
||||
Muss in Subklassen überschrieben werden, um entity-spezifische Prefixes zu nutzen.
|
||||
z.B. 'sync_lock:cbeteiligte:{entity_id}' oder 'sync_lock:document:{entity_id}'
|
||||
"""
|
||||
raise NotImplementedError("Subclass must implement _get_lock_key()")
|
||||
|
||||
def _acquire_redis_lock(self, lock_key: str) -> bool:
|
||||
"""
|
||||
Atomic Redis lock acquisition
|
||||
|
||||
Args:
|
||||
lock_key: Redis key für den Lock
|
||||
|
||||
Returns:
|
||||
True wenn Lock erfolgreich, False wenn bereits locked
|
||||
"""
|
||||
if not self.redis:
|
||||
self._log("Redis nicht verfügbar, Lock-Mechanismus deaktiviert", level='warn')
|
||||
return True # Fallback: Wenn kein Redis, immer lock erlauben
|
||||
|
||||
try:
|
||||
acquired = self.redis.set(lock_key, "locked", nx=True, ex=LOCK_TTL_SECONDS)
|
||||
return bool(acquired)
|
||||
except Exception as e:
|
||||
self._log(f"Redis lock error: {e}", level='error')
|
||||
return True # Bei Fehler: Lock erlauben, um Deadlocks zu vermeiden
|
||||
|
||||
def _release_redis_lock(self, lock_key: str) -> None:
|
||||
"""
|
||||
Redis lock freigeben
|
||||
|
||||
Args:
|
||||
lock_key: Redis key für den Lock
|
||||
"""
|
||||
if not self.redis:
|
||||
return
|
||||
|
||||
try:
|
||||
self.redis.delete(lock_key)
|
||||
except Exception as e:
|
||||
self._log(f"Redis unlock error: {e}", level='error')
|
||||
|
||||
def _get_espocrm_datetime(self, dt: Optional[datetime] = None) -> str:
|
||||
"""
|
||||
Formatiert datetime für EspoCRM (ohne Timezone!)
|
||||
|
||||
Args:
|
||||
dt: Optional datetime object (default: now UTC)
|
||||
|
||||
Returns:
|
||||
String im Format 'YYYY-MM-DD HH:MM:SS'
|
||||
"""
|
||||
if dt is None:
|
||||
dt = datetime.now(pytz.UTC)
|
||||
elif dt.tzinfo is None:
|
||||
dt = pytz.UTC.localize(dt)
|
||||
|
||||
return dt.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
async def acquire_sync_lock(self, entity_id: str, **kwargs) -> bool:
|
||||
"""
|
||||
Erwirbt Sync-Lock für eine Entity
|
||||
|
||||
Muss in Subklassen implementiert werden, um entity-spezifische
|
||||
Status-Updates durchzuführen.
|
||||
|
||||
Returns:
|
||||
True wenn Lock erfolgreich, False wenn bereits locked
|
||||
"""
|
||||
raise NotImplementedError("Subclass must implement acquire_sync_lock()")
|
||||
|
||||
async def release_sync_lock(self, entity_id: str, **kwargs) -> None:
|
||||
"""
|
||||
Gibt Sync-Lock frei und setzt finalen Status
|
||||
|
||||
Muss in Subklassen implementiert werden, um entity-spezifische
|
||||
Status-Updates durchzuführen.
|
||||
"""
|
||||
raise NotImplementedError("Subclass must implement release_sync_lock()")
|
||||
Reference in New Issue
Block a user