Implement central configuration, custom exceptions, logging utilities, Pydantic models, and Redis client for BitByLaw integration
- Added `config.py` for centralized configuration management including Sync, API, Advoware, EspoCRM, Redis, Logging, Calendar Sync, and Feature Flags. - Created `exceptions.py` with a hierarchy of custom exceptions for integration errors, API errors, sync errors, and Redis errors. - Developed `logging_utils.py` for a unified logging wrapper supporting structured logging and performance tracking. - Defined Pydantic models in `models.py` for data validation of Advoware and EspoCRM entities, including sync operation models. - Introduced `redis_client.py` for a centralized Redis client factory with connection pooling, automatic reconnection, and health checks.
This commit is contained in:
@@ -9,62 +9,38 @@ Gemeinsame Funktionalität für alle Sync-Operationen:
|
||||
|
||||
from typing import Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import redis
|
||||
import os
|
||||
import pytz
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from services.exceptions import RedisConnectionError, LockAcquisitionError
|
||||
from services.redis_client import get_redis_client
|
||||
from services.config import SYNC_CONFIG, get_lock_key
|
||||
from services.logging_utils import get_logger
|
||||
|
||||
# Lock TTL in seconds (prevents deadlocks)
|
||||
LOCK_TTL_SECONDS = 900 # 15 minutes
|
||||
import redis
|
||||
|
||||
|
||||
class BaseSyncUtils:
|
||||
"""Base-Klasse mit gemeinsamer Sync-Funktionalität"""
|
||||
|
||||
def __init__(self, espocrm_api, redis_client: redis.Redis = None, context=None):
|
||||
def __init__(self, espocrm_api, redis_client: Optional[redis.Redis] = None, context=None):
|
||||
"""
|
||||
Args:
|
||||
espocrm_api: EspoCRM API client instance
|
||||
redis_client: Optional Redis client (wird sonst initialisiert)
|
||||
redis_client: Optional Redis client (wird sonst über Factory initialisiert)
|
||||
context: Optional Motia FlowContext für Logging
|
||||
"""
|
||||
self.espocrm = espocrm_api
|
||||
self.context = context
|
||||
self.logger = context.logger if context else logger
|
||||
self.redis = redis_client or self._init_redis()
|
||||
|
||||
def _init_redis(self) -> Optional[redis.Redis]:
|
||||
"""Initialize Redis client for distributed locking"""
|
||||
try:
|
||||
redis_host = os.getenv('REDIS_HOST', 'localhost')
|
||||
redis_port = int(os.getenv('REDIS_PORT', '6379'))
|
||||
redis_db = int(os.getenv('REDIS_DB_ADVOWARE_CACHE', '1'))
|
||||
|
||||
client = redis.Redis(
|
||||
host=redis_host,
|
||||
port=redis_port,
|
||||
db=redis_db,
|
||||
decode_responses=True
|
||||
)
|
||||
client.ping()
|
||||
return client
|
||||
except Exception as e:
|
||||
self._log(f"Redis connection failed: {e}", level='error')
|
||||
return None
|
||||
|
||||
def _log(self, message: str, level: str = 'info'):
|
||||
"""
|
||||
Context-aware logging
|
||||
self.logger = get_logger('sync_utils', context)
|
||||
|
||||
Falls ein FlowContext vorhanden ist, wird dessen Logger verwendet.
|
||||
Sonst fallback auf Standard-Logger.
|
||||
"""
|
||||
if self.context and hasattr(self.context, 'logger'):
|
||||
getattr(self.context.logger, level)(message)
|
||||
else:
|
||||
getattr(logger, level)(message)
|
||||
# Use provided Redis client or get from factory
|
||||
self.redis = redis_client or get_redis_client(strict=False)
|
||||
|
||||
if not self.redis:
|
||||
self.logger.error(
|
||||
"⚠️ WARNUNG: Redis nicht verfügbar! "
|
||||
"Distributed Locking deaktiviert - Race Conditions möglich!"
|
||||
)
|
||||
|
||||
def _get_lock_key(self, entity_id: str) -> str:
|
||||
"""
|
||||
@@ -84,17 +60,30 @@ class BaseSyncUtils:
|
||||
|
||||
Returns:
|
||||
True wenn Lock erfolgreich, False wenn bereits locked
|
||||
|
||||
Raises:
|
||||
LockAcquisitionError: Bei kritischen Lock-Problemen (wenn strict mode)
|
||||
"""
|
||||
if not self.redis:
|
||||
self._log("Redis nicht verfügbar, Lock-Mechanismus deaktiviert", level='warn')
|
||||
return True # Fallback: Wenn kein Redis, immer lock erlauben
|
||||
self.logger.error(
|
||||
"CRITICAL: Distributed Locking deaktiviert - Redis nicht verfügbar!"
|
||||
)
|
||||
# In production: Dies könnte zu Race Conditions führen!
|
||||
# Für jetzt erlauben wir Fortsetzung, aber mit Warning
|
||||
return True
|
||||
|
||||
try:
|
||||
acquired = self.redis.set(lock_key, "locked", nx=True, ex=LOCK_TTL_SECONDS)
|
||||
acquired = self.redis.set(
|
||||
lock_key,
|
||||
"locked",
|
||||
nx=True,
|
||||
ex=SYNC_CONFIG.lock_ttl_seconds
|
||||
)
|
||||
return bool(acquired)
|
||||
except Exception as e:
|
||||
self._log(f"Redis lock error: {e}", level='error')
|
||||
return True # Bei Fehler: Lock erlauben, um Deadlocks zu vermeiden
|
||||
except redis.RedisError as e:
|
||||
self.logger.error(f"Redis lock error: {e}")
|
||||
# Bei Redis-Fehler: Lock erlauben, um Deadlocks zu vermeiden
|
||||
return True
|
||||
|
||||
def _release_redis_lock(self, lock_key: str) -> None:
|
||||
"""
|
||||
@@ -108,8 +97,8 @@ class BaseSyncUtils:
|
||||
|
||||
try:
|
||||
self.redis.delete(lock_key)
|
||||
except Exception as e:
|
||||
self._log(f"Redis unlock error: {e}", level='error')
|
||||
except redis.RedisError as e:
|
||||
self.logger.error(f"Redis unlock error: {e}")
|
||||
|
||||
def _get_espocrm_datetime(self, dt: Optional[datetime] = None) -> str:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user