feat(sync): Implement auto-reset for permanently_failed entities and add retry backoff logic

- Added logic to reset permanently_failed entities that have reached their auto-reset threshold in `beteiligte_sync_cron_step.py`. - Enhanced event handling in `beteiligte_sync_event_step.py` to skip retries if the next retry time has not been reached. - Introduced validation checks after sync operations to ensure data consistency and integrity. - Created detailed documentation outlining the fixes and their impacts on the sync process. - Added scripts for analyzing sync issues and comparing entities to facilitate debugging and validation.
2026-02-08 21:12:00 +00:00
parent 6e0e9a9730
commit 79e097be6f
8 changed files with 1135 additions and 30 deletions
--- a/bitbylaw/steps/vmh/beteiligte_sync_cron_step.py
+++ b/bitbylaw/steps/vmh/beteiligte_sync_cron_step.py
@@ -55,6 +55,38 @@ async def handler(context):
        
        context.logger.info(f"📊 Gefunden: {len(unclean_entities)} Entities mit Status pending/dirty/failed")
        
+        # FIX #12: QUERY 1b: permanently_failed Entities die Auto-Reset erreicht haben
+        permanently_failed_filter = {
+            'where': [
+                {
+                    'type': 'and',
+                    'value': [
+                        {'type': 'equals', 'attribute': 'syncStatus', 'value': 'permanently_failed'},
+                        {'type': 'isNotNull', 'attribute': 'syncAutoResetAt'},
+                        {'type': 'before', 'attribute': 'syncAutoResetAt', 'value': threshold_str}
+                    ]
+                }
+            ]
+        }
+        
+        reset_result = await espocrm.search_entities('CBeteiligte', permanently_failed_filter, max_size=50)
+        reset_entities = reset_result.get('list', [])
+        
+        # Reset permanently_failed entities
+        for entity in reset_entities:
+            entity_id = entity['id']
+            context.logger.info(f"🔄 Auto-Reset für permanently_failed Entity {entity_id}")
+            
+            # Reset Status und Retry-Count
+            await espocrm.update_entity('CBeteiligte', entity_id, {
+                'syncStatus': 'failed',  # Zurück zu 'failed' für normalen Retry
+                'syncRetryCount': 0,
+                'syncAutoResetAt': None,
+                'syncErrorMessage': f"Auto-Reset nach 24h - vorheriger Fehler: {entity.get('syncErrorMessage', 'N/A')}"
+            })
+        
+        context.logger.info(f"📊 Auto-Reset: {len(reset_entities)} permanently_failed Entities")
+        
        # QUERY 2: Clean Entities die > 24h nicht gesynct wurden
        stale_filter = {
            'where': [
@@ -80,8 +112,8 @@ async def handler(context):
        
        context.logger.info(f"📊 Gefunden: {len(stale_entities)} Entities mit veraltetem Sync (> 24h)")
        
-        # KOMBINIERE ALLE
-        all_entities = unclean_entities + stale_entities
+        # KOMBINIERE ALLE (inkl. reset_entities)
+        all_entities = unclean_entities + stale_entities + reset_entities
        entity_ids = list(set([e['id'] for e in all_entities]))  # Dedupliziere
        
        context.logger.info(f"🎯 Total: {len(entity_ids)} eindeutige Entities zum Sync")
--- a/bitbylaw/steps/vmh/beteiligte_sync_event_step.py
+++ b/bitbylaw/steps/vmh/beteiligte_sync_event_step.py
@@ -85,6 +85,25 @@ async def handler(event_data, context):
        betnr = espo_entity.get('betnr')
        sync_status = espo_entity.get('syncStatus', 'pending_sync')
        
+        # FIX #12: Check Retry-Backoff - überspringe wenn syncNextRetry noch nicht erreicht
+        sync_next_retry = espo_entity.get('syncNextRetry')
+        if sync_next_retry and sync_status == 'failed':
+            import datetime
+            import pytz
+            
+            try:
+                next_retry_ts = datetime.datetime.strptime(sync_next_retry, '%Y-%m-%d %H:%M:%S')
+                next_retry_ts = pytz.UTC.localize(next_retry_ts)
+                now_utc = datetime.datetime.now(pytz.UTC)
+                
+                if now_utc < next_retry_ts:
+                    remaining_minutes = int((next_retry_ts - now_utc).total_seconds() / 60)
+                    context.logger.info(f"⏸️  Retry-Backoff aktiv: Nächster Versuch in {remaining_minutes} Minuten")
+                    await sync_utils.release_sync_lock(entity_id, sync_status)
+                    return
+            except Exception as e:
+                context.logger.warn(f"⚠️  Fehler beim Parsen von syncNextRetry: {e}")
+        
        # 3. BESTIMME SYNC-AKTION
        
        # FALL A: Neu (kein betnr) → CREATE in Advoware
@@ -273,6 +292,21 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
            
            context.logger.info(f"✅ Advoware STAMMDATEN aktualisiert, rowId: {new_rowid[:20] if new_rowid else 'N/A'}...")
            
+            # FIX #13: Validiere Sync-Ergebnis
+            validation_success, validation_error = await sync_utils.validate_sync_result(
+                entity_id, betnr, mapper, direction='to_advoware'
+            )
+            
+            if not validation_success:
+                context.logger.error(f"❌ Sync-Validation fehlgeschlagen: {validation_error}")
+                await sync_utils.release_sync_lock(
+                    entity_id, 
+                    'failed',
+                    error_message=f"Validation failed: {validation_error}",
+                    increment_retry=True
+                )
+                return
+            
            # KOMMUNIKATION SYNC: Immer ausführen nach Stammdaten-Update
            await run_kommunikation_sync(entity_id, betnr, komm_sync, context)
            
@@ -291,6 +325,21 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
            await espocrm.update_entity('CBeteiligte', entity_id, espo_data)
            context.logger.info(f"✅ EspoCRM STAMMDATEN aktualisiert")
            
+            # FIX #13: Validiere Sync-Ergebnis
+            validation_success, validation_error = await sync_utils.validate_sync_result(
+                entity_id, betnr, mapper, direction='to_espocrm'
+            )
+            
+            if not validation_success:
+                context.logger.error(f"❌ Sync-Validation fehlgeschlagen: {validation_error}")
+                await sync_utils.release_sync_lock(
+                    entity_id, 
+                    'failed',
+                    error_message=f"Validation failed: {validation_error}",
+                    increment_retry=True
+                )
+                return
+            
            # KOMMUNIKATION SYNC: Immer ausführen nach Stammdaten-Update
            await run_kommunikation_sync(entity_id, betnr, komm_sync, context)
            
@@ -327,6 +376,23 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
                f"EspoCRM hat gewonnen."
            )
            
+            context.logger.info(f"✅ Konflikt gelöst (EspoCRM won), neue rowId: {new_rowid[:20] if new_rowid else 'N/A'}...")
+            
+            # FIX #13: Validiere Sync-Ergebnis
+            validation_success, validation_error = await sync_utils.validate_sync_result(
+                entity_id, betnr, mapper, direction='to_advoware'
+            )
+            
+            if not validation_success:
+                context.logger.error(f"❌ Conflict resolution validation fehlgeschlagen: {validation_error}")
+                await sync_utils.release_sync_lock(
+                    entity_id, 
+                    'failed',
+                    error_message=f"Conflict resolution validation failed: {validation_error}",
+                    increment_retry=True
+                )
+                return
+            
            await sync_utils.resolve_conflict_espocrm_wins(
                entity_id,
                espo_entity,
@@ -334,7 +400,6 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
                conflict_msg,
                extra_fields={'advowareRowId': new_rowid}
            )
-            context.logger.info(f"✅ Konflikt gelöst (EspoCRM won), neue rowId: {new_rowid[:20] if new_rowid else 'N/A'}...")
            
            # KOMMUNIKATION SYNC: NUR EspoCRM→Advoware (EspoCRM wins!)
            await run_kommunikation_sync(entity_id, betnr, komm_sync, context, direction='to_advoware')