feat(sync): Implement auto-reset for permanently_failed entities and add retry backoff logic

- Added logic to reset permanently_failed entities that have reached their auto-reset threshold in `beteiligte_sync_cron_step.py`.
- Enhanced event handling in `beteiligte_sync_event_step.py` to skip retries if the next retry time has not been reached.
- Introduced validation checks after sync operations to ensure data consistency and integrity.
- Created detailed documentation outlining the fixes and their impacts on the sync process.
- Added scripts for analyzing sync issues and comparing entities to facilitate debugging and validation.

bitbylaw/steps/vmh/beteiligte_sync_event_step.py

@@ -85,6 +85,25 @@ async def handler(event_data, context):
         betnr = espo_entity.get('betnr')
         sync_status = espo_entity.get('syncStatus', 'pending_sync')
         
+        # FIX #12: Check Retry-Backoff - überspringe wenn syncNextRetry noch nicht erreicht
+        sync_next_retry = espo_entity.get('syncNextRetry')
+        if sync_next_retry and sync_status == 'failed':
+            import datetime
+            import pytz
+            
+            try:
+                next_retry_ts = datetime.datetime.strptime(sync_next_retry, '%Y-%m-%d %H:%M:%S')
+                next_retry_ts = pytz.UTC.localize(next_retry_ts)
+                now_utc = datetime.datetime.now(pytz.UTC)
+                
+                if now_utc < next_retry_ts:
+                    remaining_minutes = int((next_retry_ts - now_utc).total_seconds() / 60)
+                    context.logger.info(f"⏸️  Retry-Backoff aktiv: Nächster Versuch in {remaining_minutes} Minuten")
+                    await sync_utils.release_sync_lock(entity_id, sync_status)
+                    return
+            except Exception as e:
+                context.logger.warn(f"⚠️  Fehler beim Parsen von syncNextRetry: {e}")
+        
         # 3. BESTIMME SYNC-AKTION
         
         # FALL A: Neu (kein betnr) → CREATE in Advoware
@@ -273,6 +292,21 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
             
             context.logger.info(f"✅ Advoware STAMMDATEN aktualisiert, rowId: {new_rowid[:20] if new_rowid else 'N/A'}...")
             
+            # FIX #13: Validiere Sync-Ergebnis
+            validation_success, validation_error = await sync_utils.validate_sync_result(
+                entity_id, betnr, mapper, direction='to_advoware'
+            )
+            
+            if not validation_success:
+                context.logger.error(f"❌ Sync-Validation fehlgeschlagen: {validation_error}")
+                await sync_utils.release_sync_lock(
+                    entity_id, 
+                    'failed',
+                    error_message=f"Validation failed: {validation_error}",
+                    increment_retry=True
+                )
+                return
+            
             # KOMMUNIKATION SYNC: Immer ausführen nach Stammdaten-Update
             await run_kommunikation_sync(entity_id, betnr, komm_sync, context)
             
@@ -291,6 +325,21 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
             await espocrm.update_entity('CBeteiligte', entity_id, espo_data)
             context.logger.info(f"✅ EspoCRM STAMMDATEN aktualisiert")
             
+            # FIX #13: Validiere Sync-Ergebnis
+            validation_success, validation_error = await sync_utils.validate_sync_result(
+                entity_id, betnr, mapper, direction='to_espocrm'
+            )
+            
+            if not validation_success:
+                context.logger.error(f"❌ Sync-Validation fehlgeschlagen: {validation_error}")
+                await sync_utils.release_sync_lock(
+                    entity_id, 
+                    'failed',
+                    error_message=f"Validation failed: {validation_error}",
+                    increment_retry=True
+                )
+                return
+            
             # KOMMUNIKATION SYNC: Immer ausführen nach Stammdaten-Update
             await run_kommunikation_sync(entity_id, betnr, komm_sync, context)
             
@@ -327,6 +376,23 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
                 f"EspoCRM hat gewonnen."
             )
             
+            context.logger.info(f"✅ Konflikt gelöst (EspoCRM won), neue rowId: {new_rowid[:20] if new_rowid else 'N/A'}...")
+            
+            # FIX #13: Validiere Sync-Ergebnis
+            validation_success, validation_error = await sync_utils.validate_sync_result(
+                entity_id, betnr, mapper, direction='to_advoware'
+            )
+            
+            if not validation_success:
+                context.logger.error(f"❌ Conflict resolution validation fehlgeschlagen: {validation_error}")
+                await sync_utils.release_sync_lock(
+                    entity_id, 
+                    'failed',
+                    error_message=f"Conflict resolution validation failed: {validation_error}",
+                    increment_retry=True
+                )
+                return
+            
             await sync_utils.resolve_conflict_espocrm_wins(
                 entity_id,
                 espo_entity,
@@ -334,7 +400,6 @@ async def handle_update(entity_id, betnr, espo_entity, espocrm, advoware, sync_u
                 conflict_msg,
                 extra_fields={'advowareRowId': new_rowid}
             )
-            context.logger.info(f"✅ Konflikt gelöst (EspoCRM won), neue rowId: {new_rowid[:20] if new_rowid else 'N/A'}...")
             
             # KOMMUNIKATION SYNC: NUR EspoCRM→Advoware (EspoCRM wins!)
             await run_kommunikation_sync(entity_id, betnr, komm_sync, context, direction='to_advoware')