feat(sync): Implement auto-reset for permanently_failed entities and add retry backoff logic
- Added logic to reset permanently_failed entities that have reached their auto-reset threshold in `beteiligte_sync_cron_step.py`. - Enhanced event handling in `beteiligte_sync_event_step.py` to skip retries if the next retry time has not been reached. - Introduced validation checks after sync operations to ensure data consistency and integrity. - Created detailed documentation outlining the fixes and their impacts on the sync process. - Added scripts for analyzing sync issues and comparing entities to facilitate debugging and validation.
This commit is contained in:
209
bitbylaw/scripts/analyze_sync_issues_104860.py
Normal file
209
bitbylaw/scripts/analyze_sync_issues_104860.py
Normal file
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Detaillierte Analyse der Sync-Probleme für Entity 104860
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
import base64
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from services.advoware import AdvowareAPI
|
||||
from services.espocrm import EspoCRMAPI
|
||||
from services.kommunikation_mapper import parse_marker, should_sync_to_espocrm
|
||||
|
||||
|
||||
class SimpleContext:
|
||||
class Logger:
|
||||
def info(self, msg): print(f"ℹ️ {msg}")
|
||||
def debug(self, msg): pass # Suppress debug
|
||||
def warn(self, msg): print(f"⚠️ {msg}")
|
||||
def warning(self, msg): print(f"⚠️ {msg}")
|
||||
def error(self, msg): print(f"❌ {msg}")
|
||||
|
||||
def __init__(self):
|
||||
self.logger = self.Logger()
|
||||
|
||||
|
||||
async def analyze():
|
||||
context = SimpleContext()
|
||||
betnr = 104860
|
||||
espo_id = "68e3e7eab49f09adb"
|
||||
|
||||
# Initialize APIs
|
||||
advoware_api = AdvowareAPI(context)
|
||||
espocrm = EspoCRMAPI(context)
|
||||
|
||||
# Fetch data
|
||||
advo_result = await advoware_api.api_call(f'api/v1/advonet/Beteiligte/{betnr}', method='GET')
|
||||
advo_entity = advo_result[0] if isinstance(advo_result, list) else advo_result
|
||||
|
||||
espo_entity = await espocrm.get_entity('CBeteiligte', espo_id)
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("DETAILLIERTE SYNC-PROBLEM ANALYSE")
|
||||
print("="*80 + "\n")
|
||||
|
||||
# ========== PROBLEM 1: NAME MISMATCH ==========
|
||||
print("🔴 PROBLEM 1: STAMMDATEN NICHT SYNCHRON")
|
||||
print("-" * 80)
|
||||
print(f"EspoCRM Name: '{espo_entity.get('name')}'")
|
||||
print(f"Advoware Name: '{advo_entity.get('name')}'")
|
||||
print(f"")
|
||||
print(f"ANALYSE:")
|
||||
print(f"- syncStatus: {espo_entity.get('syncStatus')}")
|
||||
print(f"- advowareLastSync: {espo_entity.get('advowareLastSync')}")
|
||||
print(f"- modifiedAt (EspoCRM): {espo_entity.get('modifiedAt')}")
|
||||
print(f"- geaendertAm (Advoware): {advo_entity.get('geaendertAm')}")
|
||||
print(f"")
|
||||
print(f"💡 URSACHE:")
|
||||
print(f" - Sync sagt 'clean' aber Daten sind NICHT identisch!")
|
||||
print(f" - Dies ist Problem #13: Keine Validierung von Sync-Ergebnissen")
|
||||
print(f" - Sync glaubt es war erfolgreich, aber Mapping oder API-Call fehlte")
|
||||
print()
|
||||
|
||||
# ========== PROBLEM 2: KOMMUNIKATION COUNTS ==========
|
||||
print("🟡 PROBLEM 2: KOMMUNIKATION ANZAHL-MISMATCH")
|
||||
print("-" * 80)
|
||||
|
||||
advo_kommunikationen = advo_entity.get('kommunikation', [])
|
||||
espo_emails = espo_entity.get('emailAddressData', [])
|
||||
espo_phones = espo_entity.get('phoneNumberData', [])
|
||||
|
||||
# Analysiere Advoware Kommunikationen
|
||||
advo_with_value = []
|
||||
advo_empty_slots = []
|
||||
advo_non_sync = []
|
||||
|
||||
for komm in advo_kommunikationen:
|
||||
tlf = (komm.get('tlf') or '').strip()
|
||||
bemerkung = komm.get('bemerkung', '')
|
||||
marker = parse_marker(bemerkung)
|
||||
|
||||
if not should_sync_to_espocrm(komm):
|
||||
advo_non_sync.append(komm)
|
||||
elif not tlf or (marker and marker.get('is_slot')):
|
||||
advo_empty_slots.append(komm)
|
||||
else:
|
||||
advo_with_value.append(komm)
|
||||
|
||||
print(f"Advoware Kommunikationen: {len(advo_kommunikationen)} total")
|
||||
print(f" - Mit Wert (sollten in EspoCRM sein): {len(advo_with_value)}")
|
||||
print(f" - Empty Slots: {len(advo_empty_slots)}")
|
||||
print(f" - Nicht-sync-relevant: {len(advo_non_sync)}")
|
||||
print()
|
||||
print(f"EspoCRM Kommunikationen: {len(espo_emails) + len(espo_phones)} total")
|
||||
print(f" - Emails: {len(espo_emails)}")
|
||||
print(f" - Phones: {len(espo_phones)}")
|
||||
print()
|
||||
|
||||
# Detaillierte Analyse der Empty Slots
|
||||
print("📋 Empty Slots in Advoware:")
|
||||
for i, slot in enumerate(advo_empty_slots, 1):
|
||||
marker = parse_marker(slot.get('bemerkung', ''))
|
||||
kommkz = marker.get('kommKz') if marker else 'N/A'
|
||||
rowid = slot.get('rowId', 'N/A')[:20]
|
||||
print(f" {i}. kommKz={kommkz} | rowId={rowid}... | bemerkung={slot.get('bemerkung', '')[:40]}")
|
||||
print()
|
||||
|
||||
print("💡 URSACHE:")
|
||||
print(f" - {len(advo_empty_slots)} Empty Slots werden NICHT aufgeräumt")
|
||||
print(f" - Dies ist Problem #2: Empty Slot Accumulation")
|
||||
print(f" - Nur {len(advo_with_value)} Einträge mit Wert, aber Hash beinhaltet ALLE {len(advo_kommunikationen)}")
|
||||
print()
|
||||
|
||||
# ========== PROBLEM 3: MARKER ANALYSIS ==========
|
||||
print("🟡 PROBLEM 3: MARKER VALIDIERUNG")
|
||||
print("-" * 80)
|
||||
|
||||
marker_issues = []
|
||||
|
||||
for komm in advo_with_value:
|
||||
tlf = (komm.get('tlf') or '').strip()
|
||||
bemerkung = komm.get('bemerkung', '')
|
||||
marker = parse_marker(bemerkung)
|
||||
|
||||
if marker:
|
||||
synced_value = marker.get('synced_value', '')
|
||||
if synced_value != tlf:
|
||||
marker_issues.append({
|
||||
'tlf': tlf,
|
||||
'synced_value': synced_value,
|
||||
'marker': bemerkung[:50]
|
||||
})
|
||||
|
||||
if marker_issues:
|
||||
print(f"❌ {len(marker_issues)} Marker stimmen NICHT mit aktuellem Wert überein:")
|
||||
for issue in marker_issues:
|
||||
print(f" - Aktuell: '{issue['tlf']}'")
|
||||
print(f" Marker: '{issue['synced_value']}'")
|
||||
print(f" Marker-String: {issue['marker']}...")
|
||||
print()
|
||||
print("💡 URSACHE:")
|
||||
print(" - Dies deutet auf Problem #6: Marker-Update fehlgeschlagen")
|
||||
print(" - Oder Var6 wurde erkannt aber Marker nicht aktualisiert")
|
||||
else:
|
||||
print("✅ Alle Marker stimmen mit aktuellen Werten überein")
|
||||
print()
|
||||
|
||||
# ========== PROBLEM 4: HASH COVERAGE ==========
|
||||
print("🟡 PROBLEM 4: HASH-BERECHNUNG")
|
||||
print("-" * 80)
|
||||
|
||||
import hashlib
|
||||
|
||||
# Aktueller Code (FALSCH - beinhaltet ALLE)
|
||||
all_rowids = sorted([k.get('rowId', '') for k in advo_kommunikationen if k.get('rowId')])
|
||||
wrong_hash = hashlib.md5(''.join(all_rowids).encode()).hexdigest()[:16]
|
||||
|
||||
# Korrekt (nur sync-relevante)
|
||||
sync_relevant_komm = [k for k in advo_kommunikationen if should_sync_to_espocrm(k) and (k.get('tlf') or '').strip()]
|
||||
sync_rowids = sorted([k.get('rowId', '') for k in sync_relevant_komm if k.get('rowId')])
|
||||
correct_hash = hashlib.md5(''.join(sync_rowids).encode()).hexdigest()[:16]
|
||||
|
||||
stored_hash = espo_entity.get('kommunikationHash')
|
||||
|
||||
print(f"Hash-Vergleich:")
|
||||
print(f" - Gespeichert: {stored_hash}")
|
||||
print(f" - Aktuell (ALL): {wrong_hash} {'✅' if wrong_hash == stored_hash else '❌'}")
|
||||
print(f" - Korrekt (nur sync-relevant): {correct_hash} {'✅' if correct_hash == stored_hash else '❌'}")
|
||||
print()
|
||||
print(f"Rowids einbezogen:")
|
||||
print(f" - ALL: {len(all_rowids)} Kommunikationen")
|
||||
print(f" - Sync-relevant: {len(sync_rowids)} Kommunikationen")
|
||||
print()
|
||||
print("💡 URSACHE:")
|
||||
print(" - Dies ist Problem #3: Hash beinhaltet ALLE statt nur sync-relevante")
|
||||
print(" - Empty Slots ändern Hash obwohl sie nicht in EspoCRM sind")
|
||||
print()
|
||||
|
||||
# ========== ZUSAMMENFASSUNG ==========
|
||||
print("="*80)
|
||||
print("ZUSAMMENFASSUNG DER PROBLEME")
|
||||
print("="*80)
|
||||
print()
|
||||
print("✅ BESTÄTIGT - Die folgenden Probleme existieren:")
|
||||
print()
|
||||
print("1. ❌ Problem #13: Keine Validierung von Sync-Ergebnissen")
|
||||
print(" → Stammdaten sind NICHT synchron obwohl syncStatus='clean'")
|
||||
print()
|
||||
print("2. ❌ Problem #2: Empty Slot Accumulation")
|
||||
print(f" → {len(advo_empty_slots)} Empty Slots sammeln sich an")
|
||||
print()
|
||||
print("3. ❌ Problem #3: Hash-Berechnung inkorrekt")
|
||||
print(f" → Hash beinhaltet {len(all_rowids)} statt {len(sync_rowids)} Kommunikationen")
|
||||
print()
|
||||
|
||||
if marker_issues:
|
||||
print("4. ❌ Problem #6: Marker-Update Failures")
|
||||
print(f" → {len(marker_issues)} Marker stimmen nicht mit aktuellem Wert überein")
|
||||
print()
|
||||
|
||||
print("="*80)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(analyze())
|
||||
Reference in New Issue
Block a user