feat(sync): Implement auto-reset for permanently_failed entities and add retry backoff logic
- Added logic to reset permanently_failed entities that have reached their auto-reset threshold in `beteiligte_sync_cron_step.py`. - Enhanced event handling in `beteiligte_sync_event_step.py` to skip retries if the next retry time has not been reached. - Introduced validation checks after sync operations to ensure data consistency and integrity. - Created detailed documentation outlining the fixes and their impacts on the sync process. - Added scripts for analyzing sync issues and comparing entities to facilitate debugging and validation.
This commit is contained in:
233
bitbylaw/scripts/compare_entities_104860.py
Normal file
233
bitbylaw/scripts/compare_entities_104860.py
Normal file
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Vergleicht Advoware Entity (betNr 104860) mit EspoCRM Entity (68e3e7eab49f09adb)
|
||||
um zu prüfen ob sie synchron sind.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from services.advoware import AdvowareAPI
|
||||
from services.advoware_service import AdvowareService
|
||||
from services.espocrm import EspoCRMAPI
|
||||
from services.espocrm_mapper import BeteiligteMapper
|
||||
from services.beteiligte_sync_utils import BeteiligteSync
|
||||
import hashlib
|
||||
|
||||
|
||||
class SimpleContext:
|
||||
"""Minimal context for logging"""
|
||||
class Logger:
|
||||
def info(self, msg): print(f"ℹ️ {msg}")
|
||||
def debug(self, msg): print(f"🔍 {msg}")
|
||||
def warn(self, msg): print(f"⚠️ {msg}")
|
||||
def warning(self, msg): print(f"⚠️ {msg}")
|
||||
def error(self, msg): print(f"❌ {msg}")
|
||||
|
||||
def __init__(self):
|
||||
self.logger = self.Logger()
|
||||
|
||||
|
||||
def calculate_komm_hash(kommunikationen):
|
||||
"""Berechnet Hash wie im Code"""
|
||||
komm_rowids = sorted([k.get('rowId', '') for k in kommunikationen if k.get('rowId')])
|
||||
return hashlib.md5(''.join(komm_rowids).encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
async def compare_entities():
|
||||
context = SimpleContext()
|
||||
|
||||
# IDs
|
||||
betnr = 104860
|
||||
espo_id = "68e3e7eab49f09adb"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print(f"ENTITY COMPARISON")
|
||||
print(f"{'='*80}")
|
||||
print(f"Advoware betNr: {betnr}")
|
||||
print(f"EspoCRM ID: {espo_id}")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Initialize APIs
|
||||
advoware_api = AdvowareAPI(context)
|
||||
advoware_service = AdvowareService(context)
|
||||
espocrm = EspoCRMAPI(context)
|
||||
mapper = BeteiligteMapper()
|
||||
sync_utils = BeteiligteSync(espocrm, None, context)
|
||||
|
||||
# ========== FETCH ADVOWARE ==========
|
||||
print("\n📥 Fetching Advoware Entity...")
|
||||
try:
|
||||
advo_result = await advoware_api.api_call(
|
||||
f'api/v1/advonet/Beteiligte/{betnr}',
|
||||
method='GET'
|
||||
)
|
||||
|
||||
if isinstance(advo_result, list):
|
||||
advo_entity = advo_result[0] if advo_result else None
|
||||
else:
|
||||
advo_entity = advo_result
|
||||
|
||||
if not advo_entity:
|
||||
print("❌ Advoware Entity nicht gefunden!")
|
||||
return
|
||||
|
||||
print(f"✅ Advoware Entity geladen")
|
||||
print(f" - Name: {advo_entity.get('name')}")
|
||||
print(f" - rowId: {advo_entity.get('rowId', 'N/A')[:40]}...")
|
||||
print(f" - geaendertAm: {advo_entity.get('geaendertAm')}")
|
||||
print(f" - Kommunikationen: {len(advo_entity.get('kommunikation', []))}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Laden von Advoware: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return
|
||||
|
||||
# ========== FETCH ESPOCRM ==========
|
||||
print("\n📥 Fetching EspoCRM Entity...")
|
||||
try:
|
||||
espo_entity = await espocrm.get_entity('CBeteiligte', espo_id)
|
||||
|
||||
if not espo_entity:
|
||||
print("❌ EspoCRM Entity nicht gefunden!")
|
||||
return
|
||||
|
||||
print(f"✅ EspoCRM Entity geladen")
|
||||
print(f" - Name: {espo_entity.get('name')}")
|
||||
print(f" - betnr: {espo_entity.get('betnr')}")
|
||||
print(f" - modifiedAt: {espo_entity.get('modifiedAt')}")
|
||||
print(f" - syncStatus: {espo_entity.get('syncStatus')}")
|
||||
print(f" - advowareLastSync: {espo_entity.get('advowareLastSync')}")
|
||||
print(f" - advowareRowId: {espo_entity.get('advowareRowId', 'N/A')[:40]}...")
|
||||
print(f" - kommunikationHash: {espo_entity.get('kommunikationHash')}")
|
||||
print(f" - emailAddressData: {len(espo_entity.get('emailAddressData', []))}")
|
||||
print(f" - phoneNumberData: {len(espo_entity.get('phoneNumberData', []))}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Fehler beim Laden von EspoCRM: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return
|
||||
|
||||
# ========== COMPARISON ==========
|
||||
print(f"\n{'='*80}")
|
||||
print("STAMMDATEN VERGLEICH")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
# Timestamp comparison
|
||||
comparison = sync_utils.compare_entities(espo_entity, advo_entity)
|
||||
print(f"🔍 Timestamp-Vergleich: {comparison}")
|
||||
|
||||
# Field-by-field comparison
|
||||
print("\n📊 Feld-für-Feld Vergleich (Stammdaten):\n")
|
||||
|
||||
# Map Advoware → EspoCRM für Vergleich
|
||||
advo_mapped = mapper.map_advoware_to_cbeteiligte(advo_entity)
|
||||
|
||||
fields_to_compare = [
|
||||
'name', 'rechtsform', 'geburtsdatum', 'anrede',
|
||||
'handelsregister', 'geschlecht', 'titel'
|
||||
]
|
||||
|
||||
differences = []
|
||||
for field in fields_to_compare:
|
||||
espo_val = espo_entity.get(field)
|
||||
advo_val = advo_mapped.get(field)
|
||||
|
||||
match = "✅" if espo_val == advo_val else "❌"
|
||||
print(f"{match} {field:20} | EspoCRM: {str(espo_val)[:40]:40} | Advoware: {str(advo_val)[:40]:40}")
|
||||
|
||||
if espo_val != advo_val:
|
||||
differences.append({
|
||||
'field': field,
|
||||
'espocrm': espo_val,
|
||||
'advoware': advo_val
|
||||
})
|
||||
|
||||
# ========== KOMMUNIKATION COMPARISON ==========
|
||||
print(f"\n{'='*80}")
|
||||
print("KOMMUNIKATION VERGLEICH")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
advo_kommunikationen = advo_entity.get('kommunikation', [])
|
||||
espo_emails = espo_entity.get('emailAddressData', [])
|
||||
espo_phones = espo_entity.get('phoneNumberData', [])
|
||||
|
||||
# Hash Vergleich
|
||||
current_hash = calculate_komm_hash(advo_kommunikationen)
|
||||
stored_hash = espo_entity.get('kommunikationHash')
|
||||
|
||||
print(f"📊 Kommunikations-Hash:")
|
||||
print(f" - Gespeichert in EspoCRM: {stored_hash}")
|
||||
print(f" - Aktuell in Advoware: {current_hash}")
|
||||
print(f" - Match: {'✅ JA' if current_hash == stored_hash else '❌ NEIN'}")
|
||||
|
||||
# Advoware Kommunikationen im Detail
|
||||
print(f"\n📞 Advoware Kommunikationen ({len(advo_kommunikationen)}):")
|
||||
for i, komm in enumerate(advo_kommunikationen, 1):
|
||||
tlf = (komm.get('tlf') or '').strip()
|
||||
kommkz = komm.get('kommKz', 0)
|
||||
bemerkung = komm.get('bemerkung', '')[:50]
|
||||
online = komm.get('online', False)
|
||||
rowid = komm.get('rowId', 'N/A')[:20]
|
||||
|
||||
print(f" {i}. {tlf:30} | kommKz={kommkz:2} | online={online} | rowId={rowid}...")
|
||||
if bemerkung:
|
||||
print(f" Bemerkung: {bemerkung}...")
|
||||
|
||||
# EspoCRM Emails
|
||||
print(f"\n📧 EspoCRM Emails ({len(espo_emails)}):")
|
||||
for i, email in enumerate(espo_emails, 1):
|
||||
addr = email.get('emailAddress', '')
|
||||
primary = email.get('primary', False)
|
||||
print(f" {i}. {addr:40} | primary={primary}")
|
||||
|
||||
# EspoCRM Phones
|
||||
print(f"\n📱 EspoCRM Phones ({len(espo_phones)}):")
|
||||
for i, phone in enumerate(espo_phones, 1):
|
||||
num = phone.get('phoneNumber', '')
|
||||
typ = phone.get('type', 'N/A')
|
||||
primary = phone.get('primary', False)
|
||||
print(f" {i}. {num:30} | type={typ:10} | primary={primary}")
|
||||
|
||||
# ========== SUMMARY ==========
|
||||
print(f"\n{'='*80}")
|
||||
print("ZUSAMMENFASSUNG")
|
||||
print(f"{'='*80}\n")
|
||||
|
||||
if differences:
|
||||
print(f"❌ STAMMDATEN NICHT SYNCHRON! {len(differences)} Unterschiede gefunden:")
|
||||
for diff in differences:
|
||||
print(f" - {diff['field']}: EspoCRM='{diff['espocrm']}' ≠ Advoware='{diff['advoware']}'")
|
||||
else:
|
||||
print("✅ Stammdaten sind synchron")
|
||||
|
||||
print()
|
||||
|
||||
if current_hash != stored_hash:
|
||||
print(f"❌ KOMMUNIKATION NICHT SYNCHRON! Hash stimmt nicht überein")
|
||||
else:
|
||||
print("✅ Kommunikation-Hash stimmt überein (aber könnte trotzdem Unterschiede geben)")
|
||||
|
||||
print()
|
||||
|
||||
# Total count check
|
||||
total_espo_komm = len(espo_emails) + len(espo_phones)
|
||||
total_advo_komm = len([k for k in advo_kommunikationen if (k.get('tlf') or '').strip()])
|
||||
|
||||
if total_espo_komm != total_advo_komm:
|
||||
print(f"⚠️ Anzahl-Unterschied: EspoCRM={total_espo_komm} ≠ Advoware={total_advo_komm}")
|
||||
else:
|
||||
print(f"✅ Anzahl stimmt überein: {total_espo_komm} Kommunikationen")
|
||||
|
||||
print(f"\n{'='*80}\n")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(compare_entities())
|
||||
Reference in New Issue
Block a user