Add tests for Kommunikation Sync implementation and verification scripts

- Implemented comprehensive tests for the Kommunikation Sync functionality, covering base64 encoding, marker parsing, creation, type detection, and integration scenarios.
- Added a verification script to check for unique IDs in Advoware communications, ensuring stability and integrity of the IDs.
- Created utility scripts for code validation, notification testing, and PUT response detail analysis to enhance development and testing processes.
- Updated README with details on new tools and their usage.
This commit is contained in:
2026-02-08 23:05:56 +00:00
parent a157d3fa1d
commit 7856dd1d68
37 changed files with 438 additions and 271 deletions

View File

@@ -0,0 +1,39 @@
# Beteiligte Sync - Test Scripts
Test-Scripts für die Beteiligte (Stammdaten) Synchronisation zwischen EspoCRM und Advoware.
## Scripts
### test_beteiligte_sync.py
Vollständiger Test der Beteiligte-Sync Funktionalität.
**Testet:**
- CREATE: Neu in EspoCRM → POST zu Advoware
- UPDATE: Änderung in EspoCRM → PUT zu Advoware
- Timestamp-Vergleich (espocrm_newer, advoware_newer, conflict)
- rowId-basierte Change Detection
- Lock-Management (Redis)
**Verwendung:**
```bash
cd /opt/motia-app/bitbylaw
python scripts/beteiligte_sync/test_beteiligte_sync.py
```
### compare_beteiligte.py
Vergleicht Beteiligte-Daten zwischen EspoCRM und Advoware.
**Features:**
- Field-by-Field Vergleich
- Identifiziert Abweichungen
- JSON-Output für weitere Analyse
**Verwendung:**
```bash
python scripts/beteiligte_sync/compare_beteiligte.py --entity-id <espo_id> --betnr <advo_betnr>
```
## Verwandte Dokumentation
- [../../docs/SYNC_OVERVIEW.md](../../docs/SYNC_OVERVIEW.md#beteiligte-sync-stammdaten) - Beteiligte Sync Details
- [../../services/beteiligte_sync_utils.py](../../services/beteiligte_sync_utils.py) - Implementierung

View File

@@ -0,0 +1,323 @@
#!/usr/bin/env python3
"""
Helper-Script zum Vergleichen der Beteiligten-Strukturen zwischen Advoware und EspoCRM.
Usage:
python scripts/compare_beteiligte.py <entity_id_espocrm> [advoware_id]
Examples:
# Vergleiche EspoCRM Beteiligten (automatische Suche in Advoware)
python scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc
# Vergleiche mit spezifischer Advoware ID
python scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc 12345
"""
import sys
import asyncio
import json
import os
from pathlib import Path
# Add bitbylaw directory to path for imports
bitbylaw_dir = Path(__file__).parent.parent
sys.path.insert(0, str(bitbylaw_dir))
from services.espocrm import EspoCRMAPI
from services.advoware import AdvowareAPI
from config import Config
class SimpleContext:
"""Simple context for logging"""
class Logger:
def info(self, msg):
print(f"[INFO] {msg}")
def error(self, msg):
print(f"[ERROR] {msg}")
def debug(self, msg):
print(f"[DEBUG] {msg}")
def warning(self, msg):
print(f"[WARNING] {msg}")
def __init__(self):
self.logger = self.Logger()
async def fetch_from_espocrm(entity_id: str):
"""Fetch Beteiligter from EspoCRM"""
print("\n" + "="*80)
print("ESPOCRM - Fetching Beteiligter")
print("="*80)
context = SimpleContext()
espo = EspoCRMAPI(context=context)
try:
# Try different entity types that might contain Beteiligte
entity_types = ['CBeteiligte', 'Beteiligte', 'Contact', 'Account', 'Lead', 'CVmhErstgespraech', 'CVmhBeteiligte']
for entity_type in entity_types:
try:
print(f"\nTrying entity type: {entity_type}")
result = await espo.get_entity(entity_type, entity_id)
print(f"\n✓ Success! Found in {entity_type}")
print(f"\nEntity Structure:")
print("-" * 80)
print(json.dumps(result, indent=2, ensure_ascii=False))
return result
except Exception as e:
print(f" ✗ Not found in {entity_type}: {e}")
continue
print("\n✗ Entity not found in any known entity type")
return None
except Exception as e:
print(f"\n✗ Error fetching from EspoCRM: {e}")
return None
async def fetch_from_advoware(advoware_id: str = None, search_name: str = None):
"""Fetch Beteiligter from Advoware"""
print("\n" + "="*80)
print("ADVOWARE - Fetching Beteiligter")
print("="*80)
context = SimpleContext()
advo = AdvowareAPI(context=context)
try:
# Try to fetch by ID if provided
if advoware_id:
print(f"\nFetching by ID: {advoware_id}")
# Try correct Advoware endpoint
endpoints = [
f'/api/v1/advonet/Beteiligte/{advoware_id}',
]
for endpoint in endpoints:
try:
print(f" Trying endpoint: {endpoint}")
result = await advo.api_call(endpoint, method='GET')
if result:
# Advoware gibt oft Listen zurück, nehme erstes Element
if isinstance(result, list) and len(result) > 0:
result = result[0]
print(f"\n✓ Success! Found at {endpoint}")
print(f"\nEntity Structure:")
print("-" * 80)
print(json.dumps(result, indent=2, ensure_ascii=False))
return result
except Exception as e:
print(f" ✗ Not found at {endpoint}: {e}")
continue
# Try to search by name if EspoCRM data available
if search_name:
print(f"\nSearching by name: {search_name}")
search_endpoints = [
'/api/v1/advonet/Beteiligte',
]
for endpoint in search_endpoints:
try:
print(f" Trying endpoint: {endpoint}")
result = await advo.api_call(
endpoint,
method='GET',
params={'search': search_name, 'limit': 5}
)
if result and (isinstance(result, list) and len(result) > 0 or
isinstance(result, dict) and result.get('data')):
print(f"\n✓ Found {len(result) if isinstance(result, list) else len(result.get('data', []))} results")
print(f"\nSearch Results:")
print("-" * 80)
print(json.dumps(result, indent=2, ensure_ascii=False))
return result
except Exception as e:
print(f" ✗ Search failed at {endpoint}: {e}")
continue
print("\n✗ Entity not found in Advoware")
return None
except Exception as e:
print(f"\n✗ Error fetching from Advoware: {e}")
import traceback
traceback.print_exc()
return None
async def compare_structures(espo_data: dict, advo_data: dict):
"""Compare field structures between EspoCRM and Advoware"""
print("\n" + "="*80)
print("STRUCTURE COMPARISON")
print("="*80)
if not espo_data or not advo_data:
print("\n⚠ Cannot compare - missing data from one or both systems")
return
# Extract fields
espo_fields = set(espo_data.keys()) if isinstance(espo_data, dict) else set()
# Handle Advoware data structure (might be nested)
if isinstance(advo_data, dict):
if 'data' in advo_data:
advo_data = advo_data['data']
if isinstance(advo_data, list) and len(advo_data) > 0:
advo_data = advo_data[0]
advo_fields = set(advo_data.keys()) if isinstance(advo_data, dict) else set()
print(f"\nEspoCRM Fields ({len(espo_fields)}):")
print("-" * 40)
for field in sorted(espo_fields):
value = espo_data.get(field)
value_type = type(value).__name__
print(f" {field:<30} ({value_type})")
print(f"\nAdvoware Fields ({len(advo_fields)}):")
print("-" * 40)
for field in sorted(advo_fields):
value = advo_data.get(field)
value_type = type(value).__name__
print(f" {field:<30} ({value_type})")
# Find common fields (potential mappings)
common = espo_fields & advo_fields
espo_only = espo_fields - advo_fields
advo_only = advo_fields - espo_fields
print(f"\nCommon Fields ({len(common)}):")
print("-" * 40)
for field in sorted(common):
espo_val = espo_data.get(field)
advo_val = advo_data.get(field)
match = "" if espo_val == advo_val else ""
print(f" {match} {field}")
if espo_val != advo_val:
print(f" EspoCRM: {espo_val}")
print(f" Advoware: {advo_val}")
print(f"\nEspoCRM Only ({len(espo_only)}):")
print("-" * 40)
for field in sorted(espo_only):
print(f" {field}")
print(f"\nAdvoware Only ({len(advo_only)}):")
print("-" * 40)
for field in sorted(advo_only):
print(f" {field}")
# Suggest potential mappings based on field names
print(f"\nPotential Field Mappings:")
print("-" * 40)
mapping_suggestions = []
# Common name patterns
name_patterns = [
('name', 'name'),
('firstName', 'first_name'),
('lastName', 'last_name'),
('email', 'email'),
('emailAddress', 'email'),
('phone', 'phone'),
('phoneNumber', 'phone_number'),
('address', 'address'),
('street', 'street'),
('city', 'city'),
('postalCode', 'postal_code'),
('zipCode', 'postal_code'),
('country', 'country'),
]
for espo_field, advo_field in name_patterns:
if espo_field in espo_fields and advo_field in advo_fields:
mapping_suggestions.append((espo_field, advo_field))
print(f" {espo_field:<30}{advo_field}")
return {
'espo_fields': list(espo_fields),
'advo_fields': list(advo_fields),
'common': list(common),
'espo_only': list(espo_only),
'advo_only': list(advo_only),
'suggested_mappings': mapping_suggestions
}
async def main():
"""Main function"""
if len(sys.argv) < 2:
print(__doc__)
sys.exit(1)
espocrm_id = sys.argv[1]
advoware_id = sys.argv[2] if len(sys.argv) > 2 else None
print("\n" + "="*80)
print("BETEILIGTE STRUCTURE COMPARISON TOOL")
print("="*80)
print(f"\nEspoCRM Entity ID: {espocrm_id}")
if advoware_id:
print(f"Advoware ID: {advoware_id}")
# Check environment variables
print("\nEnvironment Check:")
print("-" * 40)
print(f"ESPOCRM_API_BASE_URL: {Config.ESPOCRM_API_BASE_URL}")
print(f"ESPOCRM_API_KEY: {'✓ Set' if Config.ESPOCRM_API_KEY else '✗ Missing'}")
print(f"ADVOWARE_API_BASE_URL: {Config.ADVOWARE_API_BASE_URL}")
print(f"ADVOWARE_API_KEY: {'✓ Set' if Config.ADVOWARE_API_KEY else '✗ Missing'}")
# Fetch from EspoCRM
espo_data = await fetch_from_espocrm(espocrm_id)
# Extract name for Advoware search
search_name = None
if espo_data:
search_name = (
espo_data.get('name') or
f"{espo_data.get('firstName', '')} {espo_data.get('lastName', '')}".strip() or
None
)
# Fetch from Advoware
advo_data = await fetch_from_advoware(advoware_id, search_name)
# Compare structures
if espo_data or advo_data:
comparison = await compare_structures(espo_data, advo_data)
# Save comparison to file
output_file = Path(__file__).parent / 'beteiligte_comparison_result.json'
with open(output_file, 'w', encoding='utf-8') as f:
json.dump({
'espocrm_data': espo_data,
'advoware_data': advo_data,
'comparison': comparison
}, f, indent=2, ensure_ascii=False)
print(f"\n\n{'='*80}")
print(f"Comparison saved to: {output_file}")
print(f"{'='*80}\n")
else:
print("\n⚠ No data available for comparison")
if __name__ == '__main__':
asyncio.run(main())

View File

@@ -0,0 +1,387 @@
#!/usr/bin/env python3
"""
Beteiligte Sync Test Script
Testet die vollständige Sync-Funktionalität:
1. Mapper-Transformationen
2. Lock-Mechanismus
3. Timestamp-Vergleich
4. CREATE in Advoware (optional)
5. UPDATE Sync (optional)
6. Konflikt-Resolution
Usage:
python test_beteiligte_sync.py --test-transforms # Nur Mapper testen
python test_beteiligte_sync.py --test-live # Live-Test mit echten APIs
python test_beteiligte_sync.py --entity-id=XXX # Spezifische Entity testen
"""
import asyncio
import sys
import argparse
from datetime import datetime
import json
sys.path.insert(0, '/opt/motia-app/bitbylaw')
from services.espocrm import EspoCRMAPI
from services.advoware import AdvowareAPI
from services.espocrm_mapper import BeteiligteMapper
from services.beteiligte_sync_utils import BeteiligteSync
class MockContext:
"""Mock Context für Testing ohne Motia Workbench"""
class Logger:
def info(self, msg): print(f" {msg}")
def debug(self, msg): print(f"🔍 {msg}")
def warning(self, msg): print(f"⚠️ {msg}")
def error(self, msg): print(f"{msg}")
def __init__(self):
self.logger = self.Logger()
async def test_transforms():
"""Test 1: Mapper-Transformationen"""
print("\n" + "="*80)
print("TEST 1: Mapper-Transformationen")
print("="*80)
mapper = BeteiligteMapper()
# Test 1a: Person EspoCRM → Advoware
print("\n📤 Test 1a: Person EspoCRM → Advoware")
espo_person = {
'id': 'test123',
'firstName': 'Angela',
'lastName': 'Mustermann',
'rechtsform': 'Frau',
'emailAddress': 'angela@example.com',
'emailAddressData': [
{'emailAddress': 'angela@example.com', 'primary': True}
],
'phoneNumber': '+49123456789',
'dateOfBirth': '1980-05-15'
}
advo_result = mapper.map_cbeteiligte_to_advoware(espo_person)
print(f"✅ Mapped:")
print(json.dumps(advo_result, indent=2, ensure_ascii=False))
# Test 1b: Firma EspoCRM → Advoware
print("\n📤 Test 1b: Firma EspoCRM → Advoware")
espo_firma = {
'id': 'test456',
'firmenname': 'Mustermann GmbH',
'rechtsform': 'GmbH',
'emailAddress': 'info@mustermann.de',
'handelsregisterNummer': 'HRB 12345'
}
advo_firma = mapper.map_cbeteiligte_to_advoware(espo_firma)
print(f"✅ Mapped:")
print(json.dumps(advo_firma, indent=2, ensure_ascii=False))
# Test 1c: Advoware → EspoCRM (Person)
print("\n📥 Test 1c: Advoware → EspoCRM (Person)")
advo_person = {
'betNr': 104860,
'vorname': 'Max',
'name': 'Mustermann',
'rechtsform': 'Herr',
'emailGesch': 'max@example.com',
'telGesch': '+49987654321',
'geburtsdatum': '1975-03-20'
}
espo_result = mapper.map_advoware_to_cbeteiligte(advo_person)
print(f"✅ Mapped:")
print(json.dumps(espo_result, indent=2, ensure_ascii=False))
print("\n✅ MAPPER TESTS PASSED!")
async def test_lock_mechanism():
"""Test 2: Lock-Mechanismus"""
print("\n" + "="*80)
print("TEST 2: Lock-Mechanismus")
print("="*80)
espocrm = EspoCRMAPI()
context = MockContext()
sync_utils = BeteiligteSync(espocrm, context)
# Hole erste Entity
result = await espocrm.list_entities('CBeteiligte', max_size=1)
if not result.get('list'):
print("❌ Keine Entities gefunden zum Testen")
return
entity = result['list'][0]
entity_id = entity['id']
original_status = entity.get('syncStatus', 'clean')
print(f"\n🔒 Test Lock für Entity: {entity.get('name')} (ID: {entity_id})")
print(f" Original Status: {original_status}")
# Test Lock Acquire
print("\n1. Acquire Lock...")
lock1 = await sync_utils.acquire_sync_lock(entity_id)
print(f" Lock 1: {lock1} (erwartet: True)")
# Verify Status
entity_check = await espocrm.get_entity('CBeteiligte', entity_id)
print(f" Status nach Lock: {entity_check.get('syncStatus')} (erwartet: syncing)")
# Test Lock Already Held
print("\n2. Versuche Lock erneut zu holen (sollte fehlschlagen)...")
lock2 = await sync_utils.acquire_sync_lock(entity_id)
print(f" Lock 2: {lock2} (erwartet: False)")
# Release Lock
print("\n3. Release Lock...")
await sync_utils.release_sync_lock(entity_id, 'clean')
entity_final = await espocrm.get_entity('CBeteiligte', entity_id)
print(f" Status nach Release: {entity_final.get('syncStatus')} (erwartet: clean)")
# Restore Original
if original_status != 'clean':
await espocrm.update_entity('CBeteiligte', entity_id, {'syncStatus': original_status})
print(f" Status restored to: {original_status}")
print("\n✅ LOCK TESTS PASSED!")
async def test_timestamp_comparison():
"""Test 3: Timestamp-Vergleich"""
print("\n" + "="*80)
print("TEST 3: Timestamp-Vergleich")
print("="*80)
espocrm = EspoCRMAPI()
context = MockContext()
sync_utils = BeteiligteSync(espocrm, context)
# Test-Timestamps
now = datetime.now()
old = datetime(2026, 2, 1, 10, 0, 0)
newer = datetime(2026, 2, 7, 14, 0, 0)
print("\n📅 Test-Timestamps:")
print(f" old: {old}")
print(f" newer: {newer}")
print(f" now: {now}")
# Scenario 1: EspoCRM neuer
print("\n1. EspoCRM neuer als Advoware:")
result = sync_utils.compare_timestamps(newer, old, old)
print(f" Result: {result} (erwartet: espocrm_newer)")
# Scenario 2: Advoware neuer
print("\n2. Advoware neuer als EspoCRM:")
result = sync_utils.compare_timestamps(old, newer, old)
print(f" Result: {result} (erwartet: advoware_newer)")
# Scenario 3: Konflikt (beide geändert)
print("\n3. Beide nach last_sync geändert (Konflikt):")
result = sync_utils.compare_timestamps(newer, newer, old)
print(f" Result: {result} (erwartet: conflict)")
# Scenario 4: Keine Änderungen
print("\n4. Keine Änderungen seit last_sync:")
result = sync_utils.compare_timestamps(old, old, newer)
print(f" Result: {result} (erwartet: no_change)")
print("\n✅ TIMESTAMP TESTS PASSED!")
async def test_live_entity(entity_id=None, dry_run=True):
"""Test 4: Live Entity Sync (optional mit echtem API-Call)"""
print("\n" + "="*80)
print("TEST 4: Live Entity Sync")
print("="*80)
espocrm = EspoCRMAPI()
advoware = AdvowareAPI(MockContext())
mapper = BeteiligteMapper()
context = MockContext()
sync_utils = BeteiligteSync(espocrm, context)
# Hole Entity
if not entity_id:
result = await espocrm.list_entities('CBeteiligte', max_size=5)
entities = result.get('list', [])
# Suche eine mit betnr
entity = next((e for e in entities if e.get('betnr')), entities[0] if entities else None)
if not entity:
print("❌ Keine Entity gefunden")
return
entity_id = entity['id']
else:
entity = await espocrm.get_entity('CBeteiligte', entity_id)
print(f"\n📋 Test Entity:")
print(f" ID: {entity_id}")
print(f" Name: {entity.get('name')}")
print(f" betNr: {entity.get('betnr')}")
print(f" syncStatus: {entity.get('syncStatus')}")
print(f" modifiedAt: {entity.get('modifiedAt')}")
print(f" advowareLastSync: {entity.get('advowareLastSync')}")
betnr = entity.get('betnr')
# Test Transformation
print("\n🔄 Transformation EspoCRM → Advoware:")
advo_data = mapper.map_cbeteiligte_to_advoware(entity)
print(json.dumps(advo_data, indent=2, ensure_ascii=False))
# Wenn betnr vorhanden, teste Fetch von Advoware
if betnr:
print(f"\n📥 Fetch von Advoware (betNr={betnr}):")
try:
advo_result = await advoware.api_call(
f'api/v1/advonet/Beteiligte/{betnr}',
method='GET'
)
if isinstance(advo_result, list):
advo_entity = advo_result[0] if advo_result else None
else:
advo_entity = advo_result
if advo_entity:
print(f"✅ Von Advoware geladen:")
print(f" name: {advo_entity.get('name')}")
print(f" vorname: {advo_entity.get('vorname')}")
print(f" geaendertAm: {advo_entity.get('geaendertAm')}")
# Timestamp-Vergleich
print(f"\n⏱️ Timestamp-Vergleich:")
comparison = sync_utils.compare_timestamps(
entity.get('modifiedAt'),
advo_entity.get('geaendertAm'),
entity.get('advowareLastSync')
)
print(f" Result: {comparison}")
# Zeige was geändert wäre
changed = mapper.get_changed_fields(entity, advo_entity)
if changed:
print(f"\n📝 Geänderte Felder: {', '.join(changed)}")
else:
print(f"\n✅ Keine Feld-Unterschiede")
else:
print("❌ Keine Daten von Advoware")
except Exception as e:
print(f"⚠️ Fehler beim Fetch von Advoware: {e}")
else:
print("\n⚠️ Keine betnr vorhanden (Entity wurde noch nicht gesynct)")
if not dry_run:
print("\n🆕 Würde CREATE in Advoware ausführen:")
print(f" POST /api/v1/advonet/Beteiligte")
print(f" Data: {json.dumps(advo_data, indent=2, ensure_ascii=False)}")
print("\n⚠️ DRY RUN - Nicht ausgeführt!")
print("\n✅ LIVE ENTITY TEST COMPLETE!")
async def test_full_sync_handler(entity_id):
"""Test 5: Vollständiger Sync-Handler (simuliert Event)"""
print("\n" + "="*80)
print("TEST 5: Vollständiger Sync-Handler")
print("="*80)
# Import Handler
sys.path.insert(0, '/opt/motia-app/bitbylaw/steps/vmh')
import beteiligte_sync_event_step
# Mock Event Data
event_data = {
'entity_id': entity_id,
'action': 'sync_check',
'source': 'test_script',
'timestamp': datetime.now().isoformat()
}
context = MockContext()
print(f"\n🎬 Simuliere Event für Entity: {entity_id}")
print(f" Event: {json.dumps(event_data, indent=2)}")
print("\n⚠️ ACHTUNG: Dies führt ECHTE API-Calls aus!")
response = input("Fortfahren? (y/N): ")
if response.lower() != 'y':
print("❌ Abgebrochen")
return
print("\n🚀 Handler wird ausgeführt...\n")
try:
await beteiligte_sync_event_step.handler(event_data, context)
print("\n✅ HANDLER ERFOLGREICH!")
except Exception as e:
print(f"\n❌ HANDLER FEHLER: {e}")
import traceback
traceback.print_exc()
async def main():
parser = argparse.ArgumentParser(description='Test Beteiligte Sync')
parser.add_argument('--test-transforms', action='store_true', help='Nur Mapper testen')
parser.add_argument('--test-lock', action='store_true', help='Lock-Mechanismus testen')
parser.add_argument('--test-timestamps', action='store_true', help='Timestamp-Vergleich testen')
parser.add_argument('--test-live', action='store_true', help='Live-Test mit APIs')
parser.add_argument('--test-full-handler', action='store_true', help='Vollständiger Handler (ECHTE CALLS!)')
parser.add_argument('--entity-id', type=str, help='Spezifische Entity-ID testen')
parser.add_argument('--all', action='store_true', help='Alle Tests ausführen')
args = parser.parse_args()
# Default: Alle Tests
if not any([args.test_transforms, args.test_lock, args.test_timestamps,
args.test_live, args.test_full_handler, args.all]):
args.all = True
print("🧪 Beteiligte Sync Test Suite")
print("="*80)
try:
if args.all or args.test_transforms:
await test_transforms()
if args.all or args.test_lock:
await test_lock_mechanism()
if args.all or args.test_timestamps:
await test_timestamp_comparison()
if args.all or args.test_live:
await test_live_entity(args.entity_id, dry_run=True)
if args.test_full_handler:
if not args.entity_id:
print("\n❌ --entity-id erforderlich für --test-full-handler")
else:
await test_full_sync_handler(args.entity_id)
print("\n" + "="*80)
print("✅ ALLE TESTS ABGESCHLOSSEN!")
print("="*80)
except Exception as e:
print(f"\n❌ FEHLER: {e}")
import traceback
traceback.print_exc()
sys.exit(1)
if __name__ == '__main__':
asyncio.run(main())