From 7856dd1d6826318a6a24242874645136786c85ec Mon Sep 17 00:00:00 2001 From: bitbylaw Date: Sun, 8 Feb 2026 23:05:56 +0000 Subject: [PATCH] Add tests for Kommunikation Sync implementation and verification scripts - Implemented comprehensive tests for the Kommunikation Sync functionality, covering base64 encoding, marker parsing, creation, type detection, and integration scenarios. - Added a verification script to check for unique IDs in Advoware communications, ensuring stability and integrity of the IDs. - Created utility scripts for code validation, notification testing, and PUT response detail analysis to enhance development and testing processes. - Updated README with details on new tools and their usage. --- bitbylaw/scripts/README.md | 401 ++++++------------ bitbylaw/scripts/adressen_sync/README.md | 68 +++ .../{ => adressen_sync}/test_adressen_api.py | 0 .../test_adressen_deactivate_ordering.py | 0 .../test_adressen_delete_matching.py | 0 .../test_adressen_gueltigbis_modify.py | 0 .../test_adressen_nullen.py | 0 .../{ => adressen_sync}/test_adressen_sync.py | 0 .../test_find_hauptadresse.py | 0 .../test_hauptadresse_explizit.py | 0 .../test_hauptadresse_logic.py | 0 bitbylaw/scripts/analysis/README.md | 41 ++ .../analyze_beteiligte_endpoint.py | 0 .../analyze_sync_issues_104860.py | 0 .../{ => analysis}/compare_entities_104860.py | 0 bitbylaw/scripts/beteiligte_sync/README.md | 39 ++ .../compare_beteiligte.py | 0 .../test_beteiligte_sync.py | 0 bitbylaw/scripts/espocrm_tests/README.md | 45 ++ .../test_espocrm_hidden_ids.py | 0 .../test_espocrm_id_collections.py | 0 .../test_espocrm_id_injection.py | 0 .../test_espocrm_kommunikation.py | 0 .../test_espocrm_kommunikation_detail.py | 0 .../test_espocrm_phone_email_entities.py | 0 bitbylaw/scripts/kommunikation_sync/README.md | 67 +++ .../test_kommart_values.py | 0 .../test_kommunikation_api.py | 0 .../test_kommunikation_kommkz_deep.py | 0 .../test_kommunikation_matching_strategy.py | 0 .../test_kommunikation_readonly.py | 0 .../test_kommunikation_sync_implementation.py | 0 .../verify_advoware_kommunikation_ids.py | 0 bitbylaw/scripts/tools/README.md | 48 +++ .../scripts/{ => tools}/test_notification.py | 0 .../{ => tools}/test_put_response_detail.py | 0 bitbylaw/scripts/{ => tools}/validate_code.py | 0 37 files changed, 438 insertions(+), 271 deletions(-) create mode 100644 bitbylaw/scripts/adressen_sync/README.md rename bitbylaw/scripts/{ => adressen_sync}/test_adressen_api.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_adressen_deactivate_ordering.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_adressen_delete_matching.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_adressen_gueltigbis_modify.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_adressen_nullen.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_adressen_sync.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_find_hauptadresse.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_hauptadresse_explizit.py (100%) rename bitbylaw/scripts/{ => adressen_sync}/test_hauptadresse_logic.py (100%) create mode 100644 bitbylaw/scripts/analysis/README.md rename bitbylaw/scripts/{ => analysis}/analyze_beteiligte_endpoint.py (100%) rename bitbylaw/scripts/{ => analysis}/analyze_sync_issues_104860.py (100%) rename bitbylaw/scripts/{ => analysis}/compare_entities_104860.py (100%) create mode 100644 bitbylaw/scripts/beteiligte_sync/README.md rename bitbylaw/scripts/{ => beteiligte_sync}/compare_beteiligte.py (100%) rename bitbylaw/scripts/{ => beteiligte_sync}/test_beteiligte_sync.py (100%) create mode 100644 bitbylaw/scripts/espocrm_tests/README.md rename bitbylaw/scripts/{ => espocrm_tests}/test_espocrm_hidden_ids.py (100%) rename bitbylaw/scripts/{ => espocrm_tests}/test_espocrm_id_collections.py (100%) rename bitbylaw/scripts/{ => espocrm_tests}/test_espocrm_id_injection.py (100%) rename bitbylaw/scripts/{ => espocrm_tests}/test_espocrm_kommunikation.py (100%) rename bitbylaw/scripts/{ => espocrm_tests}/test_espocrm_kommunikation_detail.py (100%) rename bitbylaw/scripts/{ => espocrm_tests}/test_espocrm_phone_email_entities.py (100%) create mode 100644 bitbylaw/scripts/kommunikation_sync/README.md rename bitbylaw/scripts/{ => kommunikation_sync}/test_kommart_values.py (100%) rename bitbylaw/scripts/{ => kommunikation_sync}/test_kommunikation_api.py (100%) rename bitbylaw/scripts/{ => kommunikation_sync}/test_kommunikation_kommkz_deep.py (100%) rename bitbylaw/scripts/{ => kommunikation_sync}/test_kommunikation_matching_strategy.py (100%) rename bitbylaw/scripts/{ => kommunikation_sync}/test_kommunikation_readonly.py (100%) rename bitbylaw/scripts/{ => kommunikation_sync}/test_kommunikation_sync_implementation.py (100%) rename bitbylaw/scripts/{ => kommunikation_sync}/verify_advoware_kommunikation_ids.py (100%) create mode 100644 bitbylaw/scripts/tools/README.md rename bitbylaw/scripts/{ => tools}/test_notification.py (100%) rename bitbylaw/scripts/{ => tools}/test_put_response_detail.py (100%) rename bitbylaw/scripts/{ => tools}/validate_code.py (100%) diff --git a/bitbylaw/scripts/README.md b/bitbylaw/scripts/README.md index 810cae8c..e54ff773 100644 --- a/bitbylaw/scripts/README.md +++ b/bitbylaw/scripts/README.md @@ -1,296 +1,155 @@ -# Beteiligte Structure Comparison Tool +# Scripts -## Purpose +Test- und Utility-Scripts für das Motia BitByLaw Projekt. -This helper script fetches entity data from both **EspoCRM** and **Advoware** to compare their data structures. This helps understand: +## Struktur -- What fields exist in each system -- How field names differ -- Potential field mappings for synchronization -- Data type differences +``` +scripts/ +├── beteiligte_sync/ # Beteiligte (Stammdaten) Sync Tests +│ ├── test_beteiligte_sync.py +│ ├── compare_beteiligte.py +│ └── README.md +│ +├── kommunikation_sync/ # Kommunikation (Phone/Email) Sync Tests +│ ├── test_kommunikation_api.py +│ ├── test_kommunikation_sync_implementation.py +│ ├── test_kommunikation_matching_strategy.py +│ ├── test_kommunikation_kommkz_deep.py +│ ├── test_kommunikation_readonly.py +│ ├── test_kommart_values.py +│ ├── verify_advoware_kommunikation_ids.py +│ └── README.md +│ +├── adressen_sync/ # Adressen Sync Tests (geplant) +│ ├── test_adressen_api.py +│ ├── test_adressen_sync.py +│ ├── test_adressen_delete_matching.py +│ ├── test_hauptadresse_*.py +│ └── README.md +│ +├── espocrm_tests/ # EspoCRM API Tests +│ ├── test_espocrm_kommunikation.py +│ ├── test_espocrm_phone_email_entities.py +│ ├── test_espocrm_hidden_ids.py +│ └── README.md +│ +├── analysis/ # Debug & Analyse Scripts +│ ├── analyze_beteiligte_endpoint.py +│ ├── analyze_sync_issues_104860.py +│ ├── compare_entities_104860.py +│ └── README.md +│ +├── calendar_sync/ # Calendar Sync Utilities +│ ├── delete_all_calendars.py +│ ├── delete_employee_locks.py +│ └── README.md +│ +└── tools/ # Allgemeine Utilities + ├── validate_code.py + ├── test_notification.py + ├── test_put_response_detail.py + └── README.md +``` -## Usage +## Kategorien +### 1. Beteiligte Sync ([beteiligte_sync/](beteiligte_sync/)) +Tests für Stammdaten-Synchronisation zwischen EspoCRM und Advoware. +- rowId-basierte Change Detection +- CREATE/UPDATE/DELETE Operations +- Timestamp-Vergleiche & Konflikt-Handling + +### 2. Kommunikation Sync ([kommunikation_sync/](kommunikation_sync/)) +Tests für Phone/Email/Fax Synchronisation. +- Hash-basierte Change Detection +- Base64-Marker System +- 6 Sync-Varianten (Var1-6) +- Empty Slots (DELETE-Workaround) + +### 3. Adressen Sync ([adressen_sync/](adressen_sync/)) +⚠️ **Noch nicht implementiert** - API-Analyse Scripts +- API-Limitierungen Tests +- READ-ONLY Felder Identifikation +- Hauptadressen-Logik + +### 4. EspoCRM Tests ([espocrm_tests/](espocrm_tests/)) +Tests für EspoCRM Custom Entities. +- CBeteiligte Structure Tests +- Kommunikation Arrays +- Sub-Entity Relationships + +### 5. Analysis ([analysis/](analysis/)) +Debug & Analyse Scripts für spezifische Probleme. +- Endpoint-Analyse +- Entity-Vergleiche +- Sync-Issue Debugging + +### 6. Calendar Sync ([calendar_sync/](calendar_sync/)) +Utilities für Google Calendar Sync Management. +- Calendar Cleanup +- Lock Management + +### 7. Tools ([tools/](tools/)) +Allgemeine Entwickler-Tools. +- Code Validation +- Notification Tests +- Response Analysis + +## Quick Start + +### Beteiligte Sync testen ```bash -cd /opt/motia-app - -# Basic usage: Compare by EspoCRM ID (will auto-search in Advoware) -python bitbylaw/scripts/compare_beteiligte.py - -# Advanced: Specify both IDs -python bitbylaw/scripts/compare_beteiligte.py +cd /opt/motia-app/bitbylaw +python scripts/beteiligte_sync/test_beteiligte_sync.py ``` -## Examples - +### Kommunikation Sync testen ```bash -# Example 1: Fetch from EspoCRM and search in Advoware by name -python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc - -# Example 2: Fetch from both systems by ID -python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc 12345 - -# Example 3: Using the virtual environment -source python_modules/bin/activate -python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc +python scripts/kommunikation_sync/test_kommunikation_api.py +python scripts/kommunikation_sync/test_kommunikation_sync_implementation.py ``` -## Requirements - -### Environment Variables - -Make sure these are set in `.env` or environment: - +### Code validieren ```bash -# EspoCRM -ESPOCRM_API_BASE_URL=https://crm.bitbylaw.com/api/v1 -ESPOCRM_MARVIN_API_KEY=your_api_key_here - -# Advoware -ADVOWARE_API_BASE_URL=https://www2.advo-net.net:90/ -ADVOWARE_API_KEY=your_base64_encoded_key -ADVOWARE_USER=your_user -ADVOWARE_PASSWORD=your_password -ADVOWARE_KANZLEI=your_kanzlei -ADVOWARE_DATABASE=your_database -# ... (see config.py for all required vars) +python scripts/tools/validate_code.py services/kommunikation_sync_utils.py ``` -### Dependencies - +### Entity vergleichen ```bash -pip install aiohttp redis python-dotenv requests +python scripts/beteiligte_sync/compare_beteiligte.py --entity-id --betnr ``` -## Output +## Dokumentation -The script produces: +**Hauptdokumentation**: [../docs/SYNC_OVERVIEW.md](../docs/SYNC_OVERVIEW.md) -### 1. Console Output +Für detaillierte Informationen zu jedem Script siehe die README.md in den jeweiligen Unterordnern: +- [beteiligte_sync/README.md](beteiligte_sync/README.md) +- [kommunikation_sync/README.md](kommunikation_sync/README.md) +- [adressen_sync/README.md](adressen_sync/README.md) +- [espocrm_tests/README.md](espocrm_tests/README.md) +- [analysis/README.md](analysis/README.md) +- [calendar_sync/README.md](calendar_sync/README.md) +- [tools/README.md](tools/README.md) -``` -================================================================================ -BETEILIGTE STRUCTURE COMPARISON TOOL -================================================================================ +## Konventionen -EspoCRM Entity ID: 64a3f2b8c9e1234567890abc - -Environment Check: ----------------------------------------- -ESPOCRM_API_BASE_URL: https://crm.bitbylaw.com/api/v1 -ESPOCRM_API_KEY: ✓ Set -ADVOWARE_API_BASE_URL: https://www2.advo-net.net:90/ -ADVOWARE_API_KEY: ✓ Set - -================================================================================ -ESPOCRM - Fetching Beteiligter -================================================================================ - -Trying entity type: Beteiligte - -✓ Success! Found in Beteiligte - -Entity Structure: --------------------------------------------------------------------------------- -{ - "id": "64a3f2b8c9e1234567890abc", - "name": "Max Mustermann", - "firstName": "Max", - "lastName": "Mustermann", - "email": "max@example.com", - "phone": "+49123456789", - ... -} - -================================================================================ -ADVOWARE - Fetching Beteiligter -================================================================================ - -Searching by name: Max Mustermann - Trying endpoint: /contacts - -✓ Found 2 results - -Search Results: --------------------------------------------------------------------------------- -[ - { - "id": 12345, - "full_name": "Max Mustermann", - "email": "max@example.com", - ... - } -] - -================================================================================ -STRUCTURE COMPARISON -================================================================================ - -EspoCRM Fields (25): ----------------------------------------- - id (str) - name (str) - firstName (str) - lastName (str) - email (str) - phone (str) - ... - -Advoware Fields (30): ----------------------------------------- - id (int) - full_name (str) - email (str) - phone_number (str) - ... - -Common Fields (5): ----------------------------------------- - ✓ id - ✓ email - ✗ phone - EspoCRM: +49123456789 - Advoware: 0123456789 - -EspoCRM Only (20): ----------------------------------------- - firstName - lastName - ... - -Advoware Only (25): ----------------------------------------- - full_name - phone_number - ... - -Potential Field Mappings: ----------------------------------------- - firstName → first_name - lastName → last_name - email → email - phone → phone_number - ... - -================================================================================ -Comparison saved to: /opt/motia-app/bitbylaw/scripts/beteiligte_comparison_result.json -================================================================================ -``` - -### 2. JSON Output File - -Saved to `bitbylaw/scripts/beteiligte_comparison_result.json`: - -```json -{ - "espocrm_data": { - "id": "64a3f2b8c9e1234567890abc", - "name": "Max Mustermann", - ... - }, - "advoware_data": { - "id": 12345, - "full_name": "Max Mustermann", - ... - }, - "comparison": { - "espo_fields": ["id", "name", "firstName", ...], - "advo_fields": ["id", "full_name", "email", ...], - "common": ["id", "email"], - "espo_only": ["firstName", "lastName", ...], - "advo_only": ["full_name", "phone_number", ...], - "suggested_mappings": [ - ["firstName", "first_name"], - ["lastName", "last_name"], - ... - ] - } -} -``` - -## How It Works - -### 1. EspoCRM Fetch - -The script tries multiple entity types to find the data: -- `Beteiligte` (custom VMH entity) -- `Contact` (standard) -- `Account` (standard) -- `Lead` (standard) - -### 2. Advoware Fetch - -**By ID (if provided):** -- Tries: `/contacts/{id}`, `/parties/{id}`, `/clients/{id}` - -**By Name (if EspoCRM data available):** -- Searches: `/contacts?search=...`, `/parties?search=...`, `/clients?search=...` - -### 3. Comparison - -- Lists all fields from both systems -- Identifies common fields (same name) -- Shows values for common fields -- Suggests mappings based on naming patterns -- Exports full comparison to JSON - -## Troubleshooting - -### "ESPOCRM_API_KEY not set" +### Naming +- `test_*.py` - Test-Scripts +- `analyze_*.py` - Analyse-Scripts +- `compare_*.py` - Vergleichs-Scripts +- `verify_*.py` - Verifikations-Scripts +### Ausführung +Alle Scripts sollten aus dem Projekt-Root ausgeführt werden: ```bash -# Check if .env exists and has the key -cat .env | grep ESPOCRM_MARVIN_API_KEY - -# Or set it manually -export ESPOCRM_MARVIN_API_KEY=your_key_here +cd /opt/motia-app/bitbylaw +python scripts//