diff --git a/bitbylaw/.env.example b/bitbylaw/.env.example new file mode 100644 index 00000000..421bdf99 --- /dev/null +++ b/bitbylaw/.env.example @@ -0,0 +1,37 @@ +# Redis Configuration +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_DB_ADVOWARE_CACHE=1 +REDIS_DB_CALENDAR_SYNC=2 +REDIS_TIMEOUT_SECONDS=5 + +# Advoware API +ADVOWARE_API_BASE_URL=https://www2.advo-net.net:90/ +ADVOWARE_PRODUCT_ID=64 +ADVOWARE_APP_ID=your_app_id +ADVOWARE_API_KEY=your_api_key_base64 +ADVOWARE_KANZLEI=your_kanzlei +ADVOWARE_DATABASE=your_database +ADVOWARE_USER=your_user +ADVOWARE_ROLE=2 +ADVOWARE_PASSWORD=your_password +ADVOWARE_TOKEN_LIFETIME_MINUTES=55 +ADVOWARE_API_TIMEOUT_SECONDS=30 + +# EspoCRM API +ESPOCRM_API_BASE_URL=https://crm.bitbylaw.com/api/v1 +ESPOCRM_MARVIN_API_KEY=your_espocrm_api_key +ESPOCRM_API_TIMEOUT_SECONDS=30 + +# Google Calendar API (Service Account) +GOOGLE_CALENDAR_SERVICE_ACCOUNT_PATH=service-account.json + +# PostgreSQL (Calendar Sync Hub) +POSTGRES_HOST=localhost +POSTGRES_USER=calendar_sync_user +POSTGRES_PASSWORD=default_password +POSTGRES_DB_NAME=calendar_sync_db + +# Calendar Sync Settings +CALENDAR_SYNC_ANONYMIZE_GOOGLE_EVENTS=true +CALENDAR_SYNC_DEBUG_KUERZEL=SB,AI,RO,OK,BI,ST,UR,PB,VB diff --git a/bitbylaw/ENTITY_MAPPING_CBeteiligte_Advoware.md b/bitbylaw/ENTITY_MAPPING_CBeteiligte_Advoware.md new file mode 100644 index 00000000..5373146d --- /dev/null +++ b/bitbylaw/ENTITY_MAPPING_CBeteiligte_Advoware.md @@ -0,0 +1,286 @@ +# Entity-Mapping: EspoCRM CBeteiligte ↔ Advoware Beteiligte + +Basierend auf dem Vergleich von: +- **EspoCRM**: CBeteiligte Entity ID `68e4af00172be7924` +- **Advoware**: Beteiligter ID `104860` + +## Gemeinsame Felder (direkte Übereinstimmung) + +| EspoCRM Feld | Advoware Feld | Typ | Notes | +|--------------|---------------|-----|-------| +| `name` | `name` | string | Vollständiger Name | +| `rechtsform` | `rechtsform` | string | Rechtsform (z.B. "GmbH", "Frau") | +| `id` | `id` | mixed | **Achtung:** EspoCRM=string, Advoware=int | + +## Namenfelder + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `firstName` | `vorname` | ✓ Direkt | +| `lastName` | `name` | ✓ Bei Personen | +| `middleName` | - | ❌ Kein direktes Mapping | +| `firmenname` | `name` | ✓ Bei Firmen | +| - | `geburtsname` | ← Nur in Advoware | +| - | `kurzname` | ← Nur in Advoware | + +## Kontaktdaten + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `emailAddress` | `emailGesch` | ✓ Geschäftlich | +| `emailAddressData` (array) | `email` | ⚠️ Komplex: Array vs. String | +| `phoneNumber` | `telGesch` | ✓ Geschäftstelefon | +| `phoneNumberData` (array) | `telPrivat` | ⚠️ Komplex | +| - | `mobil` | ← Nur in Advoware | +| - | `faxGesch` / `faxPrivat` | ← Nur in Advoware | +| - | `autotelefon` | ← Nur in Advoware | +| - | `internet` | ← Nur in Advoware | + +**Hinweis**: Advoware hat zusätzlich `kommunikation` Array mit strukturierten Kontaktdaten. + +## Adressdaten + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `adressensIds` / `adressensNames` | `adressen` (array) | ⚠️ Beziehung | +| - | `strasse` | ← Hauptadresse in Advoware Root | +| - | `plz` | ← Hauptadresse in Advoware Root | +| - | `ort` | ← Hauptadresse in Advoware Root | +| - | `anschrift` | ← Formatierte Adresse | + +**Hinweis**: +- EspoCRM: Adressen als Related Entities (IDs/Names) +- Advoware: Hauptadresse im Root-Objekt + `adressen` Array für zusätzliche + +## Anrede & Titel + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `salutationName` | `anrede` | ✓ (z.B. "Frau", "Herr") | +| - | `bAnrede` | ← Briefanrede ("Sehr geehrte...") | +| - | `titel` | ← Titel (Dr., Prof., etc.) | +| - | `zusatz` | ← Namenszusatz | + +## Geburtsdaten + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `dateOfBirth` | `geburtsdatum` | ✓ Direkt | +| - | `sterbedatum` | ← Nur in Advoware | +| - | `familienstand` | ← Nur in Advoware | + +## Handelsregister (für Firmen) + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `handelsregisterNummer` | `handelsRegisterNummer` | ✓ Direkt | +| `handelsregisterArt` (z.B. "HRB") | - | ❌ Nur in EspoCRM | +| - | `registergericht` | ← Nur in Advoware | + +## Bankverbindungen + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| `bankverbindungensIds` / Names | `bankkverbindungen` (array) | ⚠️ Related Entity vs. Array | + +## Beteiligungen/Akten + +| EspoCRM Feld | Advoware Feld | Mapping | +|--------------|---------------|---------| +| - | `beteiligungen` (array) | ← Nur in Advoware | + +**Hinweis**: Advoware speichert die Akten-Beteiligungen direkt beim Beteiligten. + +## EspoCRM-spezifische Felder + +| Feld | Zweck | +|------|-------| +| `betnr` | Beteiligten-Nummer (= Advoware `betNr`) | +| `advowareLastSync` | Zeitstempel der letzten Synchronisation | +| `syncStatus` | Status: "clean", "dirty", "syncing" | +| `disgTyp` | DISC-Persönlichkeitstyp | +| `description` | Notizen/Beschreibung | +| `createdAt` / `createdById` / `createdByName` | Audit-Felder | +| `modifiedAt` / `modifiedById` / `modifiedByName` | Audit-Felder | +| `assignedUserId` / `assignedUserName` | Zuweisungen | +| `teamsIds` / `teamsNames` | Team-Zugehörigkeit | +| `deleted` | Soft-Delete Flag | +| `isFollowed` / `followersIds` | Social Features | + +## Advoware-spezifische Felder + +| Feld | Zweck | +|------|-------| +| `betNr` | Interne Beteiligten-Nummer | +| `rowId` | Datenbank Row-ID | +| `art` | Beteiligten-Art | +| `angelegtAm` / `angelegtVon` | Erstellt | +| `geaendertAm` / `geaendertVon` | Geändert | +| `kontaktpersonen` (array) | Kontaktpersonen bei Firmen | +| `ePost` / `bea` | Spezielle Kommunikationskanäle | + +## Mapping-Strategie + +### 1. Person (Natürliche Person) + +```python +espocrm_to_advoware = { + 'firstName': 'vorname', + 'lastName': 'name', + 'dateOfBirth': 'geburtsdatum', + 'rechtsform': 'rechtsform', # z.B. "Herr", "Frau" + 'salutationName': 'anrede', + 'emailAddress': 'emailGesch', + 'phoneNumber': 'telGesch', +} +``` + +### 2. Firma (Juristische Person) + +```python +espocrm_to_advoware = { + 'firmenname': 'name', + 'rechtsform': 'rechtsform', # z.B. "GmbH", "AG" + 'handelsregisterNummer': 'handelsRegisterNummer', + 'emailAddress': 'emailGesch', + 'phoneNumber': 'telGesch', +} +``` + +### 3. Adressen + +**EspoCRM → Advoware**: +- Lade Related Entity `Adressen` via `adressensIds` +- Mappe Hauptadresse zu Root-Feldern `strasse`, `plz`, `ort` +- Zusätzliche Adressen in `adressen` Array + +**Advoware → EspoCRM**: +- Hauptadresse aus Root-Feldern +- `adressen` Array → Related Entities in EspoCRM + +### 4. Kontaktdaten (Komplex) + +**EspoCRM `emailAddressData`**: +```json +[ + { + "emailAddress": "primary@example.com", + "primary": true, + "optOut": false, + "invalid": false + } +] +``` + +**Advoware `kommunikation`**: +```json +[ + { + "id": 88002, + "kommArt": 0, // 0=Telefon, 1=Email, etc. + "tlf": "0511/12345-60", + "online": false + } +] +``` + +**Mapping**: Erfordert Transformation basierend auf `kommArt`. + +## Sync-Richtungen + +### EspoCRM → Advoware (Webhook-getrieben) + +1. Webhook empfängt `CBeteiligte` create/update/delete +2. Mappe Felder gemäß Tabelle oben +3. `POST /api/v1/advonet/Beteiligte` (create) oder + `PUT /api/v1/advonet/Beteiligte/{betNr}` (update) +4. Update `advowareLastSync` und `syncStatus` in EspoCRM + +### Advoware → EspoCRM (Polling oder Webhook) + +1. Überwache Änderungen in Advoware +2. Mappe Felder zurück +3. `PUT /api/v1/CBeteiligte/{id}` in EspoCRM +4. Setze `syncStatus = "clean"` + +## Konflikte & Regeln + +| Szenario | Regel | +|----------|-------| +| Beide Systeme geändert | Advoware als Master (führendes System) | +| Feld nur in EspoCRM | Ignorieren beim Export, behalten | +| Feld nur in Advoware | Null/Leer in EspoCRM setzen | +| `betnr` vs. `betNr` | Sync-Link: Muss identisch sein | + +## ID-Mapping + +**Problem**: EspoCRM und Advoware haben unterschiedliche ID-Systeme. + +**Lösung**: +- EspoCRM `betnr` Feld = Advoware `betNr` +- Dies ist der Sync-Link zwischen beiden Systemen +- Bei Create in EspoCRM: `betnr` erst nach Advoware-Insert setzen +- Bei Create in Advoware: EspoCRM ID in Custom Field speichern? + +## Nächste Schritte + +1. **Mapper-Modul erstellen**: `bitbylaw/services/espocrm_mapper.py` + - `map_cbeteiligte_to_advoware(espo_data) -> advo_data` + - `map_advoware_to_cbeteiligte(advo_data) -> espo_data` + +2. **Sync-Event-Step implementieren**: `bitbylaw/steps/vmh/beteiligte_sync_event_step.py` + - Subscribe to `vmh.beteiligte.create/update/delete` + - Fetch full entity from EspoCRM + - Transform via Mapper + - Write to Advoware + - Update sync metadata + +3. **Testing**: + - Unit Tests für Mapper + - Integration Tests mit Sandbox-Daten + - Konflikt-Szenarien testen + +4. **Error Handling**: + - Retry-Logic bei API-Fehlern + - Validation vor dem Sync + - Rollback bei Fehlern? + - Logging aller Sync-Operationen + +5. **Performance**: + - Batch-Processing für mehrere Beteiligte + - Rate Limiting beachten + - Caching von Lookup-Daten + +## Beispiel-Transformation + +### EspoCRM CBeteiligte: +```json +{ + "id": "68e4af00172be7924", + "firstName": "Angela", + "lastName": "Mustermanns", + "rechtsform": "Frau", + "emailAddress": "angela@example.com", + "phoneNumber": "0511/12345", + "betnr": 104860, + "handelsregisterNummer": null +} +``` + +### Advoware Beteiligter: +```json +{ + "betNr": 104860, + "vorname": "Angela", + "name": "Mustermanns", + "rechtsform": "Frau", + "anrede": "Frau", + "emailGesch": "angela@example.com", + "telGesch": "0511/12345" +} +``` + +--- + +**Generiert am**: 2026-02-07 +**Basierend auf**: Real-Daten-Vergleich mit `scripts/compare_beteiligte.py` diff --git a/bitbylaw/ESPOCRM_INTEGRATION_NEXT_STEPS.md b/bitbylaw/ESPOCRM_INTEGRATION_NEXT_STEPS.md new file mode 100644 index 00000000..104fd891 --- /dev/null +++ b/bitbylaw/ESPOCRM_INTEGRATION_NEXT_STEPS.md @@ -0,0 +1,114 @@ +# EspoCRM Integration - Nächste Schritte + +## ✅ Bereits erstellt: + +### 1. EspoCRM Service (`services/espocrm.py`) +- Vollständiger API-Client mit allen CRUD-Operationen +- X-Api-Key Authentifizierung +- Error Handling und Logging +- Redis-Integration für Caching/Rate Limiting + +### 2. Compare Script (`scripts/compare_beteiligte.py`) +- Liest Beteiligten-Daten aus EspoCRM und Advoware +- Zeigt Struktur-Unterschiede +- Hilft beim Entity-Mapping + +## 🔧 Setup + +1. **Umgebungsvariablen setzen**: + ```bash + cp .env.example .env + # Dann .env editieren und echte Keys eintragen + ``` + +2. **EspoCRM API Key besorgen**: + - In EspoCRM Admin Panel: Administration → API Users + - Neuen API User erstellen oder bestehenden Key kopieren + - In `.env` als `ESPOCRM_MARVIN_API_KEY` eintragen + +## 🧪 Testing + +### Compare Script ausführen: +```bash +cd /opt/motia-app/bitbylaw +source python_modules/bin/activate + +# Mit EspoCRM ID (sucht automatisch in Advoware nach Namen) +python scripts/compare_beteiligte.py + +# Mit beiden IDs +python scripts/compare_beteiligte.py +``` + +**Beispiel**: +```bash +python scripts/compare_beteiligte.py 507f1f77bcf86cd799439011 12345 +``` + +### Output zeigt: +- Alle Felder aus EspoCRM +- Alle Felder aus Advoware +- Strukturunterschiede +- Mapping-Vorschläge + +## 📋 Nächste Schritte + +### 1. Entity-Mapping definieren +Basierend auf dem Compare-Output: +- `bitbylaw/services/espocrm_mapper.py` erstellen +- Mapping-Funktionen für Beteiligte ↔ Personen/Firmen +- Feld-Transformationen + +### 2. Sync Event Step implementieren +`bitbylaw/steps/vmh/beteiligte_sync_event_step.py`: +- Events von Webhooks verarbeiten +- EspoCRM API Client nutzen +- Mapper für Transformation +- In Advoware schreiben (via Proxy) +- Redis Cleanup + +### 3. Testing & Integration +- Unit Tests für Mapper +- Integration Tests mit echten APIs +- Error Handling testen +- Rate Limiting verifizieren + +## 📚 Dokumentation + +- **Service**: `services/ESPOCRM_SERVICE.md` +- **Script README**: `scripts/compare_beteiligte_README.md` +- **API Docs**: `docs/API.md` (VMH Webhooks Sektion) +- **Architektur**: `docs/ARCHITECTURE.md` (EspoCRM Integration) + +## 🔍 Tipps + +### EspoCRM Entity Types +Häufige Entity-Types in EspoCRM: +- `Contact` - Personen +- `Account` - Firmen/Organisationen +- `Lead` - Leads +- `Opportunity` - Verkaufschancen +- Custom Entities (z.B. `CVmhBeteiligte`, `CVmhErstgespraech`) + +### Advoware Mapping +- Person → `personen` Endpoint +- Firma → `firmen` Endpoint +- Beide sind "Beteiligte" in Advoware-Sprache + +### API Endpoints +```bash +# EspoCRM +curl -X GET "https://crm.bitbylaw.com/api/v1/Contact/ID" \ + -H "X-Api-Key: YOUR_KEY" + +# Advoware (via Proxy) +curl -X GET "http://localhost:3000/advoware/personen/ID" \ + -H "Authorization: Bearer YOUR_TOKEN" +``` + +## ❓ Support + +Bei Fragen siehe: +- EspoCRM API Docs: https://docs.espocrm.com/development/api/ +- Advoware Integration: `docs/ADVOWARE_SERVICE.md` +- Motia Framework: `docs/DEVELOPMENT.md` diff --git a/bitbylaw/config.py b/bitbylaw/config.py index 4eded45a..6f33eccf 100644 --- a/bitbylaw/config.py +++ b/bitbylaw/config.py @@ -38,4 +38,9 @@ class Config: # Calendar Sync settings CALENDAR_SYNC_ANONYMIZE_GOOGLE_EVENTS = os.getenv('CALENDAR_SYNC_ANONYMIZE_GOOGLE_EVENTS', 'true').lower() == 'true' CALENDAR_SYNC_DEBUG_KUERZEL = [k.strip().upper() for k in os.getenv('CALENDAR_SYNC_DEBUG_KUERZEL', 'SB,AI,RO,OK,BI,ST,UR,PB,VB').split(',')] - ADVOWARE_WRITE_PROTECTION = True \ No newline at end of file + ADVOWARE_WRITE_PROTECTION = True + + # EspoCRM API settings + ESPOCRM_API_BASE_URL = os.getenv('ESPOCRM_API_BASE_URL', 'https://crm.bitbylaw.com/api/v1') + ESPOCRM_API_KEY = os.getenv('ESPOCRM_MARVIN_API_KEY', '') + ESPOCRM_API_TIMEOUT_SECONDS = int(os.getenv('ESPOCRM_API_TIMEOUT_SECONDS', '30')) \ No newline at end of file diff --git a/bitbylaw/docs/DEPLOYMENT.md b/bitbylaw/docs/DEPLOYMENT.md index 6ed2eef6..e69de29b 100644 --- a/bitbylaw/docs/DEPLOYMENT.md +++ b/bitbylaw/docs/DEPLOYMENT.md @@ -1,624 +0,0 @@ -# Deployment Guide - -## Production Deployment - -### Prerequisites - -- Root/sudo access zum Server -- Ubuntu/Debian Linux (tested on Ubuntu 22.04+) -- Internet-Zugang für Package-Installation - -### Installation Steps - -#### 1. System Dependencies - -```bash -# Update system -sudo apt-get update -sudo apt-get upgrade -y - -# Install Node.js 18.x -curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - -sudo apt-get install -y nodejs - -# Install Python 3.13 -sudo apt-get install -y python3.13 python3.13-venv python3.13-dev - -# Install Redis -sudo apt-get install -y redis-server - -# Install Git -sudo apt-get install -y git - -# Start Redis -sudo systemctl enable redis-server -sudo systemctl start redis-server -``` - -#### 2. Application Setup - -```bash -# Create application directory -sudo mkdir -p /opt/motia-app -cd /opt/motia-app - -# Clone repository (oder rsync von Development) -git clone bitbylaw -cd bitbylaw - -# Create www-data user if not exists -sudo useradd -r -s /bin/bash www-data || true - -# Set ownership -sudo chown -R www-data:www-data /opt/motia-app -``` - -#### 3. Node.js Dependencies - -```bash -# Als www-data user -sudo -u www-data bash -cd /opt/motia-app/bitbylaw - -# Install Node.js packages -npm install - -# Build TypeScript (falls nötig) -npm run build -``` - -#### 4. Python Dependencies - -```bash -# Als www-data user -cd /opt/motia-app/bitbylaw - -# Create virtual environment -python3.13 -m venv python_modules - -# Activate -source python_modules/bin/activate - -# Install dependencies -pip install -r requirements.txt - -# Deactivate -deactivate -``` - -#### 5. Service Account Setup - -```bash -# Copy service account JSON -sudo cp service-account.json /opt/motia-app/service-account.json - -# Set secure permissions -sudo chmod 600 /opt/motia-app/service-account.json -sudo chown www-data:www-data /opt/motia-app/service-account.json -``` - -Siehe auch: [GOOGLE_SETUP_README.md](../GOOGLE_SETUP_README.md) - -#### 6. systemd Service - -Erstellen Sie `/etc/systemd/system/motia.service`: - -```ini -[Unit] -Description=Motia Backend Framework -After=network.target redis-server.service - -[Service] -Type=simple -User=www-data -WorkingDirectory=/opt/motia-app/bitbylaw - -# Environment Variables -Environment=NODE_ENV=production -Environment=NODE_OPTIONS=--max-old-space-size=8192 --inspect --heapsnapshot-signal=SIGUSR2 -Environment=HOST=0.0.0.0 -Environment=MOTIA_LOG_LEVEL=info -Environment=NPM_CONFIG_CACHE=/opt/motia-app/.npm-cache - -# Advoware Configuration (ADJUST VALUES!) -Environment=ADVOWARE_API_BASE_URL=https://www2.advo-net.net:90/ -Environment=ADVOWARE_PRODUCT_ID=64 -Environment=ADVOWARE_APP_ID=your_app_id -Environment=ADVOWARE_API_KEY=your_api_key_base64 -Environment=ADVOWARE_KANZLEI=your_kanzlei -Environment=ADVOWARE_DATABASE=your_database -Environment=ADVOWARE_USER=your_user -Environment=ADVOWARE_ROLE=2 -Environment=ADVOWARE_PASSWORD=your_password -Environment=ADVOWARE_WRITE_PROTECTION=false - -# Redis Configuration -Environment=REDIS_HOST=localhost -Environment=REDIS_PORT=6379 -Environment=REDIS_DB_ADVOWARE_CACHE=1 -Environment=REDIS_DB_CALENDAR_SYNC=2 - -# Google Calendar -Environment=GOOGLE_CALENDAR_SERVICE_ACCOUNT_PATH=/opt/motia-app/service-account.json - -# EspoCRM (if used) -Environment=ESPOCRM_MARVIN_API_KEY=your_webhook_key - -# Start Command -ExecStart=/bin/bash -c 'source /opt/motia-app/python_modules/bin/activate && /usr/bin/npm start' - -# Restart Policy -Restart=always -RestartSec=10 - -# Security -NoNewPrivileges=true -PrivateTmp=true - -[Install] -WantedBy=multi-user.target -``` - -**WICHTIG**: Passen Sie alle `your_*` Werte an! - -#### 7. Enable and Start Service - -```bash -# Reload systemd -sudo systemctl daemon-reload - -# Enable service (autostart) -sudo systemctl enable motia.service - -# Start service -sudo systemctl start motia.service - -# Check status -sudo systemctl status motia.service -``` - -#### 8. Verify Installation - -```bash -# Check logs -sudo journalctl -u motia.service -f - -# Test API -curl http://localhost:3000/health # (wenn implementiert) - -# Test Advoware Proxy -curl "http://localhost:3000/advoware/proxy?endpoint=employees" -``` - -## Reverse Proxy Setup (nginx) - -### Install nginx - -```bash -sudo apt-get install -y nginx -``` - -### Configure - -`/etc/nginx/sites-available/motia`: - -```nginx -upstream motia_backend { - server 127.0.0.1:3000; -} - -server { - listen 80; - server_name your-domain.com; - - # Redirect to HTTPS - return 301 https://$server_name$request_uri; -} - -server { - listen 443 ssl http2; - server_name your-domain.com; - - # SSL Configuration (Let's Encrypt) - ssl_certificate /etc/letsencrypt/live/your-domain.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/your-domain.com/privkey.pem; - - # Security Headers - add_header X-Frame-Options "SAMEORIGIN" always; - add_header X-Content-Type-Options "nosniff" always; - add_header X-XSS-Protection "1; mode=block" always; - - # Proxy Settings - location / { - proxy_pass http://motia_backend; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_cache_bypass $http_upgrade; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - # Timeouts - proxy_connect_timeout 60s; - proxy_send_timeout 60s; - proxy_read_timeout 60s; - } - - # Access Log - access_log /var/log/nginx/motia-access.log; - error_log /var/log/nginx/motia-error.log; -} -``` - -### Enable and Restart - -```bash -# Enable site -sudo ln -s /etc/nginx/sites-available/motia /etc/nginx/sites-enabled/ - -# Test configuration -sudo nginx -t - -# Restart nginx -sudo systemctl restart nginx -``` - -### SSL Certificate (Let's Encrypt) - -```bash -# Install certbot -sudo apt-get install -y certbot python3-certbot-nginx - -# Obtain certificate -sudo certbot --nginx -d your-domain.com - -# Auto-renewal is configured automatically -``` - -## Firewall Configuration - -```bash -# Allow SSH -sudo ufw allow 22/tcp - -# Allow HTTP/HTTPS (if using nginx) -sudo ufw allow 80/tcp -sudo ufw allow 443/tcp - -# Enable firewall -sudo ufw enable -``` - -**Wichtig**: Port 3000 NICHT öffentlich öffnen (nur via nginx reverse proxy) - -## Monitoring - -### systemd Service Status - -```bash -# Status anzeigen -sudo systemctl status motia.service - -# Ist enabled? -sudo systemctl is-enabled motia.service - -# Ist aktiv? -sudo systemctl is-active motia.service -``` - -### Logs - -```bash -# Live logs -sudo journalctl -u motia.service -f - -# Last 100 lines -sudo journalctl -u motia.service -n 100 - -# Since today -sudo journalctl -u motia.service --since today - -# Filter by priority (error only) -sudo journalctl -u motia.service -p err -``` - -### Resource Usage - -```bash -# CPU and Memory -sudo systemctl status motia.service - -# Detailed process info -ps aux | grep motia - -# Memory usage -sudo pmap $(pgrep -f "motia start") | tail -n 1 -``` - -### Redis Monitoring - -```bash -# Connect to Redis -redis-cli - -# Show info -INFO - -# Show database sizes -INFO keyspace - -# Monitor commands (real-time) -MONITOR - -# Show memory usage -MEMORY USAGE -``` - -## Backup Strategy - -### Application Code - -```bash -# Git-based backup -cd /opt/motia-app/bitbylaw -git pull origin main - -# Or: rsync backup -rsync -av /opt/motia-app/bitbylaw/ /backup/motia-app/ -``` - -### Redis Data - -```bash -# RDB snapshot (automatic by Redis) -# Location: /var/lib/redis/dump.rdb - -# Manual backup -sudo cp /var/lib/redis/dump.rdb /backup/redis-dump-$(date +%Y%m%d).rdb - -# Restore -sudo systemctl stop redis-server -sudo cp /backup/redis-dump-20260207.rdb /var/lib/redis/dump.rdb -sudo chown redis:redis /var/lib/redis/dump.rdb -sudo systemctl start redis-server -``` - -### Configuration - -```bash -# Backup systemd service -sudo cp /etc/systemd/system/motia.service /backup/motia.service - -# Backup nginx config -sudo cp /etc/nginx/sites-available/motia /backup/nginx-motia.conf - -# Backup service account -sudo cp /opt/motia-app/service-account.json /backup/service-account.json.backup -``` - -## Updates & Maintenance - -### Application Update - -```bash -# 1. Pull latest code -cd /opt/motia-app/bitbylaw -sudo -u www-data git pull origin main - -# 2. Update dependencies -sudo -u www-data npm install -sudo -u www-data bash -c 'source python_modules/bin/activate && pip install -r requirements.txt' - -# 3. Restart service -sudo systemctl restart motia.service - -# 4. Verify -sudo journalctl -u motia.service -f -``` - -### Zero-Downtime Deployment - -Für zukünftige Implementierung mit Blue-Green Deployment: - -```bash -# 1. Deploy to staging directory -# 2. Run health checks -# 3. Switch symlink -# 4. Reload service -# 5. Rollback if issues -``` - -### Database Migrations - -**Aktuell**: Keine Datenbank-Migrationen (nur Redis) - -**Zukünftig** (PostgreSQL): -```bash -# Run migrations -python manage.py migrate -``` - -## Security Hardening - -### File Permissions - -```bash -# Application files -sudo chown -R www-data:www-data /opt/motia-app -sudo chmod 755 /opt/motia-app -sudo chmod 755 /opt/motia-app/bitbylaw - -# Service account -sudo chmod 600 /opt/motia-app/service-account.json -sudo chown www-data:www-data /opt/motia-app/service-account.json - -# No world-readable secrets -sudo find /opt/motia-app -type f -name "*.json" -exec chmod 600 {} \; -``` - -### Redis Security - -```bash -# Edit Redis config -sudo nano /etc/redis/redis.conf - -# Bind to localhost only -bind 127.0.0.1 ::1 - -# Disable dangerous commands (optional) -rename-command FLUSHDB "" -rename-command FLUSHALL "" -rename-command CONFIG "" - -# Restart Redis -sudo systemctl restart redis-server -``` - -### systemd Hardening - -Bereits in Service-Datei enthalten: -- `NoNewPrivileges=true` - Verhindert Privilege-Escalation -- `PrivateTmp=true` - Isoliertes /tmp -- User: `www-data` (non-root) - -Weitere Optionen: -```ini -[Service] -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/opt/motia-app -``` - -## Disaster Recovery - -### Service Crashed - -```bash -# Check status -sudo systemctl status motia.service - -# View logs -sudo journalctl -u motia.service -n 100 - -# Restart -sudo systemctl restart motia.service - -# If still failing, check: -# - Redis is running -# - Service account file exists -# - Environment variables are set -``` - -### Redis Data Loss - -```bash -# Restore from backup -sudo systemctl stop redis-server -sudo cp /backup/redis-dump-latest.rdb /var/lib/redis/dump.rdb -sudo chown redis:redis /var/lib/redis/dump.rdb -sudo systemctl start redis-server - -# Clear specific data if corrupted -redis-cli -n 1 FLUSHDB # Advoware cache -redis-cli -n 2 FLUSHDB # Calendar sync -``` - -### Complete System Failure - -```bash -# 1. Fresh server setup (siehe Installation Steps) -# 2. Restore application code from Git/Backup -# 3. Restore configuration (systemd, nginx) -# 4. Restore service-account.json -# 5. Restore Redis data (optional, will rebuild) -# 6. Start services -``` - -## Performance Tuning - -### Node.js Memory - -In systemd service: -```ini -Environment=NODE_OPTIONS=--max-old-space-size=8192 # 8GB -``` - -### Redis Memory - -In `/etc/redis/redis.conf`: -``` -maxmemory 2gb -maxmemory-policy allkeys-lru -``` - -### Linux Kernel - -```bash -# Increase file descriptors -echo "fs.file-max = 65536" | sudo tee -a /etc/sysctl.conf -sudo sysctl -p - -# For www-data user -sudo nano /etc/security/limits.conf -# Add: -www-data soft nofile 65536 -www-data hard nofile 65536 -``` - -## Health Checks - -### Automated Monitoring - -Cron job für Health Checks: - -```bash -# /usr/local/bin/motia-health-check.sh -#!/bin/bash -if ! systemctl is-active --quiet motia.service; then - echo "Motia service is down!" | mail -s "ALERT: Motia Down" admin@example.com - systemctl start motia.service -fi -``` - -```bash -# Add to crontab -sudo crontab -e -# Add line: -*/5 * * * * /usr/local/bin/motia-health-check.sh -``` - -### External Monitoring - -Services wie Uptime Robot, Pingdom, etc. können verwendet werden: -- HTTP Endpoint: `https://your-domain.com/health` -- Check-Interval: 5 Minuten -- Alert via Email/SMS - -## Rollback Procedure - -```bash -# 1. Stop current service -sudo systemctl stop motia.service - -# 2. Revert to previous version -cd /opt/motia-app/bitbylaw -sudo -u www-data git log # Find previous commit -sudo -u www-data git reset --hard - -# 3. Restore dependencies (if needed) -sudo -u www-data npm install - -# 4. Start service -sudo systemctl start motia.service - -# 5. Verify -sudo journalctl -u motia.service -f -``` - -## Related Documentation - -- [Architecture](ARCHITECTURE.md) -- [Configuration](CONFIGURATION.md) -- [Troubleshooting](TROUBLESHOOTING.md) diff --git a/bitbylaw/docs/TROUBLESHOOTING.md b/bitbylaw/docs/TROUBLESHOOTING.md index 47ec5541..e69de29b 100644 --- a/bitbylaw/docs/TROUBLESHOOTING.md +++ b/bitbylaw/docs/TROUBLESHOOTING.md @@ -1,800 +0,0 @@ -# Troubleshooting Guide - -## Service Issues - -### Service Won't Start - -**Symptoms**: `systemctl start motia.service` schlägt fehl - -**Diagnose**: -```bash -# Check service status -sudo systemctl status motia.service - -# View detailed logs -sudo journalctl -u motia.service -n 100 --no-pager - -# Check for port conflicts -sudo netstat -tlnp | grep 3000 -``` - -**Häufige Ursachen**: - -1. **Port 3000 bereits belegt**: - ```bash - # Find process - sudo lsof -i :3000 - - # Kill process - sudo kill -9 - ``` - -2. **Fehlende Dependencies**: - ```bash - cd /opt/motia-app/bitbylaw - sudo -u www-data npm install - sudo -u www-data bash -c 'source python_modules/bin/activate && pip install -r requirements.txt' - ``` - -3. **Falsche Permissions**: - ```bash - sudo chown -R www-data:www-data /opt/motia-app - sudo chmod 600 /opt/motia-app/service-account.json - ``` - -4. **Environment Variables fehlen**: - ```bash - # Check systemd environment - sudo systemctl show motia.service -p Environment - - # Verify required vars - sudo systemctl cat motia.service | grep Environment - ``` - -### Service Keeps Crashing - -**Symptoms**: Service startet, crashed aber nach kurzer Zeit - -**Diagnose**: -```bash -# Watch logs in real-time -sudo journalctl -u motia.service -f - -# Check for OOM (Out of Memory) -dmesg | grep -i "out of memory" -sudo grep -i "killed process" /var/log/syslog -``` - -**Solutions**: - -1. **Memory Limit erhöhen**: - ```ini - # In /etc/systemd/system/motia.service - Environment=NODE_OPTIONS=--max-old-space-size=8192 - ``` - -2. **Python Memory Leak**: - ```bash - # Check memory usage - ps aux | grep python - - # Restart service periodically (workaround) - # Add to crontab: - 0 3 * * * systemctl restart motia.service - ``` - -3. **Unhandled Exception**: - ```bash - # Check error logs - sudo journalctl -u motia.service -p err - - # Add try-catch in problematic step - ``` - -## Redis Issues - -### Redis Connection Failed - -**Symptoms**: "Redis connection failed" in logs - -**Diagnose**: -```bash -# Check Redis status -sudo systemctl status redis-server - -# Test connection -redis-cli ping - -# Check config -redis-cli CONFIG GET bind -redis-cli CONFIG GET port -``` - -**Solutions**: - -1. **Redis not running**: - ```bash - sudo systemctl start redis-server - sudo systemctl enable redis-server - ``` - -2. **Wrong host/port**: - ```bash - # Check environment - echo $REDIS_HOST - echo $REDIS_PORT - - # Test connection - redis-cli -h $REDIS_HOST -p $REDIS_PORT ping - ``` - -3. **Permission denied**: - ```bash - # Check Redis log - sudo tail -f /var/log/redis/redis-server.log - - # Fix permissions - sudo chown redis:redis /var/lib/redis - sudo chmod 750 /var/lib/redis - ``` - -### Redis Out of Memory - -**Symptoms**: "OOM command not allowed" errors - -**Diagnose**: -```bash -# Check memory usage -redis-cli INFO memory - -# Check maxmemory setting -redis-cli CONFIG GET maxmemory -``` - -**Solutions**: - -1. **Increase maxmemory**: - ```bash - # In /etc/redis/redis.conf - maxmemory 2gb - maxmemory-policy allkeys-lru - - sudo systemctl restart redis-server - ``` - -2. **Clear old data**: - ```bash - # Clear cache (safe for Advoware tokens) - redis-cli -n 1 FLUSHDB - - # Clear calendar sync state - redis-cli -n 2 FLUSHDB - ``` - -3. **Check for memory leaks**: - ```bash - # Find large keys - redis-cli --bigkeys - - # Check specific key size - redis-cli MEMORY USAGE - ``` - -## Advoware API Issues - -### Authentication Failed - -**Symptoms**: "401 Unauthorized" oder "HMAC signature invalid" - -**Diagnose**: -```bash -# Check logs for auth errors -sudo journalctl -u motia.service | grep -i "auth\|token\|401" - -# Test token fetch manually -python3 << 'EOF' -from services.advoware import AdvowareAPI -api = AdvowareAPI() -token = api.get_access_token(force_refresh=True) -print(f"Token: {token[:20]}...") -EOF -``` - -**Solutions**: - -1. **Invalid API Key**: - ```bash - # Verify API Key is Base64 - echo $ADVOWARE_API_KEY | base64 -d - - # Re-encode if needed - echo -n "raw_key" | base64 - ``` - -2. **Wrong credentials**: - ```bash - # Verify environment variables - sudo systemctl show motia.service -p Environment | grep ADVOWARE - - # Update in systemd service - sudo nano /etc/systemd/system/motia.service - sudo systemctl daemon-reload - sudo systemctl restart motia.service - ``` - -3. **Token expired**: - ```bash - # Clear cached token - redis-cli -n 1 DEL advoware_access_token advoware_token_timestamp - - # Retry request (will fetch new token) - ``` - -### API Timeout - -**Symptoms**: "Request timeout" oder "API call failed" - -**Diagnose**: -```bash -# Check API response time -time curl "http://localhost:3000/advoware/proxy?endpoint=employees" - -# Check network connectivity -ping www2.advo-net.net -curl -I https://www2.advo-net.net:90/ -``` - -**Solutions**: - -1. **Increase timeout**: - ```bash - # In environment - export ADVOWARE_API_TIMEOUT_SECONDS=60 - - # Or in systemd service - Environment=ADVOWARE_API_TIMEOUT_SECONDS=60 - ``` - -2. **Network issues**: - ```bash - # Check firewall - sudo ufw status - - # Test direct connection - curl -v https://www2.advo-net.net:90/ - ``` - -3. **Advoware API down**: - ```bash - # Wait and retry - # Implement exponential backoff in code - ``` - -## Google Calendar Issues - -### Service Account Not Found - -**Symptoms**: "service-account.json not found" - -**Diagnose**: -```bash -# Check file exists -ls -la /opt/motia-app/service-account.json - -# Check permissions -ls -la /opt/motia-app/service-account.json - -# Check environment variable -echo $GOOGLE_CALENDAR_SERVICE_ACCOUNT_PATH -``` - -**Solutions**: - -1. **File missing**: - ```bash - # Copy from backup - sudo cp /backup/service-account.json /opt/motia-app/ - - # Set permissions - sudo chmod 600 /opt/motia-app/service-account.json - sudo chown www-data:www-data /opt/motia-app/service-account.json - ``` - -2. **Wrong path**: - ```bash - # Update environment - # In /etc/systemd/system/motia.service: - Environment=GOOGLE_CALENDAR_SERVICE_ACCOUNT_PATH=/opt/motia-app/service-account.json - - sudo systemctl daemon-reload - sudo systemctl restart motia.service - ``` - -### Calendar API Rate Limit - -**Symptoms**: "403 Rate limit exceeded" oder "429 Too Many Requests" - -**Diagnose**: -```bash -# Check rate limiting in logs -sudo journalctl -u motia.service | grep -i "rate\|403\|429" - -# Check Redis rate limit tokens -redis-cli -n 2 GET google_calendar_api_tokens -``` - -**Solutions**: - -1. **Wait for rate limit reset**: - ```bash - # Rate limit resets every minute - # Wait 60 seconds and retry - ``` - -2. **Adjust rate limit settings**: - ```python - # In calendar_sync_event_step.py - MAX_TOKENS = 7 # Decrease if hitting limits - REFILL_RATE_PER_MS = 7 / 1000 - ``` - -3. **Request quota increase**: - - Go to Google Cloud Console - - Navigate to "APIs & Services" → "Quotas" - - Request increase for Calendar API - -### Calendar Access Denied - -**Symptoms**: "Access denied" oder "Insufficient permissions" - -**Diagnose**: -```bash -# Check service account email -python3 << 'EOF' -import json -with open('/opt/motia-app/service-account.json') as f: - data = json.load(f) - print(f"Service Account: {data['client_email']}") -EOF - -# Test API access -python3 << 'EOF' -from google.oauth2 import service_account -from googleapiclient.discovery import build - -creds = service_account.Credentials.from_service_account_file( - '/opt/motia-app/service-account.json', - scopes=['https://www.googleapis.com/auth/calendar'] -) -service = build('calendar', 'v3', credentials=creds) -result = service.calendarList().list().execute() -print(f"Calendars: {len(result.get('items', []))}") -EOF -``` - -**Solutions**: - -1. **Calendar not shared**: - ```bash - # Share calendar with service account email - # In Google Calendar UI: Settings → Share → Add service account email - ``` - -2. **Wrong scopes**: - ```bash - # Verify scopes in code - # Should be: https://www.googleapis.com/auth/calendar - ``` - -3. **Domain-wide delegation**: - ```bash - # For G Suite, enable domain-wide delegation - # See GOOGLE_SETUP_README.md - ``` - -## Calendar Sync Issues - -### Sync Not Running - -**Symptoms**: Keine Calendar-Updates, keine Sync-Logs - -**Diagnose**: -```bash -# Check if cron is triggering -sudo journalctl -u motia.service | grep -i "calendar_sync_cron" - -# Manually trigger sync -curl -X POST "http://localhost:3000/advoware/calendar/sync" \ - -H "Content-Type: application/json" \ - -d '{"full_content": true}' - -# Check for locks -redis-cli -n 1 KEYS "calendar_sync:lock:*" -``` - -**Solutions**: - -1. **Cron not configured**: - ```python - # Verify calendar_sync_cron_step.py has correct schedule - config = { - 'schedule': '0 2 * * *', # Daily at 2 AM - } - ``` - -2. **Lock stuck**: - ```bash - # Clear all locks - python /opt/motia-app/bitbylaw/delete_employee_locks.py - - # Or manually - redis-cli -n 1 DEL calendar_sync:lock:SB - ``` - -3. **Errors in sync**: - ```bash - # Check error logs - sudo journalctl -u motia.service -p err | grep calendar - ``` - -### Duplicate Events - -**Symptoms**: Events erscheinen mehrfach in Google Calendar - -**Diagnose**: -```bash -# Check for concurrent syncs -redis-cli -n 1 KEYS "calendar_sync:lock:*" - -# Check logs for duplicate processing -sudo journalctl -u motia.service | grep -i "duplicate\|already exists" -``` - -**Solutions**: - -1. **Locking not working**: - ```bash - # Verify Redis lock TTL - redis-cli -n 1 TTL calendar_sync:lock:SB - - # Should return positive number if locked - ``` - -2. **Manual cleanup**: - ```bash - # Delete duplicates in Google Calendar UI - # Or use cleanup script (if available) - ``` - -3. **Improve deduplication logic**: - ```python - # In calendar_sync_event_step.py - # Add better event matching logic - ``` - -### Events Not Syncing - -**Symptoms**: Advoware events nicht in Google Calendar - -**Diagnose**: -```bash -# Check specific employee -curl -X POST "http://localhost:3000/advoware/calendar/sync" \ - -H "Content-Type: application/json" \ - -d '{"kuerzel": "SB", "full_content": true}' - -# Check logs for that employee -sudo journalctl -u motia.service | grep "SB" - -# Check if calendar exists -python3 << 'EOF' -from google.oauth2 import service_account -from googleapiclient.discovery import build - -creds = service_account.Credentials.from_service_account_file( - '/opt/motia-app/service-account.json', - scopes=['https://www.googleapis.com/auth/calendar'] -) -service = build('calendar', 'v3', credentials=creds) -result = service.calendarList().list().execute() -for cal in result.get('items', []): - if 'AW-SB' in cal['summary']: - print(f"Found: {cal['summary']} - {cal['id']}") -EOF -``` - -**Solutions**: - -1. **Calendar doesn't exist**: - ```bash - # Will be auto-created on first sync - # Force sync to trigger creation - ``` - -2. **Date range mismatch**: - ```python - # Check FETCH_FROM and FETCH_TO in calendar_sync_event_step.py - # Default: Previous year to 9 years ahead - ``` - -3. **Write protection enabled**: - ```bash - # Check environment - echo $ADVOWARE_WRITE_PROTECTION - - # Should be "false" for two-way sync - ``` - -## Webhook Issues - -### Webhooks Not Received - -**Symptoms**: EspoCRM sendet Webhooks, aber keine Verarbeitung - -**Diagnose**: -```bash -# Check if endpoint reachable -curl -X POST "http://localhost:3000/vmh/webhook/beteiligte/create" \ - -H "Content-Type: application/json" \ - -d '[{"id": "test-123"}]' - -# Check firewall -sudo ufw status - -# Check nginx logs (if using reverse proxy) -sudo tail -f /var/log/nginx/motia-access.log -sudo tail -f /var/log/nginx/motia-error.log -``` - -**Solutions**: - -1. **Firewall blocking**: - ```bash - # Allow port (if direct access) - sudo ufw allow 3000/tcp - - # Or use reverse proxy (recommended) - ``` - -2. **Wrong URL in EspoCRM**: - ```bash - # Verify URL in EspoCRM webhook configuration - # Should be: https://your-domain.com/vmh/webhook/beteiligte/create - ``` - -3. **SSL certificate issues**: - ```bash - # Check certificate - openssl s_client -connect your-domain.com:443 - - # Renew certificate - sudo certbot renew - ``` - -### Webhook Deduplication Not Working - -**Symptoms**: Mehrfache Verarbeitung derselben Webhooks - -**Diagnose**: -```bash -# Check Redis dedup sets -redis-cli -n 1 SMEMBERS vmh:beteiligte:create_pending -redis-cli -n 1 SMEMBERS vmh:beteiligte:update_pending -redis-cli -n 1 SMEMBERS vmh:beteiligte:delete_pending - -# Check for concurrent webhook processing -sudo journalctl -u motia.service | grep "Webhook.*received" -``` - -**Solutions**: - -1. **Redis SET not working**: - ```bash - # Test Redis SET operations - redis-cli -n 1 SADD test_set "value1" - redis-cli -n 1 SMEMBERS test_set - redis-cli -n 1 DEL test_set - ``` - -2. **Clear dedup sets**: - ```bash - # If corrupted - redis-cli -n 1 DEL vmh:beteiligte:create_pending - redis-cli -n 1 DEL vmh:beteiligte:update_pending - redis-cli -n 1 DEL vmh:beteiligte:delete_pending - ``` - -## Performance Issues - -### High CPU Usage - -**Diagnose**: -```bash -# Check CPU usage -top -p $(pgrep -f "motia start") - -# Profile with Node.js -# Already enabled with --inspect flag -# Connect to chrome://inspect -``` - -**Solutions**: - -1. **Too many parallel syncs**: - ```bash - # Reduce concurrent syncs - # Adjust DEBUG_KUERZEL to process fewer employees - ``` - -2. **Infinite loop**: - ```bash - # Check logs for repeated patterns - sudo journalctl -u motia.service | tail -n 1000 | sort | uniq -c | sort -rn - ``` - -### High Memory Usage - -**Diagnose**: -```bash -# Check memory -ps aux | grep motia | awk '{print $6}' - -# Heap snapshot (if enabled) -kill -SIGUSR2 $(pgrep -f "motia start") -# Snapshot saved to current directory -``` - -**Solutions**: - -1. **Increase memory limit**: - ```ini - # In systemd service - Environment=NODE_OPTIONS=--max-old-space-size=16384 - ``` - -2. **Memory leak**: - ```bash - # Restart service periodically - # Add to crontab: - 0 3 * * * systemctl restart motia.service - ``` - -### Slow API Responses - -**Diagnose**: -```bash -# Measure response time -time curl "http://localhost:3000/advoware/proxy?endpoint=employees" - -# Check for database/Redis latency -redis-cli --latency -``` - -**Solutions**: - -1. **Redis slow**: - ```bash - # Check slow log - redis-cli SLOWLOG GET 10 - - # Optimize Redis - redis-cli CONFIG SET tcp-backlog 511 - ``` - -2. **Advoware API slow**: - ```bash - # Increase timeout - export ADVOWARE_API_TIMEOUT_SECONDS=60 - - # Add caching layer - ``` - -## Debugging Tools - -### Enable Debug Logging - -```bash -# Set in systemd service -Environment=MOTIA_LOG_LEVEL=debug - -sudo systemctl daemon-reload -sudo systemctl restart motia.service -``` - -### Redis Debugging - -```bash -# Connect to Redis -redis-cli - -# Monitor all commands -MONITOR - -# Slow log -SLOWLOG GET 10 - -# Info -INFO all -``` - -### Python Debugging - -```python -# Add to step code -import pdb; pdb.set_trace() - -# Or use logging -context.logger.debug(f"Variable value: {variable}") -``` - -### Node.js Debugging - -```bash -# Connect to inspector -# Chrome DevTools: chrome://inspect -# VSCode: Attach to Process -``` - -## Getting Help - -### Check Logs First - -```bash -# Last 100 lines -sudo journalctl -u motia.service -n 100 - -# Errors only -sudo journalctl -u motia.service -p err - -# Specific time range -sudo journalctl -u motia.service --since "1 hour ago" -``` - -### Common Log Patterns - -**Success**: -``` -[INFO] Calendar sync completed for SB -[INFO] VMH Webhook received -``` - -**Warning**: -``` -[WARNING] Rate limit approaching -[WARNING] Lock already exists for SB -``` - -**Error**: -``` -[ERROR] Redis connection failed -[ERROR] API call failed: 401 Unauthorized -[ERROR] Unexpected error: ... -``` - -### Collect Debug Information - -```bash -# System info -uname -a -node --version -python3 --version - -# Service status -sudo systemctl status motia.service - -# Recent logs -sudo journalctl -u motia.service -n 200 > motia-logs.txt - -# Redis info -redis-cli INFO > redis-info.txt - -# Configuration (redact secrets!) -sudo systemctl show motia.service -p Environment > env.txt -``` - -## Related Documentation - -- [Architecture](ARCHITECTURE.md) -- [Configuration](CONFIGURATION.md) -- [Deployment](DEPLOYMENT.md) -- [Development Guide](DEVELOPMENT.md) diff --git a/bitbylaw/scripts/README.md b/bitbylaw/scripts/README.md new file mode 100644 index 00000000..810cae8c --- /dev/null +++ b/bitbylaw/scripts/README.md @@ -0,0 +1,296 @@ +# Beteiligte Structure Comparison Tool + +## Purpose + +This helper script fetches entity data from both **EspoCRM** and **Advoware** to compare their data structures. This helps understand: + +- What fields exist in each system +- How field names differ +- Potential field mappings for synchronization +- Data type differences + +## Usage + +```bash +cd /opt/motia-app + +# Basic usage: Compare by EspoCRM ID (will auto-search in Advoware) +python bitbylaw/scripts/compare_beteiligte.py + +# Advanced: Specify both IDs +python bitbylaw/scripts/compare_beteiligte.py +``` + +## Examples + +```bash +# Example 1: Fetch from EspoCRM and search in Advoware by name +python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc + +# Example 2: Fetch from both systems by ID +python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc 12345 + +# Example 3: Using the virtual environment +source python_modules/bin/activate +python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc +``` + +## Requirements + +### Environment Variables + +Make sure these are set in `.env` or environment: + +```bash +# EspoCRM +ESPOCRM_API_BASE_URL=https://crm.bitbylaw.com/api/v1 +ESPOCRM_MARVIN_API_KEY=your_api_key_here + +# Advoware +ADVOWARE_API_BASE_URL=https://www2.advo-net.net:90/ +ADVOWARE_API_KEY=your_base64_encoded_key +ADVOWARE_USER=your_user +ADVOWARE_PASSWORD=your_password +ADVOWARE_KANZLEI=your_kanzlei +ADVOWARE_DATABASE=your_database +# ... (see config.py for all required vars) +``` + +### Dependencies + +```bash +pip install aiohttp redis python-dotenv requests +``` + +## Output + +The script produces: + +### 1. Console Output + +``` +================================================================================ +BETEILIGTE STRUCTURE COMPARISON TOOL +================================================================================ + +EspoCRM Entity ID: 64a3f2b8c9e1234567890abc + +Environment Check: +---------------------------------------- +ESPOCRM_API_BASE_URL: https://crm.bitbylaw.com/api/v1 +ESPOCRM_API_KEY: ✓ Set +ADVOWARE_API_BASE_URL: https://www2.advo-net.net:90/ +ADVOWARE_API_KEY: ✓ Set + +================================================================================ +ESPOCRM - Fetching Beteiligter +================================================================================ + +Trying entity type: Beteiligte + +✓ Success! Found in Beteiligte + +Entity Structure: +-------------------------------------------------------------------------------- +{ + "id": "64a3f2b8c9e1234567890abc", + "name": "Max Mustermann", + "firstName": "Max", + "lastName": "Mustermann", + "email": "max@example.com", + "phone": "+49123456789", + ... +} + +================================================================================ +ADVOWARE - Fetching Beteiligter +================================================================================ + +Searching by name: Max Mustermann + Trying endpoint: /contacts + +✓ Found 2 results + +Search Results: +-------------------------------------------------------------------------------- +[ + { + "id": 12345, + "full_name": "Max Mustermann", + "email": "max@example.com", + ... + } +] + +================================================================================ +STRUCTURE COMPARISON +================================================================================ + +EspoCRM Fields (25): +---------------------------------------- + id (str) + name (str) + firstName (str) + lastName (str) + email (str) + phone (str) + ... + +Advoware Fields (30): +---------------------------------------- + id (int) + full_name (str) + email (str) + phone_number (str) + ... + +Common Fields (5): +---------------------------------------- + ✓ id + ✓ email + ✗ phone + EspoCRM: +49123456789 + Advoware: 0123456789 + +EspoCRM Only (20): +---------------------------------------- + firstName + lastName + ... + +Advoware Only (25): +---------------------------------------- + full_name + phone_number + ... + +Potential Field Mappings: +---------------------------------------- + firstName → first_name + lastName → last_name + email → email + phone → phone_number + ... + +================================================================================ +Comparison saved to: /opt/motia-app/bitbylaw/scripts/beteiligte_comparison_result.json +================================================================================ +``` + +### 2. JSON Output File + +Saved to `bitbylaw/scripts/beteiligte_comparison_result.json`: + +```json +{ + "espocrm_data": { + "id": "64a3f2b8c9e1234567890abc", + "name": "Max Mustermann", + ... + }, + "advoware_data": { + "id": 12345, + "full_name": "Max Mustermann", + ... + }, + "comparison": { + "espo_fields": ["id", "name", "firstName", ...], + "advo_fields": ["id", "full_name", "email", ...], + "common": ["id", "email"], + "espo_only": ["firstName", "lastName", ...], + "advo_only": ["full_name", "phone_number", ...], + "suggested_mappings": [ + ["firstName", "first_name"], + ["lastName", "last_name"], + ... + ] + } +} +``` + +## How It Works + +### 1. EspoCRM Fetch + +The script tries multiple entity types to find the data: +- `Beteiligte` (custom VMH entity) +- `Contact` (standard) +- `Account` (standard) +- `Lead` (standard) + +### 2. Advoware Fetch + +**By ID (if provided):** +- Tries: `/contacts/{id}`, `/parties/{id}`, `/clients/{id}` + +**By Name (if EspoCRM data available):** +- Searches: `/contacts?search=...`, `/parties?search=...`, `/clients?search=...` + +### 3. Comparison + +- Lists all fields from both systems +- Identifies common fields (same name) +- Shows values for common fields +- Suggests mappings based on naming patterns +- Exports full comparison to JSON + +## Troubleshooting + +### "ESPOCRM_API_KEY not set" + +```bash +# Check if .env exists and has the key +cat .env | grep ESPOCRM_MARVIN_API_KEY + +# Or set it manually +export ESPOCRM_MARVIN_API_KEY=your_key_here +``` + +### "Authentication failed - check API key" + +1. Verify API key in EspoCRM admin panel +2. Check API User permissions +3. Ensure API User has access to entity type + +### "Entity not found" + +- Check if entity ID is correct +- Verify entity type exists in EspoCRM +- Check API User permissions for that entity + +### "Advoware token error" + +- Verify all Advoware credentials in `.env` +- Check HMAC signature generation +- Ensure API key is base64 encoded +- Test token generation separately + +## Next Steps + +After running this script: + +1. **Review JSON output** - Check `beteiligte_comparison_result.json` +2. **Define mappings** - Create mapping table based on suggestions +3. **Implement mapper** - Create transformation functions +4. **Test sync** - Use mappings in sync event step + +Example mapping implementation: + +```python +def map_espocrm_to_advoware(espo_entity: dict) -> dict: + """Transform EspoCRM Beteiligter to Advoware format""" + return { + 'first_name': espo_entity.get('firstName'), + 'last_name': espo_entity.get('lastName'), + 'email': espo_entity.get('email'), + 'phone_number': espo_entity.get('phone'), + # Add more mappings based on comparison... + } +``` + +## Related Files + +- [services/espocrm.py](../services/espocrm.py) - EspoCRM API client +- [services/advoware.py](../services/advoware.py) - Advoware API client +- [services/ESPOCRM_SERVICE.md](../services/ESPOCRM_SERVICE.md) - EspoCRM docs +- [config.py](../config.py) - Configuration diff --git a/bitbylaw/scripts/beteiligte_comparison_result.json b/bitbylaw/scripts/beteiligte_comparison_result.json new file mode 100644 index 00000000..db1f3e0e --- /dev/null +++ b/bitbylaw/scripts/beteiligte_comparison_result.json @@ -0,0 +1,399 @@ +{ + "espocrm_data": { + "id": "68e4af00172be7924", + "name": "dasdas dasdasdas dasdasdas", + "deleted": false, + "salutationName": null, + "rechtsform": "GmbH", + "firmenname": "Filli llu GmbH", + "firstName": "dasdasdas", + "lastName": "dasdas", + "dateOfBirth": null, + "description": null, + "emailAddress": "meier@meier.de", + "phoneNumber": null, + "createdAt": "2025-10-07 06:11:12", + "modifiedAt": "2026-01-23 21:58:41", + "betnr": 1234, + "advowareLastSync": null, + "syncStatus": "clean", + "handelsregisterNummer": "12244546", + "handelsregisterArt": "HRB", + "disgTyp": "Unbekannt", + "middleName": "dasdasdas", + "emailAddressIsOptedOut": false, + "emailAddressIsInvalid": false, + "phoneNumberIsOptedOut": null, + "phoneNumberIsInvalid": null, + "streamUpdatedAt": null, + "emailAddressData": [ + { + "emailAddress": "meier@meier.de", + "lower": "meier@meier.de", + "primary": true, + "optOut": false, + "invalid": false + }, + { + "emailAddress": "a@r028tuj08wefj0w8efjw0d.de", + "lower": "a@r028tuj08wefj0w8efjw0d.de", + "primary": false, + "optOut": false, + "invalid": false + } + ], + "phoneNumberData": [], + "createdById": "68d65929f18c2afef", + "createdByName": "Admin", + "modifiedById": "68d65929f18c2afef", + "modifiedByName": "Admin", + "assignedUserId": null, + "assignedUserName": null, + "teamsIds": [], + "teamsNames": {}, + "adressensIds": [], + "adressensNames": {}, + "calls1Ids": [], + "calls1Names": {}, + "bankverbindungensIds": [], + "bankverbindungensNames": {}, + "isFollowed": false, + "followersIds": [], + "followersNames": {} + }, + "advoware_data": { + "betNr": 104860, + "kommunikation": [ + { + "rowId": "FBABAAAANJFGABAAGJDOAEAPAAAAAPGFPDAFAAAA", + "id": 88002, + "betNr": 104860, + "kommArt": 0, + "tlf": "0511/12345-60", + "bemerkung": null, + "kommKz": 0, + "online": false + }, + { + "rowId": "FBABAAAABBLIABAAGIDOAEAPAAAAAPHBEOAEAAAA", + "id": 114914, + "betNr": 104860, + "kommArt": 0, + "tlf": "kanzlei@ralup.de", + "bemerkung": null, + "kommKz": 0, + "online": true + } + ], + "kontaktpersonen": [], + "beteiligungen": [ + { + "rowId": "LAADAAAAAHMDABAAGAAEIPBAAAAADGKEMPAFAAAA", + "beteiligtenArt": "Sachverständiger", + "akte": { + "rowId": "", + "nr": 2020001684, + "az": "1684/20", + "rubrum": "Siggel / Siggel", + "referat": "SON", + "wegen": "Bruderzwist II", + "ablage": 1, + "abgelegt": null + } + }, + { + "rowId": "LAADAAAAPGKFABAAGAAEIPBAAAAADGJOMBABAAAA", + "beteiligtenArt": "Sachverständiger", + "akte": { + "rowId": "", + "nr": 2020000203, + "az": "203/20", + "rubrum": "Siggel / Siggel", + "referat": "SON", + "wegen": "Bruderzwist", + "ablage": 1, + "abgelegt": null + } + }, + { + "rowId": "LAADAAAAPJAGACAAGAAEIPBAAAAADGLDFGADAAAA", + "beteiligtenArt": "Mandant", + "akte": { + "rowId": "", + "nr": 2019001145, + "az": "1145/19", + "rubrum": "Siggel / Siggel LALA", + "referat": "VMH", + "wegen": null, + "ablage": 0, + "abgelegt": null + } + } + ], + "adressen": [ + { + "rowId": "KOADAAAAALNFAAAAFPAEIPBAAAAADGGPGAAJAAAA", + "id": 0, + "beteiligterId": 104860, + "reihenfolgeIndex": 1, + "strasse": "Musterstraße 12", + "plz": "12345", + "ort": "Musterort", + "land": "D", + "postfach": null, + "postfachPLZ": null, + "anschrift": "Frau\r\nAngela Mustermanns\r\nVorzimmer\r\nMusterstraße 12\r\n12345 Musterort", + "standardAnschrift": false, + "bemerkung": null, + "gueltigVon": null, + "gueltigBis": null + } + ], + "bankkverbindungen": [ + { + "rowId": "EPABAAAAHBNFAAAAFPNBCGAAAAAAAPDIJDAJAAAA", + "id": 54665, + "bank": null, + "ktoNr": null, + "blz": null, + "iban": null, + "bic": null, + "kontoinhaber": null, + "mandatsreferenz": null, + "mandatVom": null + } + ], + "rowId": "EMABAAAAFBNFAAAAFOAEIPBAAAAAAOMNKPAHAAAA", + "id": 104860, + "anschrift": "Frau\r\nAngela Mustermanns\r\nVorzimmer\r\nMusterstraße 12\r\n12345 Musterort", + "strasse": "Musterstraße 12", + "plz": "12345", + "ort": "Musterort", + "email": null, + "emailGesch": "kanzlei@ralup.de", + "mobil": null, + "internet": null, + "telGesch": "0511/12345-60", + "telPrivat": null, + "faxGesch": null, + "faxPrivat": null, + "autotelefon": null, + "sonstige": null, + "ePost": null, + "bea": null, + "art": null, + "vorname": "Angela", + "name": "Mustermanns", + "kurzname": null, + "geburtsname": null, + "familienstand": null, + "titel": null, + "anrede": "Frau", + "bAnrede": "Sehr geehrte Frau Mustermanns,", + "geburtsdatum": null, + "sterbedatum": null, + "zusatz": "Vorzimmer", + "rechtsform": "Frau", + "geaendertAm": null, + "geaendertVon": null, + "angelegtAm": null, + "angelegtVon": null, + "handelsRegisterNummer": null, + "registergericht": null + }, + "comparison": { + "espo_fields": [ + "emailAddressIsInvalid", + "followersNames", + "id", + "handelsregisterNummer", + "teamsNames", + "assignedUserName", + "modifiedAt", + "modifiedByName", + "betnr", + "middleName", + "disgTyp", + "bankverbindungensNames", + "phoneNumberIsOptedOut", + "adressensIds", + "emailAddressData", + "deleted", + "teamsIds", + "phoneNumber", + "isFollowed", + "advowareLastSync", + "createdById", + "createdAt", + "calls1Ids", + "handelsregisterArt", + "name", + "phoneNumberIsInvalid", + "rechtsform", + "emailAddress", + "emailAddressIsOptedOut", + "firmenname", + "description", + "adressensNames", + "createdByName", + "lastName", + "assignedUserId", + "salutationName", + "bankverbindungensIds", + "phoneNumberData", + "dateOfBirth", + "modifiedById", + "firstName", + "followersIds", + "streamUpdatedAt", + "syncStatus", + "calls1Names" + ], + "advo_fields": [ + "kontaktpersonen", + "rowId", + "id", + "angelegtVon", + "zusatz", + "bAnrede", + "faxGesch", + "bankkverbindungen", + "geburtsname", + "plz", + "adressen", + "kurzname", + "telPrivat", + "anrede", + "sonstige", + "email", + "titel", + "sterbedatum", + "faxPrivat", + "autotelefon", + "name", + "kommunikation", + "rechtsform", + "art", + "geaendertAm", + "anschrift", + "beteiligungen", + "bea", + "handelsRegisterNummer", + "registergericht", + "internet", + "ort", + "geburtsdatum", + "angelegtAm", + "mobil", + "emailGesch", + "ePost", + "strasse", + "vorname", + "familienstand", + "betNr", + "geaendertVon", + "telGesch" + ], + "common": [ + "name", + "id", + "rechtsform" + ], + "espo_only": [ + "emailAddressIsInvalid", + "followersNames", + "handelsregisterNummer", + "teamsNames", + "assignedUserName", + "modifiedAt", + "modifiedByName", + "betnr", + "middleName", + "disgTyp", + "bankverbindungensNames", + "phoneNumberIsOptedOut", + "adressensIds", + "emailAddressData", + "deleted", + "teamsIds", + "phoneNumber", + "isFollowed", + "advowareLastSync", + "createdById", + "createdAt", + "calls1Ids", + "handelsregisterArt", + "phoneNumberIsInvalid", + "emailAddress", + "emailAddressIsOptedOut", + "firmenname", + "description", + "adressensNames", + "createdByName", + "lastName", + "assignedUserId", + "salutationName", + "bankverbindungensIds", + "phoneNumberData", + "dateOfBirth", + "modifiedById", + "firstName", + "followersIds", + "streamUpdatedAt", + "syncStatus", + "calls1Names" + ], + "advo_only": [ + "kontaktpersonen", + "rowId", + "angelegtVon", + "zusatz", + "bAnrede", + "faxGesch", + "bankkverbindungen", + "geburtsname", + "plz", + "adressen", + "kurzname", + "telPrivat", + "anrede", + "sonstige", + "email", + "titel", + "sterbedatum", + "autotelefon", + "faxPrivat", + "kommunikation", + "art", + "geaendertAm", + "anschrift", + "beteiligungen", + "bea", + "handelsRegisterNummer", + "registergericht", + "internet", + "ort", + "geburtsdatum", + "angelegtAm", + "mobil", + "emailGesch", + "ePost", + "strasse", + "vorname", + "familienstand", + "betNr", + "geaendertVon", + "telGesch" + ], + "suggested_mappings": [ + [ + "name", + "name" + ], + [ + "emailAddress", + "email" + ] + ] + } +} \ No newline at end of file diff --git a/bitbylaw/scripts/compare_beteiligte.py b/bitbylaw/scripts/compare_beteiligte.py new file mode 100755 index 00000000..181cef41 --- /dev/null +++ b/bitbylaw/scripts/compare_beteiligte.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +""" +Helper-Script zum Vergleichen der Beteiligten-Strukturen zwischen Advoware und EspoCRM. + +Usage: + python scripts/compare_beteiligte.py [advoware_id] + +Examples: + # Vergleiche EspoCRM Beteiligten (automatische Suche in Advoware) + python scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc + + # Vergleiche mit spezifischer Advoware ID + python scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc 12345 +""" + +import sys +import asyncio +import json +import os +from pathlib import Path + +# Add bitbylaw directory to path for imports +bitbylaw_dir = Path(__file__).parent.parent +sys.path.insert(0, str(bitbylaw_dir)) + +from services.espocrm import EspoCRMAPI +from services.advoware import AdvowareAPI +from config import Config + + +class SimpleContext: + """Simple context for logging""" + class Logger: + def info(self, msg): + print(f"[INFO] {msg}") + + def error(self, msg): + print(f"[ERROR] {msg}") + + def debug(self, msg): + print(f"[DEBUG] {msg}") + + def warning(self, msg): + print(f"[WARNING] {msg}") + + def __init__(self): + self.logger = self.Logger() + + +async def fetch_from_espocrm(entity_id: str): + """Fetch Beteiligter from EspoCRM""" + print("\n" + "="*80) + print("ESPOCRM - Fetching Beteiligter") + print("="*80) + + context = SimpleContext() + espo = EspoCRMAPI(context=context) + + try: + # Try different entity types that might contain Beteiligte + entity_types = ['CBeteiligte', 'Beteiligte', 'Contact', 'Account', 'Lead', 'CVmhErstgespraech', 'CVmhBeteiligte'] + + for entity_type in entity_types: + try: + print(f"\nTrying entity type: {entity_type}") + result = await espo.get_entity(entity_type, entity_id) + + print(f"\n✓ Success! Found in {entity_type}") + print(f"\nEntity Structure:") + print("-" * 80) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + return result + + except Exception as e: + print(f" ✗ Not found in {entity_type}: {e}") + continue + + print("\n✗ Entity not found in any known entity type") + return None + + except Exception as e: + print(f"\n✗ Error fetching from EspoCRM: {e}") + return None + + +async def fetch_from_advoware(advoware_id: str = None, search_name: str = None): + """Fetch Beteiligter from Advoware""" + print("\n" + "="*80) + print("ADVOWARE - Fetching Beteiligter") + print("="*80) + + context = SimpleContext() + advo = AdvowareAPI(context=context) + + try: + # Try to fetch by ID if provided + if advoware_id: + print(f"\nFetching by ID: {advoware_id}") + # Try correct Advoware endpoint + endpoints = [ + f'/api/v1/advonet/Beteiligte/{advoware_id}', + ] + + for endpoint in endpoints: + try: + print(f" Trying endpoint: {endpoint}") + result = await advo.api_call(endpoint, method='GET') + + if result: + # Advoware gibt oft Listen zurück, nehme erstes Element + if isinstance(result, list) and len(result) > 0: + result = result[0] + + print(f"\n✓ Success! Found at {endpoint}") + print(f"\nEntity Structure:") + print("-" * 80) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return result + + except Exception as e: + print(f" ✗ Not found at {endpoint}: {e}") + continue + + # Try to search by name if EspoCRM data available + if search_name: + print(f"\nSearching by name: {search_name}") + search_endpoints = [ + '/api/v1/advonet/Beteiligte', + ] + + for endpoint in search_endpoints: + try: + print(f" Trying endpoint: {endpoint}") + result = await advo.api_call( + endpoint, + method='GET', + params={'search': search_name, 'limit': 5} + ) + + if result and (isinstance(result, list) and len(result) > 0 or + isinstance(result, dict) and result.get('data')): + print(f"\n✓ Found {len(result) if isinstance(result, list) else len(result.get('data', []))} results") + print(f"\nSearch Results:") + print("-" * 80) + print(json.dumps(result, indent=2, ensure_ascii=False)) + return result + + except Exception as e: + print(f" ✗ Search failed at {endpoint}: {e}") + continue + + print("\n✗ Entity not found in Advoware") + return None + + except Exception as e: + print(f"\n✗ Error fetching from Advoware: {e}") + import traceback + traceback.print_exc() + return None + + +async def compare_structures(espo_data: dict, advo_data: dict): + """Compare field structures between EspoCRM and Advoware""" + print("\n" + "="*80) + print("STRUCTURE COMPARISON") + print("="*80) + + if not espo_data or not advo_data: + print("\n⚠ Cannot compare - missing data from one or both systems") + return + + # Extract fields + espo_fields = set(espo_data.keys()) if isinstance(espo_data, dict) else set() + + # Handle Advoware data structure (might be nested) + if isinstance(advo_data, dict): + if 'data' in advo_data: + advo_data = advo_data['data'] + if isinstance(advo_data, list) and len(advo_data) > 0: + advo_data = advo_data[0] + + advo_fields = set(advo_data.keys()) if isinstance(advo_data, dict) else set() + + print(f"\nEspoCRM Fields ({len(espo_fields)}):") + print("-" * 40) + for field in sorted(espo_fields): + value = espo_data.get(field) + value_type = type(value).__name__ + print(f" {field:<30} ({value_type})") + + print(f"\nAdvoware Fields ({len(advo_fields)}):") + print("-" * 40) + for field in sorted(advo_fields): + value = advo_data.get(field) + value_type = type(value).__name__ + print(f" {field:<30} ({value_type})") + + # Find common fields (potential mappings) + common = espo_fields & advo_fields + espo_only = espo_fields - advo_fields + advo_only = advo_fields - espo_fields + + print(f"\nCommon Fields ({len(common)}):") + print("-" * 40) + for field in sorted(common): + espo_val = espo_data.get(field) + advo_val = advo_data.get(field) + match = "✓" if espo_val == advo_val else "✗" + print(f" {match} {field}") + if espo_val != advo_val: + print(f" EspoCRM: {espo_val}") + print(f" Advoware: {advo_val}") + + print(f"\nEspoCRM Only ({len(espo_only)}):") + print("-" * 40) + for field in sorted(espo_only): + print(f" {field}") + + print(f"\nAdvoware Only ({len(advo_only)}):") + print("-" * 40) + for field in sorted(advo_only): + print(f" {field}") + + # Suggest potential mappings based on field names + print(f"\nPotential Field Mappings:") + print("-" * 40) + + mapping_suggestions = [] + + # Common name patterns + name_patterns = [ + ('name', 'name'), + ('firstName', 'first_name'), + ('lastName', 'last_name'), + ('email', 'email'), + ('emailAddress', 'email'), + ('phone', 'phone'), + ('phoneNumber', 'phone_number'), + ('address', 'address'), + ('street', 'street'), + ('city', 'city'), + ('postalCode', 'postal_code'), + ('zipCode', 'postal_code'), + ('country', 'country'), + ] + + for espo_field, advo_field in name_patterns: + if espo_field in espo_fields and advo_field in advo_fields: + mapping_suggestions.append((espo_field, advo_field)) + print(f" {espo_field:<30} → {advo_field}") + + return { + 'espo_fields': list(espo_fields), + 'advo_fields': list(advo_fields), + 'common': list(common), + 'espo_only': list(espo_only), + 'advo_only': list(advo_only), + 'suggested_mappings': mapping_suggestions + } + + +async def main(): + """Main function""" + if len(sys.argv) < 2: + print(__doc__) + sys.exit(1) + + espocrm_id = sys.argv[1] + advoware_id = sys.argv[2] if len(sys.argv) > 2 else None + + print("\n" + "="*80) + print("BETEILIGTE STRUCTURE COMPARISON TOOL") + print("="*80) + print(f"\nEspoCRM Entity ID: {espocrm_id}") + if advoware_id: + print(f"Advoware ID: {advoware_id}") + + # Check environment variables + print("\nEnvironment Check:") + print("-" * 40) + print(f"ESPOCRM_API_BASE_URL: {Config.ESPOCRM_API_BASE_URL}") + print(f"ESPOCRM_API_KEY: {'✓ Set' if Config.ESPOCRM_API_KEY else '✗ Missing'}") + print(f"ADVOWARE_API_BASE_URL: {Config.ADVOWARE_API_BASE_URL}") + print(f"ADVOWARE_API_KEY: {'✓ Set' if Config.ADVOWARE_API_KEY else '✗ Missing'}") + + # Fetch from EspoCRM + espo_data = await fetch_from_espocrm(espocrm_id) + + # Extract name for Advoware search + search_name = None + if espo_data: + search_name = ( + espo_data.get('name') or + f"{espo_data.get('firstName', '')} {espo_data.get('lastName', '')}".strip() or + None + ) + + # Fetch from Advoware + advo_data = await fetch_from_advoware(advoware_id, search_name) + + # Compare structures + if espo_data or advo_data: + comparison = await compare_structures(espo_data, advo_data) + + # Save comparison to file + output_file = Path(__file__).parent / 'beteiligte_comparison_result.json' + with open(output_file, 'w', encoding='utf-8') as f: + json.dump({ + 'espocrm_data': espo_data, + 'advoware_data': advo_data, + 'comparison': comparison + }, f, indent=2, ensure_ascii=False) + + print(f"\n\n{'='*80}") + print(f"Comparison saved to: {output_file}") + print(f"{'='*80}\n") + else: + print("\n⚠ No data available for comparison") + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/bitbylaw/services/ESPOCRM_SERVICE.md b/bitbylaw/services/ESPOCRM_SERVICE.md new file mode 100644 index 00000000..69be758e --- /dev/null +++ b/bitbylaw/services/ESPOCRM_SERVICE.md @@ -0,0 +1,403 @@ +# EspoCRM API Service + +## Overview + +Python client for EspoCRM REST API integration. Provides type-safe, async operations for managing entities in EspoCRM. + +## Features + +- ✅ API Key authentication +- ✅ Async/await support (aiohttp) +- ✅ Full CRUD operations +- ✅ Entity search and filtering +- ✅ Error handling with custom exceptions +- ✅ Optional Redis integration for caching +- ✅ Logging via Motia context + +## Installation + +```python +from services.espocrm import EspoCRMAPI + +# Initialize with optional context for logging +espo = EspoCRMAPI(context=context) +``` + +## Configuration + +Add to `.env` or environment: + +```bash +# EspoCRM API Configuration +ESPOCRM_API_BASE_URL=https://crm.bitbylaw.com/api/v1 +ESPOCRM_MARVIN_API_KEY=your_api_key_here +ESPOCRM_API_TIMEOUT_SECONDS=30 +``` + +Required in `config.py`: + +```python +class Config: + ESPOCRM_API_BASE_URL = os.getenv('ESPOCRM_API_BASE_URL', 'https://crm.bitbylaw.com/api/v1') + ESPOCRM_API_KEY = os.getenv('ESPOCRM_MARVIN_API_KEY', '') + ESPOCRM_API_TIMEOUT_SECONDS = int(os.getenv('ESPOCRM_API_TIMEOUT_SECONDS', '30')) +``` + +## API Methods + +### Get Single Entity + +```python +async def get_entity(entity_type: str, entity_id: str) -> Dict[str, Any] +``` + +**Usage:** +```python +# Get Beteiligter by ID +result = await espo.get_entity('Beteiligte', '64a3f2b8c9e1234567890abc') +print(result['name']) +``` + +### List Entities + +```python +async def list_entities( + entity_type: str, + where: Optional[List[Dict]] = None, + select: Optional[str] = None, + order_by: Optional[str] = None, + offset: int = 0, + max_size: int = 50 +) -> Dict[str, Any] +``` + +**Usage:** +```python +# List all Beteiligte with status "Active" +result = await espo.list_entities( + 'Beteiligte', + where=[{ + 'type': 'equals', + 'attribute': 'status', + 'value': 'Active' + }], + select='id,name,email', + max_size=100 +) + +for entity in result['list']: + print(entity['name']) +print(f"Total: {result['total']}") +``` + +**Complex Filters:** +```python +# OR condition +where=[{ + 'type': 'or', + 'value': [ + {'type': 'equals', 'attribute': 'status', 'value': 'Zurückgestellt'}, + {'type': 'equals', 'attribute': 'status', 'value': 'Warte auf neuen Anruf'} + ] +}] + +# AND condition +where=[ + {'type': 'equals', 'attribute': 'status', 'value': 'Active'}, + {'type': 'greaterThan', 'attribute': 'createdAt', 'value': '2026-01-01'} +] +``` + +### Create Entity + +```python +async def create_entity(entity_type: str, data: Dict[str, Any]) -> Dict[str, Any] +``` + +**Usage:** +```python +# Create new Beteiligter +result = await espo.create_entity('Beteiligte', { + 'name': 'Max Mustermann', + 'email': 'max@example.com', + 'phone': '+49123456789', + 'status': 'New' +}) +print(f"Created with ID: {result['id']}") +``` + +### Update Entity + +```python +async def update_entity( + entity_type: str, + entity_id: str, + data: Dict[str, Any] +) -> Dict[str, Any] +``` + +**Usage:** +```python +# Update Beteiligter status +result = await espo.update_entity( + 'Beteiligte', + '64a3f2b8c9e1234567890abc', + {'status': 'Converted'} +) +``` + +### Delete Entity + +```python +async def delete_entity(entity_type: str, entity_id: str) -> bool +``` + +**Usage:** +```python +# Delete Beteiligter +success = await espo.delete_entity('Beteiligte', '64a3f2b8c9e1234567890abc') +``` + +### Search Entities + +```python +async def search_entities( + entity_type: str, + query: str, + fields: Optional[List[str]] = None +) -> List[Dict[str, Any]] +``` + +**Usage:** +```python +# Full-text search +results = await espo.search_entities('Beteiligte', 'Mustermann') +for entity in results: + print(entity['name']) +``` + +## Common Entity Types + +Based on EspoCRM standard and VMH customization: + +- `Beteiligte` - Custom entity for VMH participants +- `CVmhErstgespraech` - Custom entity for VMH initial consultations +- `Contact` - Standard contacts +- `Account` - Companies/Organizations +- `Lead` - Sales leads +- `Opportunity` - Sales opportunities +- `Case` - Support cases +- `Meeting` - Calendar meetings +- `Call` - Phone calls +- `Email` - Email records + +## Error Handling + +```python +from services.espocrm import EspoCRMError, EspoCRMAuthError + +try: + result = await espo.get_entity('Beteiligte', entity_id) +except EspoCRMAuthError as e: + # Invalid API key + context.logger.error(f"Authentication failed: {e}") +except EspoCRMError as e: + # General API error (404, 403, etc.) + context.logger.error(f"API error: {e}") +``` + +## Authentication + +EspoCRM uses **API Key authentication** via `X-Api-Key` header. + +**Create API Key in EspoCRM:** +1. Login as admin +2. Go to Administration → API Users +3. Create new API User +4. Copy API Key +5. Set permissions for API User + +**Headers sent automatically:** +``` +X-Api-Key: your_api_key_here +Content-Type: application/json +Accept: application/json +``` + +## Integration Examples + +### In Motia Step + +```python +from services.espocrm import EspoCRMAPI + +config = { + 'type': 'event', + 'name': 'Sync Beteiligter to Advoware', + 'subscribes': ['vmh.beteiligte.create'] +} + +async def handler(event, context): + entity_id = event['data']['entity_id'] + + # Fetch from EspoCRM + espo = EspoCRMAPI(context=context) + beteiligter = await espo.get_entity('Beteiligte', entity_id) + + context.logger.info(f"Processing: {beteiligter['name']}") + + # Transform and sync to Advoware... + # ... +``` + +### In Cron Step + +```python +from services.espocrm import EspoCRMAPI +from datetime import datetime, timedelta + +config = { + 'type': 'cron', + 'cron': '*/5 * * * *', + 'name': 'Check Expired Callbacks' +} + +async def handler(input, context): + espo = EspoCRMAPI(context=context) + + # Find expired callbacks + now = datetime.utcnow().isoformat() + 'Z' + + result = await espo.list_entities( + 'CVmhErstgespraech', + where=[ + {'type': 'lessThan', 'attribute': 'nchsterAnruf', 'value': now}, + {'type': 'equals', 'attribute': 'status', 'value': 'Warte auf neuen Anruf'} + ] + ) + + # Update status for expired entries + for entry in result['list']: + await espo.update_entity( + 'CVmhErstgespraech', + entry['id'], + {'status': 'Neu'} + ) + context.logger.info(f"Reset status for {entry['id']}") +``` + +## Helper Script: Compare Structures + +Compare entity structures between EspoCRM and Advoware: + +```bash +# Compare by EspoCRM ID (auto-search in Advoware) +python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc + +# Compare with specific Advoware ID +python bitbylaw/scripts/compare_beteiligte.py 64a3f2b8c9e1234567890abc 12345 +``` + +**Output:** +- Entity data from both systems +- Field structure comparison +- Suggested field mappings +- JSON output saved to `scripts/beteiligte_comparison_result.json` + +## Performance + +### Timeout + +Default: 30 seconds (configurable via `ESPOCRM_API_TIMEOUT_SECONDS`) + +```python +# Custom timeout for specific call +result = await espo.api_call('/Beteiligte', timeout_seconds=60) +``` + +### Pagination + +```python +# Fetch in pages +offset = 0 +max_size = 50 + +while True: + result = await espo.list_entities( + 'Beteiligte', + offset=offset, + max_size=max_size + ) + + entities = result['list'] + if not entities: + break + + # Process entities... + + offset += len(entities) + + if len(entities) < max_size: + break # Last page +``` + +### Rate Limiting + +Optional Redis-based rate limiting can be implemented: + +```python +# Check rate limit before API call +rate_limit_key = f'espocrm:rate_limit:{entity_type}' +if espo.redis_client: + count = espo.redis_client.incr(rate_limit_key) + espo.redis_client.expire(rate_limit_key, 60) # 1 minute window + + if count > 100: # Max 100 requests per minute + raise Exception("Rate limit exceeded") +``` + +## Testing + +```python +import pytest +from services.espocrm import EspoCRMAPI + +@pytest.mark.asyncio +async def test_get_entity(): + espo = EspoCRMAPI() + + # Mock or use test entity ID + result = await espo.get_entity('Contact', 'test-id-123') + + assert 'id' in result + assert result['id'] == 'test-id-123' +``` + +## Logging + +All operations are logged via context.logger: + +``` +[INFO] [EspoCRM] EspoCRM API initialized with base URL: https://crm.bitbylaw.com/api/v1 +[DEBUG] [EspoCRM] API call: GET https://crm.bitbylaw.com/api/v1/Beteiligte/123 +[DEBUG] [EspoCRM] Response status: 200 +[INFO] [EspoCRM] Getting Beteiligte with ID: 123 +``` + +## Related Files + +- [services/espocrm.py](./espocrm.py) - Implementation +- [scripts/compare_beteiligte.py](../scripts/compare_beteiligte.py) - Comparison tool +- [steps/crm-bbl-vmh-reset-nextcall_step.py](../../steps/crm-bbl-vmh-reset-nextcall_step.py) - Example usage +- [config.py](../config.py) - Configuration + +## EspoCRM API Documentation + +Official docs: https://docs.espocrm.com/development/api/ + +**Key Concepts:** +- RESTful API with JSON +- Entity-based operations +- Filter operators: `equals`, `notEquals`, `greaterThan`, `lessThan`, `like`, `contains`, `in`, `isNull`, `isNotNull` +- Boolean operators: `and` (default), `or` +- Metadata API: `/Metadata` (for entity definitions) diff --git a/bitbylaw/services/advoware.py b/bitbylaw/services/advoware.py index 1b880a50..ab231cbe 100644 --- a/bitbylaw/services/advoware.py +++ b/bitbylaw/services/advoware.py @@ -122,7 +122,9 @@ class AdvowareAPI: params: Optional[Dict] = None, json_data: Optional[Dict] = None, files: Optional[Any] = None, data: Optional[Any] = None, timeout_seconds: Optional[int] = None) -> Any: - url = self.API_BASE_URL + endpoint + # Bereinige doppelte Slashes + endpoint = endpoint.lstrip('/') + url = self.API_BASE_URL.rstrip('/') + '/' + endpoint effective_timeout = aiohttp.ClientTimeout(total=timeout_seconds or Config.ADVOWARE_API_TIMEOUT_SECONDS) token = self.get_access_token() # Sync call effective_headers = headers.copy() if headers else {} diff --git a/bitbylaw/services/espocrm.py b/bitbylaw/services/espocrm.py new file mode 100644 index 00000000..f85d4dbf --- /dev/null +++ b/bitbylaw/services/espocrm.py @@ -0,0 +1,276 @@ +import aiohttp +import asyncio +import logging +import redis +from typing import Optional, Dict, Any, List +from config import Config + +logger = logging.getLogger(__name__) + +class EspoCRMError(Exception): + """Base exception for EspoCRM API errors""" + pass + +class EspoCRMAuthError(EspoCRMError): + """Authentication error""" + pass + +class EspoCRMAPI: + """ + EspoCRM API Client for bitbylaw integration. + + Supports: + - API Key authentication (X-Api-Key header) + - Standard REST operations (GET, POST, PUT, DELETE) + - Entity management (Beteiligte, CVmhErstgespraech, etc.) + """ + + def __init__(self, context=None): + self.context = context + self._log("EspoCRMAPI __init__ started", level='debug') + + # Configuration + self.api_base_url = Config.ESPOCRM_API_BASE_URL + self.api_key = Config.ESPOCRM_API_KEY + + if not self.api_key: + raise EspoCRMAuthError("ESPOCRM_MARVIN_API_KEY not configured in environment") + + self._log(f"EspoCRM API initialized with base URL: {self.api_base_url}") + + # Optional Redis for caching/rate limiting + try: + self.redis_client = redis.Redis( + host=Config.REDIS_HOST, + port=int(Config.REDIS_PORT), + db=int(Config.REDIS_DB_ADVOWARE_CACHE), + socket_timeout=Config.REDIS_TIMEOUT_SECONDS, + socket_connect_timeout=Config.REDIS_TIMEOUT_SECONDS, + decode_responses=True + ) + self.redis_client.ping() + self._log("Connected to Redis for EspoCRM operations") + except Exception as e: + self._log(f"Could not connect to Redis: {e}. Continuing without caching.", level='warning') + self.redis_client = None + + def _log(self, message: str, level: str = 'info'): + """Log message via context.logger if available, otherwise use module logger""" + log_func = getattr(logger, level, logger.info) + if self.context and hasattr(self.context, 'logger'): + ctx_log_func = getattr(self.context.logger, level, self.context.logger.info) + ctx_log_func(f"[EspoCRM] {message}") + else: + log_func(f"[EspoCRM] {message}") + + def _get_headers(self) -> Dict[str, str]: + """Generate request headers with API key""" + return { + 'X-Api-Key': self.api_key, + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + async def api_call( + self, + endpoint: str, + method: str = 'GET', + params: Optional[Dict] = None, + json_data: Optional[Dict] = None, + timeout_seconds: Optional[int] = None + ) -> Any: + """ + Make an API call to EspoCRM. + + Args: + endpoint: API endpoint (e.g., '/Beteiligte/123' or '/CVmhErstgespraech') + method: HTTP method (GET, POST, PUT, DELETE) + params: Query parameters + json_data: JSON body for POST/PUT + timeout_seconds: Request timeout + + Returns: + Parsed JSON response or None + + Raises: + EspoCRMError: On API errors + """ + # Ensure endpoint starts with / + if not endpoint.startswith('/'): + endpoint = '/' + endpoint + + url = self.api_base_url.rstrip('/') + endpoint + headers = self._get_headers() + effective_timeout = aiohttp.ClientTimeout( + total=timeout_seconds or Config.ESPOCRM_API_TIMEOUT_SECONDS + ) + + self._log(f"API call: {method} {url}", level='debug') + if params: + self._log(f"Params: {params}", level='debug') + + async with aiohttp.ClientSession(timeout=effective_timeout) as session: + try: + async with session.request( + method, + url, + headers=headers, + params=params, + json=json_data + ) as response: + # Log response status + self._log(f"Response status: {response.status}", level='debug') + + # Handle errors + if response.status == 401: + raise EspoCRMAuthError("Authentication failed - check API key") + elif response.status == 403: + raise EspoCRMError("Access forbidden") + elif response.status == 404: + raise EspoCRMError(f"Resource not found: {endpoint}") + elif response.status >= 400: + error_text = await response.text() + raise EspoCRMError(f"API error {response.status}: {error_text}") + + # Parse response + if response.content_type == 'application/json': + result = await response.json() + self._log(f"Response received", level='debug') + return result + else: + # For DELETE or other non-JSON responses + return None + + except aiohttp.ClientError as e: + self._log(f"API call failed: {e}", level='error') + raise EspoCRMError(f"Request failed: {e}") from e + + async def get_entity(self, entity_type: str, entity_id: str) -> Dict[str, Any]: + """ + Get a single entity by ID. + + Args: + entity_type: Entity type (e.g., 'Beteiligte', 'CVmhErstgespraech') + entity_id: Entity ID + + Returns: + Entity data as dict + """ + self._log(f"Getting {entity_type} with ID: {entity_id}") + return await self.api_call(f"/{entity_type}/{entity_id}", method='GET') + + async def list_entities( + self, + entity_type: str, + where: Optional[List[Dict]] = None, + select: Optional[str] = None, + order_by: Optional[str] = None, + offset: int = 0, + max_size: int = 50 + ) -> Dict[str, Any]: + """ + List entities with filtering and pagination. + + Args: + entity_type: Entity type + where: Filter conditions (EspoCRM format) + select: Comma-separated field list + order_by: Sort field + offset: Pagination offset + max_size: Max results per page + + Returns: + Dict with 'list' and 'total' keys + """ + params = { + 'offset': offset, + 'maxSize': max_size + } + + if where: + params['where'] = where + if select: + params['select'] = select + if order_by: + params['orderBy'] = order_by + + self._log(f"Listing {entity_type} entities") + return await self.api_call(f"/{entity_type}", method='GET', params=params) + + async def create_entity( + self, + entity_type: str, + data: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Create a new entity. + + Args: + entity_type: Entity type + data: Entity data + + Returns: + Created entity with ID + """ + self._log(f"Creating {entity_type} entity") + return await self.api_call(f"/{entity_type}", method='POST', json_data=data) + + async def update_entity( + self, + entity_type: str, + entity_id: str, + data: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Update an existing entity. + + Args: + entity_type: Entity type + entity_id: Entity ID + data: Updated fields + + Returns: + Updated entity + """ + self._log(f"Updating {entity_type} with ID: {entity_id}") + return await self.api_call(f"/{entity_type}/{entity_id}", method='PUT', json_data=data) + + async def delete_entity(self, entity_type: str, entity_id: str) -> bool: + """ + Delete an entity. + + Args: + entity_type: Entity type + entity_id: Entity ID + + Returns: + True if successful + """ + self._log(f"Deleting {entity_type} with ID: {entity_id}") + await self.api_call(f"/{entity_type}/{entity_id}", method='DELETE') + return True + + async def search_entities( + self, + entity_type: str, + query: str, + fields: Optional[List[str]] = None + ) -> List[Dict[str, Any]]: + """ + Search entities by text query. + + Args: + entity_type: Entity type + query: Search query + fields: Fields to search in + + Returns: + List of matching entities + """ + where = [{ + 'type': 'textFilter', + 'value': query + }] + + result = await self.list_entities(entity_type, where=where) + return result.get('list', [])