Enhance EspoCRM API and Webhook Handling

- Improved logging for file uploads in EspoCRMAPI to include upload parameters and error details. - Updated cron job configurations for calendar sync and participant sync to trigger every 15 minutes on the first minute of the hour. - Enhanced document create, delete, and update webhook handlers to determine and log the entity type. - Refactored document sync event handler to include entity type in sync operations and logging. - Added a new test script for uploading preview images to EspoCRM and verifying the upload process. - Created a test script for document thumbnail generation, including document creation, file upload, webhook triggering, and preview verification.
2026-03-03 16:53:55 +00:00
parent 0e521f22f8
commit c45bfb7233
10 changed files with 738 additions and 84 deletions
--- a/services/document_sync_utils.py
+++ b/services/document_sync_utils.py
@@ -61,12 +61,13 @@ class DocumentSync:
        else:
            getattr(logger, level)(message)
    
-    async def acquire_sync_lock(self, entity_id: str) -> bool:
+    async def acquire_sync_lock(self, entity_id: str, entity_type: str = 'CDokumente') -> bool:
        """
        Atomic distributed lock via Redis + syncStatus update
        
        Args:
            entity_id: EspoCRM Document ID
+            entity_type: Entity-Type (CDokumente oder Document)
            
        Returns:
            True wenn Lock erfolgreich, False wenn bereits im Sync
@@ -78,19 +79,20 @@ class DocumentSync:
                acquired = self.redis.set(lock_key, "locked", nx=True, ex=LOCK_TTL_SECONDS)
                
                if not acquired:
-                    self._log(f"Redis lock bereits aktiv für Document {entity_id}", level='warn')
+                    self._log(f"Redis lock bereits aktiv für {entity_type} {entity_id}", level='warn')
                    return False
            
-            # STEP 2: Update syncStatus (für UI visibility) - falls Feld existiert
-            # NOTE: Ggf. muss syncStatus bei Document Entity erst angelegt werden
-            try:
-                await self.espocrm.update_entity('Document', entity_id, {
-                    'xaiSyncStatus': 'syncing'
-                })
-            except Exception as e:
-                self._log(f"Konnte xaiSyncStatus nicht setzen (Feld existiert evtl. nicht): {e}", level='debug')
+            # STEP 2: Update syncStatus (für UI visibility) - nur bei Document Entity
+            # CDokumente hat dieses Feld nicht - überspringen
+            if entity_type == 'Document':
+                try:
+                    await self.espocrm.update_entity(entity_type, entity_id, {
+                        'xaiSyncStatus': 'syncing'
+                    })
+                except Exception as e:
+                    self._log(f"Konnte xaiSyncStatus nicht setzen: {e}", level='debug')
            
-            self._log(f"Sync-Lock für Document {entity_id} erworben")
+            self._log(f"Sync-Lock für {entity_type} {entity_id} erworben")
            return True
            
        except Exception as e:
@@ -109,7 +111,8 @@ class DocumentSync:
        entity_id: str,
        success: bool = True,
        error_message: Optional[str] = None,
-        extra_fields: Optional[Dict[str, Any]] = None
+        extra_fields: Optional[Dict[str, Any]] = None,
+        entity_type: str = 'CDokumente'
    ) -> None:
        """
        Gibt Sync-Lock frei und setzt finalen Status
@@ -119,29 +122,31 @@ class DocumentSync:
            success: Ob Sync erfolgreich war
            error_message: Optional: Fehlermeldung
            extra_fields: Optional: Zusätzliche Felder (z.B. xaiFileId, xaiCollections)
+            entity_type: Entity-Type (CDokumente oder Document)
        """
        try:
            update_data = {}
            
-            # Status-Feld (falls vorhanden)
-            try:
-                update_data['xaiSyncStatus'] = 'synced' if success else 'failed'
-                
-                if error_message:
-                    update_data['xaiSyncError'] = error_message[:2000]
-                else:
-                    update_data['xaiSyncError'] = None
-            except:
-                pass  # Felder existieren evtl. nicht
+            # Status-Felder nur bei Document Entity (CDokumente hat diese Felder nicht)
+            if entity_type == 'Document':
+                try:
+                    update_data['xaiSyncStatus'] = 'synced' if success else 'failed'
+                    
+                    if error_message:
+                        update_data['xaiSyncError'] = error_message[:2000]
+                    else:
+                        update_data['xaiSyncError'] = None
+                except:
+                    pass  # Felder existieren evtl. nicht
            
            # Merge extra fields (z.B. xaiFileId, xaiCollections)
            if extra_fields:
                update_data.update(extra_fields)
            
            if update_data:
-                await self.espocrm.update_entity('Document', entity_id, update_data)
+                await self.espocrm.update_entity(entity_type, entity_id, update_data)
            
-            self._log(f"Sync-Lock released: Document {entity_id} → {'success' if success else 'failed'}")
+            self._log(f"Sync-Lock released: {entity_type} {entity_id} → {'success' if success else 'failed'}")
            
            # Release Redis lock
            if self.redis:
@@ -322,12 +327,17 @@ class DocumentSync:
        
        return result
    
-    async def get_document_download_info(self, document_id: str) -> Optional[Dict[str, Any]]:
+    async def get_document_download_info(self, document_id: str, entity_type: str = 'CDokumente') -> Optional[Dict[str, Any]]:
        """
        Holt Download-Informationen für ein Document
        
+        Args:
+            document_id: ID des Documents
+            entity_type: Entity-Type (CDokumente oder Document)
+        
        Returns:
            Dict mit:
+                - attachment_id: ID des Attachments
                - download_url: URL zum Download
                - filename: Dateiname
                - mime_type: MIME-Type
@@ -335,25 +345,49 @@ class DocumentSync:
        """
        try:
            # Hole vollständiges Document
-            doc = await self.espocrm.get_entity('Document', document_id)
+            doc = await self.espocrm.get_entity(entity_type, document_id)
            
-            # EspoCRM Document hat Attachments (Attachment ID in attachmentsIds)
-            attachment_ids = doc.get('attachmentsIds') or []
+            # EspoCRM Documents können Files auf verschiedene Arten speichern:
+            # CDokumente: dokumentId/dokumentName (Custom Entity)
+            # Document: fileId/fileName ODER attachmentsIds
            
-            if not attachment_ids:
-                self._log(f"⚠️  Document {document_id} hat keine Attachments", level='warn')
+            attachment_id = None
+            filename = None
+            
+            # Prüfe zuerst dokumentId (CDokumente Custom Entity)
+            if doc.get('dokumentId'):
+                attachment_id = doc.get('dokumentId')
+                filename = doc.get('dokumentName')
+                self._log(f"📎 CDokumente verwendet dokumentId: {attachment_id}")
+            
+            # Fallback: fileId (Standard Document Entity)
+            elif doc.get('fileId'):
+                attachment_id = doc.get('fileId')
+                filename = doc.get('fileName')
+                self._log(f"📎 Document verwendet fileId: {attachment_id}")
+            
+            # Fallback 2: attachmentsIds (z.B. bei zusätzlichen Attachments)
+            elif doc.get('attachmentsIds'):
+                attachment_ids = doc.get('attachmentsIds')
+                if attachment_ids:
+                    attachment_id = attachment_ids[0]
+                    self._log(f"📎 Document verwendet attachmentsIds: {attachment_id}")
+            
+            if not attachment_id:
+                self._log(f"⚠️  {entity_type} {document_id} hat weder dokumentId, fileId noch attachmentsIds", level='warn')
+                self._log(f"   Verfügbare Felder: {list(doc.keys())}")
                return None
            
-            # Nehme erstes Attachment (Documents haben normalerweise nur 1 File)
-            attachment_id = attachment_ids[0]
-            
            # Hole Attachment-Details
            attachment = await self.espocrm.get_entity('Attachment', attachment_id)
            
+            # Filename: Nutze dokumentName/fileName falls vorhanden, sonst aus Attachment
+            final_filename = filename or attachment.get('name', 'unknown')
+            
            return {
                'attachment_id': attachment_id,
                'download_url': f"/api/v1/Attachment/file/{attachment_id}",
-                'filename': attachment.get('name', 'unknown'),
+                'filename': final_filename,
                'mime_type': attachment.get('type', 'application/octet-stream'),
                'size': attachment.get('size', 0)
            }
@@ -476,7 +510,8 @@ class DocumentSync:
        xai_file_id: Optional[str] = None,
        collection_ids: Optional[List[str]] = None,
        file_hash: Optional[str] = None,
-        preview_data: Optional[bytes] = None
+        preview_data: Optional[bytes] = None,
+        entity_type: str = 'CDokumente'
    ) -> None:
        """
        Updated Document-Metadaten nach erfolgreichem xAI-Sync
@@ -487,20 +522,29 @@ class DocumentSync:
            collection_ids: Liste der xAI Collection IDs (optional)
            file_hash: MD5/SHA Hash des gesyncten Files
            preview_data: Vorschaubild (WebP) als bytes
+            entity_type: Entity-Type (CDokumente oder Document)
        """
        try:
            update_data = {}
            
            # Nur xAI-Felder updaten wenn vorhanden
            if xai_file_id:
-                update_data['xaiFileId'] = xai_file_id
+                # CDokumente verwendet xaiId, Document verwendet xaiFileId
+                if entity_type == 'CDokumente':
+                    update_data['xaiId'] = xai_file_id
+                else:
+                    update_data['xaiFileId'] = xai_file_id
            
            if collection_ids is not None:
                update_data['xaiCollections'] = collection_ids
            
            # Nur Status auf "Gesynct" setzen wenn xAI-File-ID vorhanden
            if xai_file_id:
-                update_data['dateiStatus'] = 'Gesynct'
+                # CDokumente verwendet fileStatus, Document verwendet dateiStatus
+                if entity_type == 'CDokumente':
+                    update_data['fileStatus'] = 'synced'
+                else:
+                    update_data['dateiStatus'] = 'Gesynct'
            
            # Hash speichern für zukünftige Change Detection
            if file_hash:
@@ -508,40 +552,78 @@ class DocumentSync:
            
            # Preview als Attachment hochladen (falls vorhanden)
            if preview_data:
-                await self._upload_preview_to_espocrm(document_id, preview_data)
+                await self._upload_preview_to_espocrm(document_id, preview_data, entity_type)
            
            # Nur updaten wenn es etwas zu updaten gibt
            if update_data:
-                await self.espocrm.update_entity('Document', document_id, update_data)
-                self._log(f"✅ Sync-Metadaten aktualisiert für Document {document_id}: {list(update_data.keys())}")
+                await self.espocrm.update_entity(entity_type, document_id, update_data)
+                self._log(f"✅ Sync-Metadaten aktualisiert für {entity_type} {document_id}: {list(update_data.keys())}")
            
        except Exception as e:
            self._log(f"❌ Fehler beim Update von Sync-Metadaten: {e}", level='error')
            raise
    
-    async def _upload_preview_to_espocrm(self, document_id: str, preview_data: bytes) -> None:
+    async def _upload_preview_to_espocrm(self, document_id: str, preview_data: bytes, entity_type: str = 'CDokumente') -> None:
        """
        Lädt Preview-Image als Attachment zu EspoCRM hoch
        
        Args:
            document_id: Document ID
            preview_data: WebP Preview als bytes
+            entity_type: Entity-Type (CDokumente oder Document)
        """
        try:
-            self._log(f"📤 Uploading preview image ({len(preview_data)} bytes)...")
+            self._log(f"📤 Uploading preview image to {entity_type} ({len(preview_data)} bytes)...")
            
-            # Upload via EspoCRM Attachment API
-            await self.espocrm.upload_attachment(
-                file_content=preview_data,
-                filename='preview.webp',
-                parent_type='Document',
-                parent_id=document_id,
-                field='preview',
-                mime_type='image/webp',
-                role='Attachment'
-            )
+            # EspoCRM erwartet base64-encoded file im Format: data:mime/type;base64,xxxxx
+            import base64
+            import aiohttp
            
-            self._log(f"✅ Preview erfolgreich hochgeladen")
+            # Base64-encode preview data
+            base64_data = base64.b64encode(preview_data).decode('ascii')
+            file_data_uri = f"data:image/webp;base64,{base64_data}"
+            
+            # Upload via JSON POST mit base64-encoded file field
+            url = self.espocrm.api_base_url.rstrip('/') + '/Attachment'
+            headers = {
+                'X-Api-Key': self.espocrm.api_key,
+                'Content-Type': 'application/json'
+            }
+            
+            payload = {
+                'name': 'preview.webp',
+                'type': 'image/webp',
+                'role': 'Attachment',
+                'field': 'preview',
+                'relatedType': entity_type,
+                'relatedId': document_id,
+                'file': file_data_uri
+            }
+            
+            self._log(f"📤 Posting to {url} with base64-encoded file ({len(base64_data)} chars)")
+            self._log(f"   relatedType={entity_type}, relatedId={document_id}, field=preview")
+            
+            timeout = aiohttp.ClientTimeout(total=30)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.post(url, headers=headers, json=payload) as response:
+                    self._log(f"Upload response status: {response.status}")
+                    
+                    if response.status >= 400:
+                        error_text = await response.text()
+                        self._log(f"❌ Upload failed: {error_text}", level='error')
+                        raise Exception(f"Upload error {response.status}: {error_text}")
+                    
+                    result = await response.json()
+                    attachment_id = result.get('id')
+                    self._log(f"✅ Preview Attachment created: {attachment_id}")
+            
+            # Update Entity mit previewId
+            self._log(f"📝 Updating {entity_type} with previewId...")
+            await self.espocrm.update_entity(entity_type, document_id, {
+                'previewId': attachment_id,
+                'previewName': 'preview.webp'
+            })
+            self._log(f"✅ {entity_type} previewId/previewName aktualisiert")
            
        except Exception as e:
            self._log(f"❌ Fehler beim Preview-Upload: {e}", level='error')
--- a/services/espocrm.py
+++ b/services/espocrm.py
@@ -341,12 +341,14 @@ class EspoCRMAPI:
        form_data.add_field('role', role)
        form_data.add_field('name', filename)
        
+        self._log(f"Upload params: parentType={parent_type}, parentId={parent_id}, field={field}, role={role}")
+        
        effective_timeout = aiohttp.ClientTimeout(total=self.api_timeout_seconds)
        
        async with aiohttp.ClientSession(timeout=effective_timeout) as session:
            try:
                async with session.post(url, headers=headers, data=form_data) as response:
-                    self._log(f"Upload response status: {response.status}", level='debug')
+                    self._log(f"Upload response status: {response.status}")
                    
                    if response.status == 401:
                        raise EspoCRMAuthError("Authentication failed - check API key")
@@ -356,6 +358,7 @@ class EspoCRMAPI:
                        raise EspoCRMError(f"Attachment endpoint not found")
                    elif response.status >= 400:
                        error_text = await response.text()
+                        self._log(f"❌ Upload failed with {response.status}. Response: {error_text}", level='error')
                        raise EspoCRMError(f"Upload error {response.status}: {error_text}")
                    
                    # Parse response