feat: Implement AI Knowledge Sync Utilities and RAGFlow Service

- Added `aiknowledge_sync_utils.py` for provider-agnostic synchronization logic for CAIKnowledge entities, supporting both xAI and RAGFlow. - Introduced lifecycle management for CAIKnowledge entities including states: new, active, paused, and deactivated. - Implemented change detection using Blake3 hash for efficient document synchronization. - Created `ragflow_service.py` to handle dataset and document management with RAGFlow API. - Added daily cron job in `aiknowledge_daily_cron_step.py` to synchronize active CAIKnowledge entities with unclean or failed statuses. - Developed `aiknowledge_sync_event_step.py` to process synchronization events from webhooks and cron jobs.
2026-03-26 21:38:42 +00:00
parent 439101f35d
commit 9b2fb5ae4a
8 changed files with 1406 additions and 1 deletions
--- a/services/ragflow_service.py
+++ b/services/ragflow_service.py
@@ -0,0 +1,511 @@
+"""RAGFlow Dataset & Document Service"""
+import os
+import asyncio
+from functools import partial
+from typing import Optional, List, Dict, Any
+from services.logging_utils import get_service_logger
+
+RAGFLOW_DEFAULT_BASE_URL = "http://192.168.1.64:9380"
+
+# Defaults fuer Dokument-Analyse
+RAGFLOW_AUTO_KEYWORDS = 14
+RAGFLOW_AUTO_QUESTIONS = 7
+
+
+def _base_to_dict(obj: Any) -> Any:
+    """
+    Konvertiert ragflow_sdk.modules.base.Base rekursiv zu einem plain dict.
+    Filtert den internen 'rag'-Client-Key heraus.
+    """
+    try:
+        from ragflow_sdk.modules.base import Base
+        if isinstance(obj, Base):
+            return {k: _base_to_dict(v) for k, v in vars(obj).items() if k != 'rag'}
+    except ImportError:
+        pass
+    if isinstance(obj, dict):
+        return {k: _base_to_dict(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_base_to_dict(i) for i in obj]
+    return obj
+
+
+class RAGFlowService:
+    """
+    Client fuer RAGFlow API via ragflow-sdk (Python SDK).
+
+    Wrapt das synchrone SDK in asyncio.run_in_executor, sodass
+    es nahtlos in Motia-Steps (async) verwendet werden kann.
+
+    Dataflow beim Upload:
+      upload_document() →
+        1. upload_documents([{blob}])      # Datei hochladen
+        2. doc.update({meta_fields})       # blake3 + advoware-Felder setzen
+        3. async_parse_documents([id])     # Parsing starten (chunk_method=laws)
+
+    Benoetigte Umgebungsvariablen:
+    - RAGFLOW_API_KEY   – API Key
+    - RAGFLOW_BASE_URL  – Optional, URL Override (Default: http://192.168.1.64:9380)
+    """
+
+    SUPPORTED_MIME_TYPES = {
+        'application/pdf',
+        'application/msword',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        'application/vnd.ms-excel',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        'application/vnd.oasis.opendocument.text',
+        'application/epub+zip',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        'text/plain',
+        'text/html',
+        'text/markdown',
+        'text/csv',
+        'text/xml',
+        'application/json',
+        'application/xml',
+    }
+
+    def __init__(self, ctx=None):
+        self.api_key = os.getenv('RAGFLOW_API_KEY', '')
+        base_url_env = os.getenv('RAGFLOW_BASE_URL', '')
+        self.base_url = base_url_env or RAGFLOW_DEFAULT_BASE_URL
+        self.ctx = ctx
+        self.logger = get_service_logger('ragflow', ctx)
+        self._rag = None
+
+        if not self.api_key:
+            raise ValueError("RAGFLOW_API_KEY not configured in environment")
+
+    def _log(self, msg: str, level: str = 'info') -> None:
+        log_func = getattr(self.logger, level, self.logger.info)
+        log_func(msg)
+
+    def _get_client(self):
+        """Gibt RAGFlow SDK Client zurueck (lazy init, sync)."""
+        if self._rag is None:
+            from ragflow_sdk import RAGFlow
+            self._rag = RAGFlow(api_key=self.api_key, base_url=self.base_url)
+        return self._rag
+
+    async def _run(self, func, *args, **kwargs):
+        """Fuehrt synchrone SDK-Funktion in ThreadPoolExecutor aus."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, partial(func, *args, **kwargs))
+
+    # ========== Dataset Management ==========
+
+    async def create_dataset(
+        self,
+        name: str,
+        chunk_method: str = 'laws',
+        embedding_model: Optional[str] = None,
+        description: Optional[str] = None,
+        auto_keywords: int = RAGFLOW_AUTO_KEYWORDS,
+        auto_questions: int = RAGFLOW_AUTO_QUESTIONS,
+    ) -> Dict:
+        """
+        Erstellt ein neues RAGFlow Dataset (entspricht xAI Collection).
+
+        Verwendet standardmaessig chunk_method='laws' (optimiert fuer Rechtsdokumente).
+        Setzt nach der Erstellung auto_keywords=14 und auto_questions=7.
+
+        Returns:
+            dict mit 'id', 'name', 'chunk_method', 'parser_config', etc.
+        """
+        self._log(
+            f"📚 Creating dataset: {name} "
+            f"(chunk_method={chunk_method}, keywords={auto_keywords}, questions={auto_questions})"
+        )
+
+        def _create():
+            rag = self._get_client()
+            kwargs = dict(name=name, chunk_method=chunk_method)
+            if embedding_model:
+                kwargs['embedding_model'] = embedding_model
+            if description:
+                kwargs['description'] = description
+            dataset = rag.create_dataset(**kwargs)
+
+            # parser_config kann erst nach create via update() gesetzt werden
+            dataset.update({
+                'parser_config': {
+                    'auto_keywords': auto_keywords,
+                    'auto_questions': auto_questions,
+                }
+            })
+            return self._dataset_to_dict(dataset)
+
+        result = await self._run(_create)
+        self._log(f"✅ Dataset created: {result.get('id')} ({name})")
+        return result
+
+    async def get_dataset_by_name(self, name: str) -> Optional[Dict]:
+        """
+        Sucht Dataset nach Name. Gibt None zurueck wenn nicht gefunden.
+        """
+        def _find():
+            rag = self._get_client()
+            # list_datasets(name=...) hat Permission-Bugs – lokal filtern
+            all_datasets = rag.list_datasets(page_size=100)
+            for ds in all_datasets:
+                if getattr(ds, 'name', None) == name:
+                    return self._dataset_to_dict(ds)
+            return None
+
+        result = await self._run(_find)
+        if result:
+            self._log(f"🔍 Dataset found: {result.get('id')} ({name})")
+        return result
+
+    async def ensure_dataset(
+        self,
+        name: str,
+        chunk_method: str = 'laws',
+        embedding_model: Optional[str] = None,
+        description: Optional[str] = None,
+        auto_keywords: int = RAGFLOW_AUTO_KEYWORDS,
+        auto_questions: int = RAGFLOW_AUTO_QUESTIONS,
+    ) -> Dict:
+        """
+        Gibt bestehendes Dataset zurueck oder erstellt ein neues (get-or-create).
+        Entspricht xAI create_collection mit idempotency.
+
+        Returns:
+            dict mit 'id', 'name', etc.
+        """
+        existing = await self.get_dataset_by_name(name)
+        if existing:
+            self._log(f"✅ Dataset exists: {existing.get('id')} ({name})")
+            return existing
+        return await self.create_dataset(
+            name=name,
+            chunk_method=chunk_method,
+            embedding_model=embedding_model,
+            description=description,
+            auto_keywords=auto_keywords,
+            auto_questions=auto_questions,
+        )
+
+    async def delete_dataset(self, dataset_id: str) -> None:
+        """
+        Loescht ein Dataset inklusive aller Dokumente.
+        Entspricht xAI delete_collection.
+        """
+        self._log(f"🗑️  Deleting dataset: {dataset_id}")
+
+        def _delete():
+            rag = self._get_client()
+            rag.delete_datasets(ids=[dataset_id])
+
+        await self._run(_delete)
+        self._log(f"✅ Dataset deleted: {dataset_id}")
+
+    async def list_datasets(self) -> List[Dict]:
+        """Listet alle Datasets auf."""
+        def _list():
+            rag = self._get_client()
+            return [self._dataset_to_dict(d) for d in rag.list_datasets()]
+
+        result = await self._run(_list)
+        self._log(f"📋 Listed {len(result)} datasets")
+        return result
+
+    # ========== Document Management ==========
+
+    async def upload_document(
+        self,
+        dataset_id: str,
+        file_content: bytes,
+        filename: str,
+        mime_type: str = 'application/octet-stream',
+        blake3_hash: Optional[str] = None,
+        espocrm_id: Optional[str] = None,
+        description: Optional[str] = None,
+        advoware_art: Optional[str] = None,
+        advoware_bemerkung: Optional[str] = None,
+    ) -> Dict:
+        """
+        Laedt ein Dokument in ein Dataset hoch.
+
+        Ablauf (3 Schritte):
+          1. upload_documents()         – Datei hochladen
+          2. doc.update(meta_fields)    – Metadaten setzen inkl. blake3_hash
+          3. async_parse_documents()    – Parsing mit chunk_method=laws starten
+
+        Meta-Felder die gesetzt werden:
+          - blake3_hash        (fuer Change Detection, entspricht xAI BLAKE3)
+          - espocrm_id         (Rueckreferenz zu EspoCRM CDokument)
+          - description        (Dokumentbeschreibung)
+          - advoware_art       (Advoware Dokumenten-Art)
+          - advoware_bemerkung (Advoware Bemerkung/Notiz)
+
+        Returns:
+            dict mit 'id', 'name', 'run', 'meta_fields', etc.
+        """
+        if mime_type == 'application/octet-stream' and filename.lower().endswith('.pdf'):
+            mime_type = 'application/pdf'
+
+        self._log(
+            f"📤 Uploading {len(file_content)} bytes to dataset {dataset_id}: "
+            f"{filename} ({mime_type})"
+        )
+
+        def _upload_and_tag():
+            rag = self._get_client()
+            datasets = rag.list_datasets(id=dataset_id)
+            if not datasets:
+                raise RuntimeError(f"Dataset not found: {dataset_id}")
+            dataset = datasets[0]
+
+            # Schritt 1: Upload
+            dataset.upload_documents([{
+                'display_name': filename,
+                'blob': file_content,
+            }])
+
+            # Dokument-ID ermitteln (neuestes mit passendem Namen)
+            base_name = filename.split('/')[-1]
+            docs = dataset.list_documents(keywords=base_name, page_size=10)
+            doc = None
+            for d in docs:
+                if d.name == filename or d.name == base_name:
+                    doc = d
+                    break
+            if doc is None and docs:
+                doc = docs[0]  # Fallback
+            if doc is None:
+                raise RuntimeError(f"Document not found after upload: {filename}")
+
+            # Schritt 2: Meta-Fields setzen
+            meta: Dict[str, str] = {}
+            if blake3_hash:
+                meta['blake3_hash'] = blake3_hash
+            if espocrm_id:
+                meta['espocrm_id'] = espocrm_id
+            if description:
+                meta['description'] = description
+            if advoware_art:
+                meta['advoware_art'] = advoware_art
+            if advoware_bemerkung:
+                meta['advoware_bemerkung'] = advoware_bemerkung
+
+            if meta:
+                doc.update({'meta_fields': meta})
+
+            # Schritt 3: Parsing starten
+            dataset.async_parse_documents([doc.id])
+
+            return self._document_to_dict(doc)
+
+        result = await self._run(_upload_and_tag)
+        self._log(
+            f"✅ Document uploaded & parsing started: {result.get('id')} ({filename})"
+        )
+        return result
+
+    async def update_document_meta(
+        self,
+        dataset_id: str,
+        doc_id: str,
+        blake3_hash: Optional[str] = None,
+        description: Optional[str] = None,
+        advoware_art: Optional[str] = None,
+        advoware_bemerkung: Optional[str] = None,
+    ) -> None:
+        """
+        Aktualisiert nur die Metadaten eines Dokuments (ohne Re-Upload).
+        Entspricht xAI PATCH-Metadata-Only.
+        Startet Parsing neu, da Chunk-Injection von meta_fields abhaengt.
+        """
+        self._log(f"✏️  Updating metadata for document {doc_id}")
+
+        def _update():
+            rag = self._get_client()
+            datasets = rag.list_datasets(id=dataset_id)
+            if not datasets:
+                raise RuntimeError(f"Dataset not found: {dataset_id}")
+            dataset = datasets[0]
+            docs = dataset.list_documents(id=doc_id)
+            if not docs:
+                raise RuntimeError(f"Document not found: {doc_id}")
+            doc = docs[0]
+
+            # Bestehende meta_fields lesen und mergen
+            existing_meta = _base_to_dict(doc.meta_fields) or {}
+            if blake3_hash is not None:
+                existing_meta['blake3_hash'] = blake3_hash
+            if description is not None:
+                existing_meta['description'] = description
+            if advoware_art is not None:
+                existing_meta['advoware_art'] = advoware_art
+            if advoware_bemerkung is not None:
+                existing_meta['advoware_bemerkung'] = advoware_bemerkung
+
+            doc.update({'meta_fields': existing_meta})
+            # Re-parsing noetig damit Chunks aktualisierte Metadata enthalten
+            dataset.async_parse_documents([doc.id])
+
+        await self._run(_update)
+        self._log(f"✅ Metadata updated and re-parsing started: {doc_id}")
+
+    async def remove_document(self, dataset_id: str, doc_id: str) -> None:
+        """
+        Loescht ein Dokument aus einem Dataset.
+        Entspricht xAI remove_from_collection.
+        """
+        self._log(f"🗑️  Removing document {doc_id} from dataset {dataset_id}")
+
+        def _delete():
+            rag = self._get_client()
+            datasets = rag.list_datasets(id=dataset_id)
+            if not datasets:
+                raise RuntimeError(f"Dataset not found: {dataset_id}")
+            datasets[0].delete_documents(ids=[doc_id])
+
+        await self._run(_delete)
+        self._log(f"✅ Document removed: {doc_id}")
+
+    async def list_documents(self, dataset_id: str) -> List[Dict]:
+        """
+        Listet alle Dokumente in einem Dataset auf (paginiert).
+        Entspricht xAI list_collection_documents.
+        """
+        self._log(f"📋 Listing documents in dataset {dataset_id}")
+
+        def _list():
+            rag = self._get_client()
+            datasets = rag.list_datasets(id=dataset_id)
+            if not datasets:
+                raise RuntimeError(f"Dataset not found: {dataset_id}")
+            dataset = datasets[0]
+            docs = []
+            page = 1
+            while True:
+                batch = dataset.list_documents(page=page, page_size=100)
+                if not batch:
+                    break
+                docs.extend(batch)
+                if len(batch) < 100:
+                    break
+                page += 1
+            return [self._document_to_dict(d) for d in docs]
+
+        result = await self._run(_list)
+        self._log(f"✅ Listed {len(result)} documents")
+        return result
+
+    async def get_document(self, dataset_id: str, doc_id: str) -> Optional[Dict]:
+        """Holt ein einzelnes Dokument by ID. None wenn nicht gefunden."""
+        def _get():
+            rag = self._get_client()
+            datasets = rag.list_datasets(id=dataset_id)
+            if not datasets:
+                return None
+            docs = datasets[0].list_documents(id=doc_id)
+            if not docs:
+                return None
+            return self._document_to_dict(docs[0])
+
+        result = await self._run(_get)
+        if result:
+            self._log(f"📄 Document found: {result.get('name')} (run={result.get('run')})")
+        return result
+
+    async def wait_for_parsing(
+        self,
+        dataset_id: str,
+        doc_id: str,
+        timeout_seconds: int = 120,
+        poll_interval: float = 3.0,
+    ) -> Dict:
+        """
+        Wartet bis das Parsing eines Dokuments abgeschlossen ist.
+
+        Returns:
+            Aktueller Dokument-State als dict.
+
+        Raises:
+            TimeoutError: Wenn Parsing nicht innerhalb timeout_seconds fertig wird.
+            RuntimeError: Wenn Parsing fehlschlaegt.
+        """
+        self._log(f"⏳ Waiting for parsing: {doc_id} (timeout={timeout_seconds}s)")
+        elapsed = 0.0
+
+        while elapsed < timeout_seconds:
+            doc = await self.get_document(dataset_id, doc_id)
+            if doc is None:
+                raise RuntimeError(f"Document disappeared during parsing: {doc_id}")
+
+            run_status = doc.get('run', 'UNSTART')
+            if run_status == 'DONE':
+                self._log(
+                    f"✅ Parsing done: {doc_id} "
+                    f"(chunks={doc.get('chunk_count')}, tokens={doc.get('token_count')})"
+                )
+                return doc
+            elif run_status in ('FAIL', 'CANCEL'):
+                raise RuntimeError(
+                    f"Parsing failed for {doc_id}: status={run_status}, "
+                    f"msg={doc.get('progress_msg', '')}"
+                )
+
+            await asyncio.sleep(poll_interval)
+            elapsed += poll_interval
+
+        raise TimeoutError(
+            f"Parsing timeout after {timeout_seconds}s for document {doc_id}"
+        )
+
+    # ========== MIME Type Support ==========
+
+    def is_mime_type_supported(self, mime_type: str) -> bool:
+        """Prueft ob RAGFlow diesen MIME-Type verarbeiten kann."""
+        return mime_type.lower().strip() in self.SUPPORTED_MIME_TYPES
+
+    # ========== Internal Helpers ==========
+
+    def _dataset_to_dict(self, dataset) -> Dict:
+        """Konvertiert RAGFlow DataSet Objekt zu dict (inkl. parser_config unwrap)."""
+        return {
+            'id': getattr(dataset, 'id', None),
+            'name': getattr(dataset, 'name', None),
+            'chunk_method': getattr(dataset, 'chunk_method', None),
+            'embedding_model': getattr(dataset, 'embedding_model', None),
+            'description': getattr(dataset, 'description', None),
+            'chunk_count': getattr(dataset, 'chunk_count', 0),
+            'document_count': getattr(dataset, 'document_count', 0),
+            'parser_config': _base_to_dict(getattr(dataset, 'parser_config', {})),
+        }
+
+    def _document_to_dict(self, doc) -> Dict:
+        """
+        Konvertiert RAGFlow Document Objekt zu dict.
+
+        meta_fields wird via _base_to_dict() zu einem plain dict unwrapped.
+        Enthaelt blake3_hash, espocrm_id, description, advoware_art,
+        advoware_bemerkung sofern gesetzt.
+        """
+        raw_meta = getattr(doc, 'meta_fields', None)
+        meta_dict = _base_to_dict(raw_meta) if raw_meta is not None else {}
+
+        return {
+            'id': getattr(doc, 'id', None),
+            'name': getattr(doc, 'name', None),
+            'dataset_id': getattr(doc, 'dataset_id', None),
+            'chunk_method': getattr(doc, 'chunk_method', None),
+            'size': getattr(doc, 'size', 0),
+            'token_count': getattr(doc, 'token_count', 0),
+            'chunk_count': getattr(doc, 'chunk_count', 0),
+            'run': getattr(doc, 'run', 'UNSTART'),
+            'progress': getattr(doc, 'progress', 0.0),
+            'progress_msg': getattr(doc, 'progress_msg', ''),
+            'source_type': getattr(doc, 'source_type', 'local'),
+            'created_by': getattr(doc, 'created_by', ''),
+            'process_duration': getattr(doc, 'process_duration', 0.0),
+            # Metadaten (blake3_hash hier drin wenn gesetzt)
+            'meta_fields': meta_dict,
+            'blake3_hash': meta_dict.get('blake3_hash'),
+            'espocrm_id': meta_dict.get('espocrm_id'),
+            'parser_config': _base_to_dict(getattr(doc, 'parser_config', None)),
+        }