Refactor Akte and Document Sync Logic
- Removed the old VMH Document xAI Sync Handler implementation. - Introduced new xAI Upload Utilities for shared upload logic across sync flows. - Created a unified Akte sync structure with cron polling and event handling. - Implemented Akte Sync Cron Poller to manage pending Aktennummern with a debounce mechanism. - Developed Akte Sync Event Handler for synchronized processing across Advoware and xAI. - Enhanced logging and error handling throughout the new sync processes. - Ensured compatibility with existing Redis and EspoCRM services.
This commit is contained in:
135
src/steps/akte/akte_sync_cron_step.py
Normal file
135
src/steps/akte/akte_sync_cron_step.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""
|
||||
Akte Sync - Cron Poller
|
||||
|
||||
Polls Redis Sorted Set for pending Aktennummern every 10 seconds.
|
||||
Respects a 10-second debounce window so that rapid filesystem events
|
||||
(e.g. many files being updated at once) are batched into a single sync.
|
||||
|
||||
Redis keys (same as advoware-watcher writes to):
|
||||
advoware:pending_aktennummern – Sorted Set { aktennummer → timestamp }
|
||||
advoware:processing_aktennummern – Set (tracks active syncs)
|
||||
|
||||
Eligibility check (either flag triggers a sync):
|
||||
syncSchalter == True AND aktivierungsstatus in valid list → Advoware sync
|
||||
aiAktivierungsstatus in valid list → xAI sync
|
||||
"""
|
||||
|
||||
from motia import FlowContext, cron
|
||||
|
||||
|
||||
config = {
|
||||
"name": "Akte Sync - Cron Poller",
|
||||
"description": "Poll Redis for pending Aktennummern and emit akte.sync events (10 s debounce)",
|
||||
"flows": ["akte-sync"],
|
||||
"triggers": [cron("*/10 * * * * *")],
|
||||
"enqueues": ["akte.sync"],
|
||||
}
|
||||
|
||||
PENDING_KEY = "advoware:pending_aktennummern"
|
||||
PROCESSING_KEY = "advoware:processing_aktennummern"
|
||||
DEBOUNCE_SECS = 10
|
||||
|
||||
VALID_ADVOWARE_STATUSES = {'import', 'neu', 'new', 'aktiv', 'active'}
|
||||
VALID_AI_STATUSES = {'new', 'neu', 'aktiv', 'active'}
|
||||
|
||||
|
||||
async def handler(input_data: None, ctx: FlowContext) -> None:
|
||||
import time
|
||||
from services.redis_client import get_redis_client
|
||||
from services.espocrm import EspoCRMAPI
|
||||
|
||||
ctx.logger.info("=" * 60)
|
||||
ctx.logger.info("⏰ AKTE CRON POLLER")
|
||||
|
||||
redis_client = get_redis_client(strict=False)
|
||||
if not redis_client:
|
||||
ctx.logger.error("❌ Redis unavailable")
|
||||
ctx.logger.info("=" * 60)
|
||||
return
|
||||
|
||||
espocrm = EspoCRMAPI(ctx)
|
||||
cutoff = time.time() - DEBOUNCE_SECS
|
||||
|
||||
pending_count = redis_client.zcard(PENDING_KEY)
|
||||
processing_count = redis_client.scard(PROCESSING_KEY)
|
||||
ctx.logger.info(f" Pending : {pending_count}")
|
||||
ctx.logger.info(f" Processing : {processing_count}")
|
||||
|
||||
# Pull oldest entry that has passed the debounce window
|
||||
old_entries = redis_client.zrangebyscore(PENDING_KEY, min=0, max=cutoff, start=0, num=1)
|
||||
|
||||
if not old_entries:
|
||||
if pending_count > 0:
|
||||
ctx.logger.info(f"⏸️ {pending_count} pending – all too recent (< {DEBOUNCE_SECS}s)")
|
||||
else:
|
||||
ctx.logger.info("✓ Queue empty")
|
||||
ctx.logger.info("=" * 60)
|
||||
return
|
||||
|
||||
aktennr = old_entries[0]
|
||||
if isinstance(aktennr, bytes):
|
||||
aktennr = aktennr.decode()
|
||||
|
||||
score = redis_client.zscore(PENDING_KEY, aktennr) or 0
|
||||
age = time.time() - score
|
||||
redis_client.zrem(PENDING_KEY, aktennr)
|
||||
redis_client.sadd(PROCESSING_KEY, aktennr)
|
||||
|
||||
ctx.logger.info(f"📋 Aktennummer: {aktennr} (age={age:.1f}s)")
|
||||
|
||||
try:
|
||||
# ── Lookup in EspoCRM ──────────────────────────────────────
|
||||
result = await espocrm.list_entities(
|
||||
'CAkten',
|
||||
where=[{
|
||||
'type': 'equals',
|
||||
'attribute': 'aktennummer',
|
||||
'value': aktennr,
|
||||
}],
|
||||
max_size=1,
|
||||
)
|
||||
|
||||
if not result or not result.get('list'):
|
||||
ctx.logger.warn(f"⚠️ No CAkten found for aktennummer={aktennr} – removing")
|
||||
redis_client.srem(PROCESSING_KEY, aktennr)
|
||||
ctx.logger.info("=" * 60)
|
||||
return
|
||||
|
||||
akte = result['list'][0]
|
||||
akte_id = akte['id']
|
||||
sync_schalter = akte.get('syncSchalter', False)
|
||||
aktivierungsstatus = str(akte.get('aktivierungsstatus') or '').lower()
|
||||
ai_status = str(akte.get('aiAktivierungsstatus') or '').lower()
|
||||
|
||||
advoware_eligible = sync_schalter and aktivierungsstatus in VALID_ADVOWARE_STATUSES
|
||||
xai_eligible = ai_status in VALID_AI_STATUSES
|
||||
|
||||
ctx.logger.info(f" Akte ID : {akte_id}")
|
||||
ctx.logger.info(f" aktivierungsstatus : {aktivierungsstatus} ({'✅' if advoware_eligible else '⏭️'})")
|
||||
ctx.logger.info(f" aiAktivierungsstatus : {ai_status} ({'✅' if xai_eligible else '⏭️'})")
|
||||
|
||||
if not advoware_eligible and not xai_eligible:
|
||||
ctx.logger.warn(f"⚠️ Akte {aktennr} not eligible for any sync – removing")
|
||||
redis_client.srem(PROCESSING_KEY, aktennr)
|
||||
ctx.logger.info("=" * 60)
|
||||
return
|
||||
|
||||
# ── Emit sync event ────────────────────────────────────────
|
||||
await ctx.enqueue({
|
||||
'topic': 'akte.sync',
|
||||
'data': {
|
||||
'aktennummer': aktennr,
|
||||
'akte_id': akte_id,
|
||||
},
|
||||
})
|
||||
ctx.logger.info(f"📤 akte.sync emitted (akte_id={akte_id})")
|
||||
|
||||
except Exception as e:
|
||||
ctx.logger.error(f"❌ Error processing {aktennr}: {e}")
|
||||
# Requeue for retry
|
||||
redis_client.zadd(PENDING_KEY, {aktennr: time.time()})
|
||||
redis_client.srem(PROCESSING_KEY, aktennr)
|
||||
raise
|
||||
|
||||
finally:
|
||||
ctx.logger.info("=" * 60)
|
||||
Reference in New Issue
Block a user