bsiggel/motia-iii
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): Enhance Akte sync process with batch processing and retry logic for failed events

Browse Source
This commit is contained in:
bsiggel
2026-03-26 11:13:37 +00:00
parent 52114a3c95
commit ef32373dc9
2 changed files with 52 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/steps/crm/akte/akte_sync_cron_step.py
View File

@@ -28,14 +28,15 @@ config = {
PENDING_ADVO_KEY = "advoware:pending_aktennummern"
PROCESSING_ADVO_KEY = "advoware:processing_aktennummern"
# Queue 2: written by EspoCRM webhook (keyed by entity ID)
# Queue 2: retry queue for failed akte.sync events (EspoCRM webhooks now emit directly)
PENDING_ID_KEY = "akte:pending_entity_ids"
PROCESSING_ID_KEY = "akte:processing_entity_ids"
DEBOUNCE_SECS = 10
BATCH_SIZE = 5 # max items to process per queue per cron tick
VALID_ADVOWARE_STATUSES = {'import', 'neu', 'new', 'aktiv', 'active'}
VALID_AI_STATUSES = {'new', 'neu', 'aktiv', 'active'}
VALID_ADVOWARE_STATUSES = frozenset({'import', 'neu', 'new', 'aktiv', 'active'})
VALID_AI_STATUSES = frozenset({'new', 'neu', 'aktiv', 'active'})
async def handler(input_data: None, ctx: FlowContext) -> None:
@@ -60,23 +61,18 @@ async def handler(input_data: None, ctx: FlowContext) -> None:
ctx.logger.info(f" Pending (aktennr) : {advo_pending}")
ctx.logger.info(f" Pending (akte_id) : {id_pending}")
processed = False
processed_count = 0
# ── Queue 1: Advoware Watcher (by Aktennummer) ─────────────────────
advo_entries = redis_client.zrangebyscore(PENDING_ADVO_KEY, min=0, max=cutoff, start=0, num=1)
if advo_entries:
aktennr = advo_entries[0]
if isinstance(aktennr, bytes):
aktennr = aktennr.decode()
advo_entries = redis_client.zrangebyscore(PENDING_ADVO_KEY, min=0, max=cutoff, start=0, num=BATCH_SIZE)
for raw in advo_entries:
aktennr = raw.decode() if isinstance(raw, bytes) else raw
score = redis_client.zscore(PENDING_ADVO_KEY, aktennr) or 0
age = time.time() - score
redis_client.zrem(PENDING_ADVO_KEY, aktennr)
redis_client.sadd(PROCESSING_ADVO_KEY, aktennr)
processed_count += 1
ctx.logger.info(f"📋 Aktennummer: {aktennr} (age={age:.1f}s)")
processed = True
try:
result = await espocrm.list_entities(
'CAkten',
@@ -85,51 +81,44 @@ async def handler(input_data: None, ctx: FlowContext) -> None:
)
if not result or not result.get('list'):
ctx.logger.warn(f"⚠️ No CAkten found for aktennummer={aktennr} – removing")
redis_client.srem(PROCESSING_ADVO_KEY, aktennr)
else:
akte = result['list'][0]
await _emit_if_eligible(akte, aktennr, ctx)
redis_client.srem(PROCESSING_ADVO_KEY, aktennr)
except Exception as e:
ctx.logger.error(f"❌ Error (aktennr queue) {aktennr}: {e}")
redis_client.zadd(PENDING_ADVO_KEY, {aktennr: time.time()})
finally:
redis_client.srem(PROCESSING_ADVO_KEY, aktennr)
raise
# ── Queue 2: EspoCRM Webhook (by Entity ID) ────────────────────────
id_entries = redis_client.zrangebyscore(PENDING_ID_KEY, min=0, max=cutoff, start=0, num=1)
if id_entries:
akte_id = id_entries[0]
if isinstance(akte_id, bytes):
akte_id = akte_id.decode()
# ── Queue 2: Retry queue for failed syncs ──────────────────────────
id_entries = redis_client.zrangebyscore(PENDING_ID_KEY, min=0, max=cutoff, start=0, num=BATCH_SIZE)
for raw in id_entries:
akte_id = raw.decode() if isinstance(raw, bytes) else raw
score = redis_client.zscore(PENDING_ID_KEY, akte_id) or 0
age = time.time() - score
redis_client.zrem(PENDING_ID_KEY, akte_id)
redis_client.sadd(PROCESSING_ID_KEY, akte_id)
ctx.logger.info(f"📋 Entity ID: {akte_id} (age={age:.1f}s)")
processed = True
processed_count += 1
ctx.logger.info(f"📋 Entity ID (retry): {akte_id} (age={age:.1f}s)")
try:
akte = await espocrm.get_entity('CAkten', akte_id)
if not akte:
ctx.logger.warn(f"⚠️ No CAkten found for id={akte_id} – removing")
redis_client.srem(PROCESSING_ID_KEY, akte_id)
else:
await _emit_if_eligible(akte, None, ctx)
redis_client.srem(PROCESSING_ID_KEY, akte_id)
except Exception as e:
ctx.logger.error(f"❌ Error (entity-id queue) {akte_id}: {e}")
ctx.logger.error(f"❌ Error (retry queue) {akte_id}: {e}")
redis_client.zadd(PENDING_ID_KEY, {akte_id: time.time()})
finally:
redis_client.srem(PROCESSING_ID_KEY, akte_id)
raise
if not processed:
if not processed_count:
if advo_pending > 0 or id_pending > 0:
ctx.logger.info(f"⏸️ Entries pending but all too recent (< {DEBOUNCE_SECS}s)")
else:
ctx.logger.info("✓ Both queues empty")
else:
ctx.logger.info(f"✓ Processed {processed_count} item(s)")
ctx.logger.info("=" * 60)