fix: Remove deprecated AI Chat Completions and Models List API implementations

- Enhanced the AI Chat Completions API to support true streaming using async generators and proper SSE headers.
- Updated endpoint paths to align with OpenAI's API versioning.
- Improved logging for request details and error handling.
- Added a new AI Models List API to return available models compatible with chat completions.
- Refactored code for better readability and maintainability, including the extraction of common functionalities.
- Introduced a VMH-specific Chat Completions API with similar features and structure.

=0.3.0

@@ -1,49 +0,0 @@
-Requirement already satisfied: langchain in ./.venv/lib/python3.13/site-packages (1.2.12)
-Requirement already satisfied: langchain-xai in ./.venv/lib/python3.13/site-packages (1.2.2)
-Requirement already satisfied: langchain-core in ./.venv/lib/python3.13/site-packages (1.2.18)
-Requirement already satisfied: langgraph<1.2.0,>=1.1.1 in ./.venv/lib/python3.13/site-packages (from langchain) (1.1.2)
-Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in ./.venv/lib/python3.13/site-packages (from langchain) (2.12.5)
-Requirement already satisfied: jsonpatch<2.0.0,>=1.33.0 in ./.venv/lib/python3.13/site-packages (from langchain-core) (1.33)
-Requirement already satisfied: langsmith<1.0.0,>=0.3.45 in ./.venv/lib/python3.13/site-packages (from langchain-core) (0.7.17)
-Requirement already satisfied: packaging>=23.2.0 in ./.venv/lib/python3.13/site-packages (from langchain-core) (26.0)
-Requirement already satisfied: pyyaml<7.0.0,>=5.3.0 in ./.venv/lib/python3.13/site-packages (from langchain-core) (6.0.3)
-Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in ./.venv/lib/python3.13/site-packages (from langchain-core) (9.1.4)
-Requirement already satisfied: typing-extensions<5.0.0,>=4.7.0 in ./.venv/lib/python3.13/site-packages (from langchain-core) (4.15.0)
-Requirement already satisfied: uuid-utils<1.0,>=0.12.0 in ./.venv/lib/python3.13/site-packages (from langchain-core) (0.14.1)
-Requirement already satisfied: jsonpointer>=1.9 in ./.venv/lib/python3.13/site-packages (from jsonpatch<2.0.0,>=1.33.0->langchain-core) (3.0.0)
-Requirement already satisfied: langgraph-checkpoint<5.0.0,>=2.1.0 in ./.venv/lib/python3.13/site-packages (from langgraph<1.2.0,>=1.1.1->langchain) (4.0.1)
-Requirement already satisfied: langgraph-prebuilt<1.1.0,>=1.0.8 in ./.venv/lib/python3.13/site-packages (from langgraph<1.2.0,>=1.1.1->langchain) (1.0.8)
-Requirement already satisfied: langgraph-sdk<0.4.0,>=0.3.0 in ./.venv/lib/python3.13/site-packages (from langgraph<1.2.0,>=1.1.1->langchain) (0.3.11)
-Requirement already satisfied: xxhash>=3.5.0 in ./.venv/lib/python3.13/site-packages (from langgraph<1.2.0,>=1.1.1->langchain) (3.6.0)
-Requirement already satisfied: ormsgpack>=1.12.0 in ./.venv/lib/python3.13/site-packages (from langgraph-checkpoint<5.0.0,>=2.1.0->langgraph<1.2.0,>=1.1.1->langchain) (1.12.2)
-Requirement already satisfied: httpx>=0.25.2 in ./.venv/lib/python3.13/site-packages (from langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (0.28.1)
-Requirement already satisfied: orjson>=3.11.5 in ./.venv/lib/python3.13/site-packages (from langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (3.11.7)
-Requirement already satisfied: requests-toolbelt>=1.0.0 in ./.venv/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core) (1.0.0)
-Requirement already satisfied: requests>=2.0.0 in ./.venv/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core) (2.32.5)
-Requirement already satisfied: zstandard>=0.23.0 in ./.venv/lib/python3.13/site-packages (from langsmith<1.0.0,>=0.3.45->langchain-core) (0.25.0)
-Requirement already satisfied: anyio in ./.venv/lib/python3.13/site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (4.12.1)
-Requirement already satisfied: certifi in ./.venv/lib/python3.13/site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (2026.2.25)
-Requirement already satisfied: httpcore==1.* in ./.venv/lib/python3.13/site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (1.0.9)
-Requirement already satisfied: idna in ./.venv/lib/python3.13/site-packages (from httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (3.11)
-Requirement already satisfied: h11>=0.16 in ./.venv/lib/python3.13/site-packages (from httpcore==1.*->httpx>=0.25.2->langgraph-sdk<0.4.0,>=0.3.0->langgraph<1.2.0,>=1.1.1->langchain) (0.16.0)
-Requirement already satisfied: annotated-types>=0.6.0 in ./.venv/lib/python3.13/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.7.0)
-Requirement already satisfied: pydantic-core==2.41.5 in ./.venv/lib/python3.13/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.41.5)
-Requirement already satisfied: typing-inspection>=0.4.2 in ./.venv/lib/python3.13/site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.4.2)
-Requirement already satisfied: aiohttp<4.0.0,>=3.9.1 in ./.venv/lib/python3.13/site-packages (from langchain-xai) (3.13.3)
-Requirement already satisfied: langchain-openai<2.0.0,>=1.1.7 in ./.venv/lib/python3.13/site-packages (from langchain-xai) (1.1.11)
-Requirement already satisfied: aiohappyeyeballs>=2.5.0 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (2.6.1)
-Requirement already satisfied: aiosignal>=1.4.0 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (1.4.0)
-Requirement already satisfied: attrs>=17.3.0 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (25.4.0)
-Requirement already satisfied: frozenlist>=1.1.1 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (1.8.0)
-Requirement already satisfied: multidict<7.0,>=4.5 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (6.7.1)
-Requirement already satisfied: propcache>=0.2.0 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (0.4.1)
-Requirement already satisfied: yarl<2.0,>=1.17.0 in ./.venv/lib/python3.13/site-packages (from aiohttp<4.0.0,>=3.9.1->langchain-xai) (1.22.0)
-Requirement already satisfied: openai<3.0.0,>=2.26.0 in ./.venv/lib/python3.13/site-packages (from langchain-openai<2.0.0,>=1.1.7->langchain-xai) (2.26.0)
-Requirement already satisfied: tiktoken<1.0.0,>=0.7.0 in ./.venv/lib/python3.13/site-packages (from langchain-openai<2.0.0,>=1.1.7->langchain-xai) (0.12.0)
-Requirement already satisfied: distro<2,>=1.7.0 in ./.venv/lib/python3.13/site-packages (from openai<3.0.0,>=2.26.0->langchain-openai<2.0.0,>=1.1.7->langchain-xai) (1.9.0)
-Requirement already satisfied: jiter<1,>=0.10.0 in ./.venv/lib/python3.13/site-packages (from openai<3.0.0,>=2.26.0->langchain-openai<2.0.0,>=1.1.7->langchain-xai) (0.13.0)
-Requirement already satisfied: sniffio in ./.venv/lib/python3.13/site-packages (from openai<3.0.0,>=2.26.0->langchain-openai<2.0.0,>=1.1.7->langchain-xai) (1.3.1)
-Requirement already satisfied: tqdm>4 in ./.venv/lib/python3.13/site-packages (from openai<3.0.0,>=2.26.0->langchain-openai<2.0.0,>=1.1.7->langchain-xai) (4.67.3)
-Requirement already satisfied: charset_normalizer<4,>=2 in ./.venv/lib/python3.13/site-packages (from requests>=2.0.0->langsmith<1.0.0,>=0.3.45->langchain-core) (3.4.4)
-Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.13/site-packages (from requests>=2.0.0->langsmith<1.0.0,>=0.3.45->langchain-core) (2.6.3)
-Requirement already satisfied: regex>=2022.1.18 in ./.venv/lib/python3.13/site-packages (from tiktoken<1.0.0,>=0.7.0->langchain-openai<2.0.0,>=1.1.7->langchain-xai) (2026.2.28)

docs/INDEX.md

@@ -3,6 +3,7 @@
 > **For AI Assistants**: This document contains all critical patterns, conventions, and best practices. Read this first to understand the codebase structure and ensure consistency.
 
 **Quick Navigation:**
+- [iii Platform & Development Workflow](#iii-platform--development-workflow) - Platform evolution and CLI tools
 - [Core Concepts](#core-concepts) - System architecture and patterns
 - [Design Principles](#design-principles) - Event Storm & Bidirectional References
 - [Step Development](#step-development-best-practices) - How to create new steps
@@ -23,6 +24,244 @@
 
 ---
 
+## iii Platform & Development Workflow
+
+### Platform Evolution (v0.8 → v0.9+)
+
+**Status:** March 2026 - iii v0.9+ production-ready
+
+iii has evolved from an all-in-one development tool to a **modular, production-grade event engine** with clear separation between development and deployment workflows.
+
+#### Structural Changes Overview
+
+| Component | Before (v0.2-v0.7) | Now (v0.9+) | Impact |
+|-----------|-------------------|-------------|--------|
+| **Console/Dashboard** | Integrated in engine process (port 3111) | Separate process (`iii-cli console` or `dev`) | More flexibility, less resource overhead, better scaling |
+| **CLI Tool** | Minimal or non-existent | `iii-cli` is the central dev tool | Terminal-based dev workflow, scriptable, faster iteration |
+| **Project Structure** | Steps anywhere in project | **Recommended:** `src/` + `src/steps/` | Cleaner structure, reliable hot-reload |
+| **Hot-Reload/Watcher** | Integrated in engine | Separate `shell::ExecModule` with `watch` paths | Only Python/TS files watched (configurable) |
+| **Start & Services** | Single `iii` process | Engine (`iii` or `iii-cli start`) + Console separate | Better for production (engine) vs dev (console) |
+| **Config Handling** | YAML + ENV | YAML + ENV + CLI flags prioritized | More control via CLI flags |
+| **Observability** | Basic | Enhanced (OTel, Rollups, Alerts, Traces) | Production-ready telemetry |
+| **Streams & State** | KV-Store (file/memory) | More adapters + file_based default | Better persistence handling |
+
+**Key Takeaway:** iii is now a **modular, production-ready engine** where development (CLI + separate console) is clearly separated from production deployment.
+
+---
+
+### Development Workflow with iii-cli
+
+**`iii-cli` is your primary tool for local development, debugging, and testing.**
+
+#### Essential Commands
+
+| Command | Purpose | When to Use | Example |
+|---------|---------|------------|---------|
+| `iii-cli dev` | Start dev server with hot-reload + integrated console | Local development, immediate feedback on code changes | `iii-cli dev` |
+| `iii-cli console` | Start dashboard only (separate port) | When you only need the console (no dev reload) | `iii-cli console --host 0.0.0.0 --port 3113` |
+| `iii-cli start` | Start engine standalone (like `motia.service`) | Testing engine in isolation | `iii-cli start -c iii-config.yaml` |
+| `iii-cli logs` | Live logs of all flows/workers/triggers | Debugging, error investigation | `iii-cli logs --level debug` |
+| `iii-cli trace <id>` | Show detailed trace information (OTel) | Debug specific request/flow | `iii-cli trace abc123` |
+| `iii-cli state ls` | List states (KV storage) | Verify state persistence | `iii-cli state ls` |
+| `iii-cli state get` | Get specific state value | Inspect state content | `iii-cli state get key` |
+| `iii-cli stream ls` | List all streams + groups | Inspect stream/websocket connections | `iii-cli stream ls` |
+| `iii-cli flow list` | Show all registered flows/triggers | Overview of active steps & endpoints | `iii-cli flow list` |
+| `iii-cli worker logs` | Worker logs (Python/TS execution) | Debug issues in step handlers | `iii-cli worker logs` |
+
+#### Typical Development Workflow
+
+```bash
+# 1. Navigate to project
+cd /opt/motia-iii/bitbylaw
+
+# 2. Start dev mode (hot-reload + console on port 3113)
+iii-cli dev --host 0.0.0.0 --port 3113 --engine-port 3111
+
+# Alternative: Separate engine + console
+# Terminal 1:
+iii-cli start -c iii-config.yaml
+
+# Terminal 2:
+iii-cli console --host 0.0.0.0 --port 3113 \
+  --engine-host 192.168.1.62 --engine-port 3111
+
+# 3. Watch logs live (separate terminal)
+iii-cli logs -f
+
+# 4. Debug specific trace
+iii-cli trace <trace-id-from-logs>
+
+# 5. Inspect state
+iii-cli state ls
+iii-cli state get document:sync:status
+
+# 6. Verify flows registered
+iii-cli flow list
+```
+
+#### Development vs. Production
+
+**Development:**
+- Use `iii-cli dev` for hot-reload
+- Console accessible on localhost:3113
+- Logs visible in terminal
+- Immediate feedback on code changes
+
+**Production:**
+- `systemd` service runs `iii-cli start`
+- Console runs separately (if needed)
+- Logs via `journalctl -u motia.service -f`
+- No hot-reload (restart service for changes)
+
+**Example Production Service:**
+```ini
+[Unit]
+Description=Motia III Engine
+After=network.target redis.service
+
+[Service]
+Type=simple
+User=motia
+WorkingDirectory=/opt/motia-iii/bitbylaw
+ExecStart=/usr/local/bin/iii-cli start -c /opt/motia-iii/bitbylaw/iii-config.yaml
+Restart=always
+RestartSec=10
+Environment="PATH=/usr/local/bin:/usr/bin"
+
+[Install]
+WantedBy=multi-user.target
+```
+
+#### Project Structure Best Practices
+
+**Recommended Structure (v0.9+):**
+```
+bitbylaw/
+├── iii-config.yaml           # Main configuration
+├── src/                      # Source code root
+│   └── steps/                # All steps here (hot-reload reliable)
+│       ├── __init__.py
+│       ├── vmh/
+│       │   ├── __init__.py
+│       │   ├── document_sync_event_step.py
+│       │   └── webhook/
+│       │       ├── __init__.py
+│       │       └── document_create_api_step.py
+│       └── advoware_proxy/
+│           └── ...
+├── services/                 # Shared business logic
+│   ├── __init__.py
+│   ├── xai_service.py
+│   ├── espocrm.py
+│   └── ...
+└── tests/                    # Test files
+```
+
+**Why `src/steps/` is recommended:**
+- **Hot-reload works reliably** - Watcher detects changes correctly
+- **Cleaner project** - Source code isolated from config/docs
+- **IDE support** - Better navigation and refactoring
+- **Deployment** - Easier to package
+
+**Note:** Old structure (steps in root) still works, but hot-reload may be less reliable.
+
+#### Hot-Reload Configuration
+
+**Hot-reload is configured via `shell::ExecModule` in `iii-config.yaml`:**
+
+```yaml
+modules:
+  - type: shell::ExecModule
+    config:
+      watch:
+        - "src/**/*.py"      # Watch Python files in src/
+        - "services/**/*.py" # Watch service files
+        # Add more patterns as needed
+      ignore:
+        - "**/__pycache__/**"
+        - "**/*.pyc"
+        - "**/tests/**"
+```
+
+**Behavior:**
+- Only files matching `watch` patterns trigger reload
+- Changes in `ignore` patterns are ignored
+- Reload is automatic in `iii-cli dev` mode
+- Production mode (`iii-cli start`) does NOT watch files
+
+---
+
+### Observability & Debugging
+
+#### OpenTelemetry Integration
+
+**iii v0.9+ has built-in OpenTelemetry support:**
+
+```python
+# Traces are automatically created for:
+# - HTTP requests
+# - Queue processing
+# - Cron execution
+# - Service calls (if instrumented)
+
+# Access trace ID in handler:
+async def handler(request: ApiRequest, ctx: FlowContext) -> ApiResponse:
+    trace_id = ctx.trace_id  # Use for debugging
+    ctx.logger.info(f"Trace ID: {trace_id}")
+```
+
+**View traces:**
+```bash
+# Get trace details
+iii-cli trace <trace-id>
+
+# Filter logs by trace
+iii-cli logs --trace <trace-id>
+```
+
+#### Debugging Workflow
+
+**1. Live Logs:**
+```bash
+# All logs
+iii-cli logs -f
+
+# Specific level
+iii-cli logs --level error
+
+# With grep
+iii-cli logs -f | grep "document_sync"
+```
+
+**2. State Inspection:**
+```bash
+# List all state keys
+iii-cli state ls
+
+# Get specific state
+iii-cli state get sync:document:last_run
+```
+
+**3. Flow Verification:**
+```bash
+# List all registered flows
+iii-cli flow list
+
+# Verify endpoint exists
+iii-cli flow list | grep "/vmh/webhook"
+```
+
+**4. Worker Issues:**
+```bash
+# Worker-specific logs
+iii-cli worker logs
+
+# Check worker health
+iii-cli worker status
+```
+
+---
+
 ## Core Concepts
 
 ### System Overview
@@ -1271,24 +1510,41 @@ sudo systemctl enable motia.service
 sudo systemctl enable iii-console.service
 ```
 
-**Manual (Development):**
+**Development (iii-cli):**
 ```bash
-# Start iii Engine
+# Option 1: Dev mode with integrated console and hot-reload
 cd /opt/motia-iii/bitbylaw
-/opt/bin/iii -c iii-config.yaml
+iii-cli dev --host 0.0.0.0 --port 3113 --engine-port 3111
 
-# Start iii Console (Web UI)
+# Option 2: Separate engine and console
-/opt/bin/iii-console --enable-flow --host 0.0.0.0 --port 3113 \
+# Terminal 1: Start engine
-  --engine-host 192.168.67.233 --engine-port 3111 --ws-port 3114
+iii-cli start -c iii-config.yaml
+
+# Terminal 2: Start console
+iii-cli console --host 0.0.0.0 --port 3113 \
+  --engine-host 192.168.1.62 --engine-port 3111
+
+# Option 3: Manual (legacy)
+/opt/bin/iii -c iii-config.yaml
 ```
 
 ### Check Registered Steps
 
+**Using iii-cli (recommended):**
+```bash
+# List all flows and triggers
+iii-cli flow list
+
+# Filter for specific step
+iii-cli flow list | grep document_sync
+```
+
+**Using curl (legacy):**
 ```bash
 curl http://localhost:3111/_console/functions | python3 -m json.tool
 ```
 
-### Test HTTP Endpoint
+### Test HTTP Endpoints
 
 ```bash
 # Test document webhook
@@ -1298,6 +1554,11 @@ curl -X POST "http://localhost:3111/vmh/webhook/document/create" \
 
 # Test advoware proxy
 curl "http://localhost:3111/advoware/proxy?endpoint=employees"
+
+# Test beteiligte sync
+curl -X POST "http://localhost:3111/vmh/webhook/beteiligte/create" \
+  -H "Content-Type: application/json" \
+  -d '{"entity_type": "CBeteiligte", "entity_id": "abc123", "action": "create"}'
 ```
 
 ### Manually Trigger Cron
@@ -1308,36 +1569,208 @@ curl -X POST "http://localhost:3111/_console/cron/trigger" \
   -d '{"function_id": "steps::VMH Beteiligte Sync Cron::trigger::0"}'
 ```
 
-### View Logs
+### View and Debug Logs
 
+**Using iii-cli (recommended):**
 ```bash
-# Live logs via journalctl
+# Live logs (all)
-journalctl -u motia-iii -f
+iii-cli logs -f
+
+# Live logs with specific level
+iii-cli logs -f --level error
+iii-cli logs -f --level debug
+
+# Filter by component
+iii-cli logs -f | grep "document_sync"
+
+# Worker-specific logs
+iii-cli worker logs
+
+# Get specific trace
+iii-cli trace <trace-id>
+
+# Filter logs by trace ID
+iii-cli logs --trace <trace-id>
+```
+
+**Using journalctl (production):**
+```bash
+# Live logs
+journalctl -u motia.service -f
 
 # Search for specific step
-journalctl --since "today" | grep -i "document sync"
+journalctl -u motia.service --since "today" | grep -i "document sync"
 
+# Show errors only
+journalctl -u motia.service -p err -f
+
+# Last 100 lines
+journalctl -u motia.service -n 100
+
+# Specific time range
+journalctl -u motia.service --since "2026-03-19 10:00" --until "2026-03-19 11:00"
+```
+
+**Using log files (legacy):**
+```bash
 # Check for errors
 tail -100 /opt/motia-iii/bitbylaw/iii_new.log | grep -i error
+
+# Follow log file
+tail -f /opt/motia-iii/bitbylaw/iii_new.log
+```
+
+### Inspect State and Streams
+
+**State Management:**
+```bash
+# List all state keys
+iii-cli state ls
+
+# Get specific state value
+iii-cli state get document:sync:last_run
+
+# Set state (if needed for testing)
+iii-cli state set test:key "test value"
+
+# Delete state
+iii-cli state delete test:key
+```
+
+**Stream Management:**
+```bash
+# List all active streams
+iii-cli stream ls
+
+# Inspect specific stream
+iii-cli stream info <stream-id>
+
+# List consumer groups
+iii-cli stream groups <stream-name>
+```
+
+### Debugging Workflow
+
+**1. Identify the Issue:**
+```bash
+# Check if step is registered
+iii-cli flow list | grep my_step
+
+# View recent errors
+iii-cli logs --level error -n 50
+
+# Check service status
+sudo systemctl status motia.service
+```
+
+**2. Get Detailed Information:**
+```bash
+# Live tail logs for specific step
+iii-cli logs -f | grep "document_sync"
+
+# Check worker processes
+iii-cli worker logs
+
+# Inspect state
+iii-cli state ls
+```
+
+**3. Test Specific Functionality:**
+```bash
+# Trigger webhook manually
+curl -X POST http://localhost:3111/vmh/webhook/...
+
+# Check response and logs
+iii-cli logs -f | grep "webhook"
+
+# Verify state changed
+iii-cli state get entity:sync:status
+```
+
+**4. Trace Specific Request:**
+```bash
+# Make request, note trace ID from logs
+curl -X POST http://localhost:3111/vmh/webhook/document/create ...
+
+# Get full trace
+iii-cli trace <trace-id>
+
+# View all logs for this trace
+iii-cli logs --trace <trace-id>
+```
+
+### Performance Monitoring
+
+**Check System Resources:**
+```bash
+# CPU and memory usage
+htop
+
+# Process-specific
+ps aux | grep iii
+
+# Redis memory
+redis-cli info memory
+
+# File descriptors
+lsof -p $(pgrep -f "iii-cli start")
+```
+
+**Check Processing Metrics:**
+```bash
+# Queue lengths (if using Redis streams)
+redis-cli XINFO STREAM vmh:document:sync
+
+# Pending messages
+redis-cli XPENDING vmh:document:sync group1
+
+# Lock status
+redis-cli KEYS "lock:*"
 ```
 
 ### Common Issues
 
 **Step not showing up:**
 1. Check file naming: Must end with `_step.py`
-2. Check for import errors: `grep -i "importerror\|traceback" iii.log`
+2. Check for syntax errors: `iii-cli logs --level error`
-3. Verify `config` dict is present
+3. Check for import errors: `iii-cli logs | grep -i "importerror\|traceback"`
-4. Restart iii engine
+4. Verify `config` dict is present
+5. Restart: `sudo systemctl restart motia.service` or restart `iii-cli dev`
+6. Verify hot-reload working: Check terminal output in `iii-cli dev`
 
 **Redis connection failed:**
 - Check `REDIS_HOST` and `REDIS_PORT` environment variables
 - Verify Redis is running: `redis-cli ping`
+- Check Redis logs: `journalctl -u redis -f`
 - Service will work without Redis but with warnings
 
+**Hot-reload not working:**
+- Verify using `iii-cli dev` (not `iii-cli start`)
+- Check `watch` patterns in `iii-config.yaml`
+- Ensure files are in watched directories (`src/**/*.py`)
+- Look for watcher errors: `iii-cli logs | grep -i "watch"`
+
+**Handler not triggered:**
+- Verify endpoint registered: `iii-cli flow list`
+- Check HTTP method matches (GET, POST, etc.)
+- Test with curl to isolate issue
+- Check trigger configuration in step's `config` dict
+
 **AttributeError '_log' not found:**
 - Ensure service inherits from `BaseSyncUtils` OR
 - Implement `_log()` method manually
 
+**Trace not found:**
+- Ensure OpenTelemetry enabled in config
+- Check if trace ID is valid format
+- Use `iii-cli logs` with filters instead
+
+**Console not accessible:**
+- Check if console service running: `systemctl status iii-console.service`
+- Verify port not blocked by firewall: `sudo ufw status`
+- Check console logs: `journalctl -u iii-console.service -f`
+- Try accessing via `localhost:3113` instead of public IP
+
 ---
 
 ## Key Patterns Summary

iii-config.yaml

@@ -78,6 +78,6 @@ modules:
   - class: modules::shell::ExecModule
     config:
       watch:
-        - steps/**/*.py
+        - src/steps/**/*.py
       exec:
-        - /opt/bin/uv run python -m motia.cli run --dir steps
+        - /usr/local/bin/uv run python -m motia.cli run --dir src/steps

services/aiknowledge_sync_utils.py

@@ -12,6 +12,7 @@ import hashlib
 import json
 from typing import Dict, Any, Optional, List, Tuple
 from datetime import datetime
+from urllib.parse import unquote
 
 from services.sync_utils_base import BaseSyncUtils
 from services.models import (

services/langchain_xai_service.py

@@ -17,7 +17,7 @@ class LangChainXAIService:
     
     Usage:
         service = LangChainXAIService(ctx)
-        model = service.get_chat_model(model="grok-2-latest")
+        model = service.get_chat_model(model="grok-4-1-fast-reasoning")
         model_with_tools = service.bind_file_search(model, collection_id)
         result = await service.invoke_chat(model_with_tools, messages)
     """
@@ -46,7 +46,7 @@ class LangChainXAIService:
     
     def get_chat_model(
         self,
-        model: str = "grok-2-latest",
+        model: str = "grok-4-1-fast-reasoning",
         temperature: float = 0.7,
         max_tokens: Optional[int] = None
     ):
@@ -54,7 +54,7 @@ class LangChainXAIService:
         Initialisiert ChatXAI Model.
         
         Args:
-            model: Model name (default: grok-2-latest)
+            model: Model name (default: grok-4-1-fast-reasoning)
             temperature: Sampling temperature 0.0-1.0
             max_tokens: Optional max tokens for response
             
@@ -84,6 +84,72 @@ class LangChainXAIService:
         
         return ChatXAI(**kwargs)
     
+    def bind_tools(
+        self,
+        model,
+        collection_id: Optional[str] = None,
+        enable_web_search: bool = False,
+        web_search_config: Optional[Dict[str, Any]] = None,
+        max_num_results: int = 10
+    ):
+        """
+        Bindet xAI Tools (file_search und/oder web_search) an Model.
+        
+        Args:
+            model: ChatXAI model instance
+            collection_id: Optional xAI Collection ID für file_search
+            enable_web_search: Enable web search tool (default: False)
+            web_search_config: Optional web search configuration:
+                {
+                    'allowed_domains': ['example.com'],  # Max 5 domains
+                    'excluded_domains': ['spam.com'],    # Max 5 domains
+                    'enable_image_understanding': True
+                }
+            max_num_results: Max results from file search (default: 10)
+            
+        Returns:
+            Model with requested tools bound (file_search and/or web_search)
+        """
+        tools = []
+        
+        # Add file_search tool if collection_id provided
+        if collection_id:
+            self._log(f"🔍 Binding file_search: collection={collection_id}")
+            tools.append({
+                "type": "file_search",
+                "vector_store_ids": [collection_id],
+                "max_num_results": max_num_results
+            })
+        
+        # Add web_search tool if enabled
+        if enable_web_search:
+            self._log("🌐 Binding web_search")
+            web_search_tool = {"type": "web_search"}
+            
+            # Add optional web search filters
+            if web_search_config:
+                if 'allowed_domains' in web_search_config:
+                    domains = web_search_config['allowed_domains'][:5]  # Max 5
+                    web_search_tool['filters'] = {'allowed_domains': domains}
+                    self._log(f"   Allowed domains: {domains}")
+                elif 'excluded_domains' in web_search_config:
+                    domains = web_search_config['excluded_domains'][:5]  # Max 5
+                    web_search_tool['filters'] = {'excluded_domains': domains}
+                    self._log(f"   Excluded domains: {domains}")
+                
+                if web_search_config.get('enable_image_understanding'):
+                    web_search_tool['enable_image_understanding'] = True
+                    self._log("   Image understanding: enabled")
+            
+            tools.append(web_search_tool)
+        
+        if not tools:
+            self._log("⚠️  No tools to bind (no collection_id and web_search disabled)", level='warn')
+            return model
+        
+        self._log(f"🔧 Binding {len(tools)} tool(s) to model")
+        return model.bind_tools(tools)
+    
     def bind_file_search(
         self,
         model,
@@ -91,25 +157,15 @@ class LangChainXAIService:
         max_num_results: int = 10
     ):
         """
-        Bindet xAI file_search Tool an Model.
+        Legacy method: Bindet nur file_search Tool an Model.
         
-        Args:
+        Use bind_tools() for more flexibility.
-            model: ChatXAI model instance
-            collection_id: xAI Collection ID (vector store)
-            max_num_results: Max results from file search (default: 10)
-            
-        Returns:
-            Model with bound file_search tool
         """
-        self._log(f"🔍 Binding file_search: collection={collection_id}, max_results={max_num_results}")
+        return self.bind_tools(
-        
+            model=model,
-        tools = [{
+            collection_id=collection_id,
-            "type": "file_search",
+            max_num_results=max_num_results
-            "vector_store_ids": [collection_id],
+        )
-            "max_num_results": max_num_results
-        }]
-        
-        return model.bind_tools(tools)
     
     async def invoke_chat(
         self,

services/xai_service.py

@@ -63,14 +63,31 @@ class XAIService:
         Raises:
             RuntimeError: bei HTTP-Fehler oder fehlendem file_id in der Antwort
         """
-        self._log(f"📤 Uploading {len(file_content)} bytes to xAI: {filename}")
+        # Normalize MIME type: xAI needs correct Content-Type for proper processing
+        # If generic octet-stream but file is clearly a PDF, fix it
+        if mime_type == 'application/octet-stream' and filename.lower().endswith('.pdf'):
+            mime_type = 'application/pdf'
+            self._log(f"⚠️  Corrected MIME type to application/pdf for {filename}")
+        
+        self._log(f"📤 Uploading {len(file_content)} bytes to xAI: {filename} ({mime_type})")
 
         session = await self._get_session()
         url = f"{XAI_FILES_URL}/v1/files"
         headers = {"Authorization": f"Bearer {self.api_key}"}
 
-        form = aiohttp.FormData()
+        # Create multipart form with explicit UTF-8 filename encoding
-        form.add_field('file', file_content, filename=filename, content_type=mime_type)
+        # aiohttp automatically URL-encodes filenames with special chars,
+        # but xAI expects raw UTF-8 in the filename parameter
+        form = aiohttp.FormData(quote_fields=False)
+        form.add_field(
+            'file',
+            file_content,
+            filename=filename,
+            content_type=mime_type
+        )
+        # CRITICAL: purpose="file_search" enables proper PDF processing
+        # Without this, xAI throws "internal error" on complex PDFs
+        form.add_field('purpose', 'file_search')
 
         async with session.post(url, data=form, headers=headers) as response:
             try:
@@ -95,9 +112,12 @@ class XAIService:
 
     async def add_to_collection(self, collection_id: str, file_id: str) -> None:
         """
-        Fügt eine Datei einer xAI-Collection hinzu.
+        Fügt eine Datei einer xAI-Collection (Vector Store) hinzu.
 
-        POST https://management-api.x.ai/v1/collections/{collection_id}/documents/{file_id}
+        POST https://api.x.ai/v1/vector_stores/{vector_store_id}/files
+
+        Uses the OpenAI-compatible API pattern for adding files to vector stores.
+        This triggers proper indexing and processing.
 
         Raises:
             RuntimeError: bei HTTP-Fehler
@@ -105,13 +125,16 @@ class XAIService:
         self._log(f"📚 Adding file {file_id} to collection {collection_id}")
 
         session = await self._get_session()
-        url = f"{XAI_MANAGEMENT_URL}/v1/collections/{collection_id}/documents/{file_id}"
+        # Use the OpenAI-compatible endpoint (not management API)
+        url = f"{XAI_FILES_URL}/v1/vector_stores/{collection_id}/files"
         headers = {
-            "Authorization": f"Bearer {self.management_key}",
+            "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json",
         }
 
-        async with session.post(url, headers=headers) as response:
+        payload = {"file_id": file_id}
+
+        async with session.post(url, json=payload, headers=headers) as response:
             if response.status not in (200, 201):
                 raw = await response.text()
                 raise RuntimeError(

steps/vmh/xai_chat_completion_api_step.py

@@ -1,439 +0,0 @@
-"""VMH xAI Chat Completions API
-
-OpenAI-kompatible Chat Completions API mit xAI/LangChain Backend.
-Unterstützt file_search über xAI Collections (RAG).
-"""
-import json
-import time
-from typing import Any, Dict, List, Optional
-from motia import FlowContext, http, ApiRequest, ApiResponse
-
-
-config = {
-    "name": "VMH xAI Chat Completions API",
-    "description": "OpenAI-compatible Chat Completions API with xAI LangChain backend",
-    "flows": ["vmh-chat"],
-    "triggers": [
-        http("POST", "/vmh/v1/chat/completions")
-    ],
-}
-
-
-async def handler(request: ApiRequest, ctx: FlowContext[Any]) -> ApiResponse:
-    """
-    OpenAI-compatible Chat Completions endpoint.
-    
-    Request Body (OpenAI format):
-        {
-            "model": "grok-2-latest",
-            "messages": [
-                {"role": "system", "content": "You are helpful"},
-                {"role": "user", "content": "1234/56 Was ist der Stand?"}
-            ],
-            "temperature": 0.7,
-            "max_tokens": 2000,
-            "stream": false,
-            "extra_body": {
-                "collection_id": "col_abc123"  // Optional: override auto-detection
-            }
-        }
-    
-    Aktenzeichen-Erkennung (Priority):
-        1. extra_body.collection_id (explicit override)
-        2. First user message starts with Aktenzeichen (e.g., "1234/56 ...")
-        3. Error 400 if no collection_id found (strict mode)
-    
-    Response (OpenAI format):
-        Non-Streaming:
-            {
-                "id": "chatcmpl-...",
-                "object": "chat.completion",
-                "created": 1234567890,
-                "model": "grok-2-latest",
-                "choices": [{
-                    "index": 0,
-                    "message": {"role": "assistant", "content": "..."},
-                    "finish_reason": "stop"
-                }],
-                "usage": {"prompt_tokens": X, "completion_tokens": Y, "total_tokens": Z}
-            }
-        
-        Streaming (SSE):
-            data: {"id":"chatcmpl-...","choices":[{"delta":{"content":"Hello"},...}]}
-            data: {"id":"chatcmpl-...","choices":[{"delta":{"content":" world"},...}]}
-            data: {"choices":[{"delta":{},"finish_reason":"stop"}]}
-            data: [DONE]
-    """
-    from services.langchain_xai_service import LangChainXAIService
-    from services.aktenzeichen_utils import extract_aktenzeichen, normalize_aktenzeichen
-    from services.espocrm import EspoCRMAPI
-    
-    ctx.logger.info("=" * 80)
-    ctx.logger.info("💬 VMH CHAT COMPLETIONS API")
-    ctx.logger.info("=" * 80)
-    
-    try:
-        # Parse request body
-        body = request.body or {}
-        
-        if not isinstance(body, dict):
-            ctx.logger.error(f"❌ Invalid request body type: {type(body)}")
-            return ApiResponse(
-                status=400,
-                body={'error': 'Request body must be JSON object'}
-            )
-        
-        # Extract parameters
-        model_name = body.get('model', 'grok-4-1-fast-reasoning')
-        messages = body.get('messages', [])
-        temperature = body.get('temperature', 0.7)
-        max_tokens = body.get('max_tokens')
-        stream = body.get('stream', False)
-        extra_body = body.get('extra_body', {})
-        
-        ctx.logger.info(f"📋 Model: {model_name}")
-        ctx.logger.info(f"📋 Messages: {len(messages)}")
-        ctx.logger.info(f"📋 Stream: {stream}")
-        ctx.logger.debug(f"Messages: {json.dumps(messages, indent=2, ensure_ascii=False)}")
-        
-        # Validate messages
-        if not messages or not isinstance(messages, list):
-            ctx.logger.error("❌ Missing or invalid messages array")
-            return ApiResponse(
-                status=400,
-                body={'error': 'messages must be non-empty array'}
-            )
-        
-        # Determine collection_id (Priority: extra_body > Aktenzeichen > error)
-        collection_id: Optional[str] = None
-        aktenzeichen: Optional[str] = None
-        
-        # Priority 1: Explicit collection_id in extra_body
-        if 'collection_id' in extra_body:
-            collection_id = extra_body['collection_id']
-            ctx.logger.info(f"🔍 Collection ID from extra_body: {collection_id}")
-        
-        # Priority 2: Extract Aktenzeichen from first user message
-        else:
-            for msg in messages:
-                if msg.get('role') == 'user':
-                    content = msg.get('content', '')
-                    aktenzeichen_raw = extract_aktenzeichen(content)
-                    
-                    if aktenzeichen_raw:
-                        aktenzeichen = normalize_aktenzeichen(aktenzeichen_raw)
-                        ctx.logger.info(f"🔍 Aktenzeichen detected: {aktenzeichen}")
-                        
-                        # Lookup collection_id via EspoCRM
-                        collection_id = await lookup_collection_by_aktenzeichen(
-                            aktenzeichen, ctx
-                        )
-                        
-                        if collection_id:
-                            ctx.logger.info(f"✅ Collection found: {collection_id}")
-                            
-                            # Remove Aktenzeichen from message (clean prompt)
-                            from services.aktenzeichen_utils import remove_aktenzeichen
-                            msg['content'] = remove_aktenzeichen(content)
-                            ctx.logger.debug(f"Cleaned message: {msg['content']}")
-                        else:
-                            ctx.logger.warn(f"⚠️  No collection found for {aktenzeichen}")
-                    
-                    break  # Only check first user message
-        
-        # Priority 3: Error if no collection_id (strict mode)
-        if not collection_id:
-            ctx.logger.error("❌ No collection_id found (neither extra_body nor Aktenzeichen)")
-            ctx.logger.error("   Provide collection_id in extra_body or start message with Aktenzeichen")
-            return ApiResponse(
-                status=400,
-                body={
-                    'error': 'collection_id required',
-                    'message': 'Provide collection_id in extra_body or start message with Aktenzeichen (e.g., "1234/56 question")'
-                }
-            )
-        
-        # Initialize LangChain xAI Service
-        try:
-            langchain_service = LangChainXAIService(ctx)
-        except ValueError as e:
-            ctx.logger.error(f"❌ Service initialization failed: {e}")
-            return ApiResponse(
-                status=500,
-                body={'error': 'Service configuration error', 'details': str(e)}
-            )
-        
-        # Create ChatXAI model
-        model = langchain_service.get_chat_model(
-            model=model_name,
-            temperature=temperature,
-            max_tokens=max_tokens
-        )
-        
-        # Bind file_search tool
-        model_with_tools = langchain_service.bind_file_search(
-            model=model,
-            collection_id=collection_id,
-            max_num_results=10
-        )
-        
-        # Generate completion_id
-        completion_id = f"chatcmpl-{ctx.traceId[:12]}" if hasattr(ctx, 'traceId') else f"chatcmpl-{int(time.time())}"
-        created_ts = int(time.time())
-        
-        # Branch: Streaming vs Non-Streaming
-        if stream:
-            ctx.logger.info("🌊 Starting streaming response...")
-            return await handle_streaming_response(
-                model_with_tools=model_with_tools,
-                messages=messages,
-                completion_id=completion_id,
-                created_ts=created_ts,
-                model_name=model_name,
-                langchain_service=langchain_service,
-                ctx=ctx
-            )
-        else:
-            ctx.logger.info("📦 Starting non-streaming response...")
-            return await handle_non_streaming_response(
-                model_with_tools=model_with_tools,
-                messages=messages,
-                completion_id=completion_id,
-                created_ts=created_ts,
-                model_name=model_name,
-                langchain_service=langchain_service,
-                ctx=ctx
-            )
-    
-    except Exception as e:
-        ctx.logger.error("=" * 80)
-        ctx.logger.error("❌ ERROR: CHAT COMPLETIONS API")
-        ctx.logger.error("=" * 80)
-        ctx.logger.error(f"Error: {e}", exc_info=True)
-        ctx.logger.error(f"Request body: {json.dumps(request.body, indent=2, ensure_ascii=False)}")
-        ctx.logger.error("=" * 80)
-        
-        return ApiResponse(
-            status=500,
-            body={
-                'error': 'Internal server error',
-                'message': str(e)
-            }
-        )
-
-
-async def handle_non_streaming_response(
-    model_with_tools,
-    messages: List[Dict[str, Any]],
-    completion_id: str,
-    created_ts: int,
-    model_name: str,
-    langchain_service,
-    ctx: FlowContext
-) -> ApiResponse:
-    """
-    Handle non-streaming chat completion.
-    
-    Returns:
-        ApiResponse with OpenAI-format JSON body
-    """
-    try:
-        # Invoke model
-        result = await langchain_service.invoke_chat(model_with_tools, messages)
-        
-        # Extract content
-        content = result.content if hasattr(result, 'content') else str(result)
-        
-        # Build OpenAI-compatible response
-        response_body = {
-            'id': completion_id,
-            'object': 'chat.completion',
-            'created': created_ts,
-            'model': model_name,
-            'choices': [{
-                'index': 0,
-                'message': {
-                    'role': 'assistant',
-                    'content': content
-                },
-                'finish_reason': 'stop'
-            }],
-            'usage': {
-                'prompt_tokens': 0,  # LangChain doesn't expose token counts easily
-                'completion_tokens': 0,
-                'total_tokens': 0
-            }
-        }
-        
-        # Log token usage (if available)
-        if hasattr(result, 'usage_metadata'):
-            usage = result.usage_metadata
-            prompt_tokens = getattr(usage, 'input_tokens', 0)
-            completion_tokens = getattr(usage, 'output_tokens', 0)
-            response_body['usage'] = {
-                'prompt_tokens': prompt_tokens,
-                'completion_tokens': completion_tokens,
-                'total_tokens': prompt_tokens + completion_tokens
-            }
-            ctx.logger.info(f"📊 Token Usage: prompt={prompt_tokens}, completion={completion_tokens}")
-        
-        ctx.logger.info(f"✅ Chat completion: {len(content)} chars")
-        ctx.logger.info("=" * 80)
-        
-        return ApiResponse(
-            status=200,
-            body=response_body
-        )
-    
-    except Exception as e:
-        ctx.logger.error(f"❌ Non-streaming completion failed: {e}", exc_info=True)
-        raise
-
-
-async def handle_streaming_response(
-    model_with_tools,
-    messages: List[Dict[str, Any]],
-    completion_id: str,
-    created_ts: int,
-    model_name: str,
-    langchain_service,
-    ctx: FlowContext
-):
-    """
-    Handle streaming chat completion via SSE.
-    
-    Returns:
-        Streaming response generator
-    """
-    async def stream_generator():
-        try:
-            # Set SSE headers
-            await ctx.response.status(200)
-            await ctx.response.headers({
-                "Content-Type": "text/event-stream",
-                "Cache-Control": "no-cache",
-                "Connection": "keep-alive"
-            })
-            
-            ctx.logger.info("🌊 Streaming started")
-            
-            # Stream chunks
-            chunk_count = 0
-            total_content = ""
-            
-            async for chunk in langchain_service.astream_chat(model_with_tools, messages):
-                # Extract delta content
-                delta = chunk.content if hasattr(chunk, "content") else ""
-                
-                if delta:
-                    total_content += delta
-                    chunk_count += 1
-                    
-                    # Build SSE data
-                    data = {
-                        "id": completion_id,
-                        "object": "chat.completion.chunk",
-                        "created": created_ts,
-                        "model": model_name,
-                        "choices": [{
-                            "index": 0,
-                            "delta": {"content": delta},
-                            "finish_reason": None
-                        }]
-                    }
-                    
-                    # Send SSE event
-                    await ctx.response.stream(f"data: {json.dumps(data, ensure_ascii=False)}\n\n")
-            
-            # Send finish event
-            finish_data = {
-                "id": completion_id,
-                "object": "chat.completion.chunk",
-                "created": created_ts,
-                "model": model_name,
-                "choices": [{
-                    "index": 0,
-                    "delta": {},
-                    "finish_reason": "stop"
-                }]
-            }
-            await ctx.response.stream(f"data: {json.dumps(finish_data)}\n\n")
-            
-            # Send [DONE]
-            await ctx.response.stream("data: [DONE]\n\n")
-            
-            # Close stream
-            await ctx.response.close()
-            
-            ctx.logger.info(f"✅ Streaming completed: {chunk_count} chunks, {len(total_content)} chars")
-            ctx.logger.info("=" * 80)
-        
-        except Exception as e:
-            ctx.logger.error(f"❌ Streaming failed: {e}", exc_info=True)
-            
-            # Send error event
-            error_data = {
-                "error": {
-                    "message": str(e),
-                    "type": "server_error"
-                }
-            }
-            await ctx.response.stream(f"data: {json.dumps(error_data)}\n\n")
-            await ctx.response.close()
-    
-    return stream_generator()
-
-
-async def lookup_collection_by_aktenzeichen(
-    aktenzeichen: str,
-    ctx: FlowContext
-) -> Optional[str]:
-    """
-    Lookup xAI Collection ID for Aktenzeichen via EspoCRM.
-    
-    Search strategy:
-        1. Search for Raeumungsklage with matching advowareAkteBezeichner
-        2. Return xaiCollectionId if found
-    
-    Args:
-        aktenzeichen: Normalized Aktenzeichen (e.g., "1234/56")
-        ctx: Motia context
-        
-    Returns:
-        Collection ID or None if not found
-    """
-    try:
-        # Initialize EspoCRM API
-        espocrm = EspoCRMAPI(ctx)
-        
-        # Search Räumungsklage by advowareAkteBezeichner
-        ctx.logger.info(f"🔍 Searching Räumungsklage for Aktenzeichen: {aktenzeichen}")
-        
-        search_result = await espocrm.search_entities(
-            entity_type='Raeumungsklage',
-            where=[{
-                'type': 'equals',
-                'attribute': 'advowareAkteBezeichner',
-                'value': aktenzeichen
-            }],
-            select=['id', 'xaiCollectionId', 'advowareAkteBezeichner'],
-            maxSize=1
-        )
-        
-        if search_result and len(search_result) > 0:
-            entity = search_result[0]
-            collection_id = entity.get('xaiCollectionId')
-            
-            if collection_id:
-                ctx.logger.info(f"✅ Found Räumungsklage: {entity.get('id')}")
-                return collection_id
-            else:
-                ctx.logger.warn(f"⚠️  Räumungsklage found but no xaiCollectionId: {entity.get('id')}")
-        else:
-            ctx.logger.warn(f"⚠️  No Räumungsklage found for {aktenzeichen}")
-        
-        return None
-    
-    except Exception as e:
-        ctx.logger.error(f"❌ Collection lookup failed: {e}", exc_info=True)
-        return None