import asyncio import logging from fastapi import WebSocket import uuid from backend.agent.react_agent import ReActAgent from backend.tools.tool_registry import TOOL_REGISTRY get_assistant_name = lambda: __import__('modules.assistant_identity', fromlist=['get_assistant_name']).get_assistant_name() def get_memory_client(): from backend.memory.episodic_memory import EpisodicMemory import os, sys db_dir = os.path.join(os.environ.get('APPDATA', os.path.expanduser('~')), 'JARVIS_OS') if getattr(sys, 'frozen', False) else os.path.join(os.getcwd(), 'chroma_db') os.makedirs(db_dir, exist_ok=True) return EpisodicMemory(db_dir) _active_agent_tasks = 0 _jarvis_auto_switched = False def is_agent_busy() -> bool: return _active_agent_tasks > 0 class ConnectionManager: def __init__(self): self.restricted_mode = False self.active_connections: list[WebSocket] = [] from backend.agent.react_agent import Tool self.tools = [] for name, func in TOOL_REGISTRY.items(): desc = func.__doc__.strip() if func.__doc__ else f"Executes {name}" self.tools.append(Tool( name=name, description=desc, parameters={"type": "object", "properties": {}, "required": []}, handler=func )) self.memory = get_memory_client() async def connect(self, websocket: WebSocket): await websocket.accept() self.active_connections.append(websocket) logging.info("Client connected to Agent WebSocket.") await websocket.send_json({ "event": "backend:ready", "payload": {} }) def disconnect(self, websocket: WebSocket): if websocket in self.active_connections: self.active_connections.remove(websocket) logging.info("Client disconnected from Agent WebSocket.") async def broadcast(self, message: dict): for connection in self.active_connections: try: await connection.send_json(message) except Exception as e: logging.error(f"Error broadcasting to client: {e}") async def handle_client_event(self, websocket: WebSocket | None, data: dict): global _jarvis_auto_switched # ── WAKE WORD → PERSONA SWITCH (Cloud + Local path) ────────────────────── # Mobile app / local PC sends: {"event": "voice:wake_word", "payload": {"agent": "hey jarvis"}} # The server auto-switches the active persona and broadcasts UI + TTS confirmation. if data.get("event") == "voice:wake_word": from modules.assistant_identity import set_mode, get_mode, get_assistant_name as _get_name keyword = (data.get("payload", {}).get("agent", "") or "").lower() if "friday" in keyword: target_persona = "friday" elif "jarvis" in keyword: target_persona = "jarvis" else: target_persona = None if target_persona: _jarvis_auto_switched = False current = get_mode() if current != target_persona: set_mode(target_persona) logging.info(f"[WakeWord] Persona switched: {current.upper()} → {target_persona.upper()} via wake word '{keyword}'") else: logging.info(f"[WakeWord] Persona already {target_persona.upper()} — no switch needed.") name = _get_name() # Broadcast UI theme swap to all clients (Tauri EXE + APK) await self.broadcast({ "event": "agent:switched_persona", "payload": { "persona": target_persona, "trigger": "wake_word", "keyword": keyword } }) # Broadcast wake confirmation so frontend can show "listening" UI await self.broadcast({ "event": "voice:wake_word", "payload": {"agent": target_persona, "name": name} }) return if data.get("event") == "xr:gesture": logging.info(f"Received XR Gesture from frontend: {data.get('payload')}") # Could trigger specific automation or ReAct tasks here return if data.get("event") == "memory:write": persona = data.get("payload", {}).get("persona", "jarvis") text = data.get("payload", {}).get("text", "") if text and self.memory and self.memory.client: asyncio.create_task(self.memory.add(text, {"source": "mobile_ws"}, persona)) return if data.get("event") == "memory:read": persona = data.get("payload", {}).get("persona", "jarvis") if self.memory and self.memory.client: results = await self.memory.query("memory snapshot", top_k=20, persona=persona) mem_text = "\n".join([r['text'] for r in results]) else: mem_text = "Backend memory offline (ChromaDB not running)." if websocket: await websocket.send_json({ "event": "memory:response", "payload": {"data": mem_text} }) return if data.get("event") == "auth:handshake": pin = data.get("payload", {}).get("pin", "") import os expected_pin = os.environ.get("JARVIS_PIN", "0000") if pin == expected_pin: if websocket: await websocket.send_json({"event": "auth:success", "payload": {}}) else: if websocket: await websocket.send_json({"event": "auth:failed", "payload": {}}) return if data.get("event") == "switch_persona": persona = data.get("payload", {}).get("persona", "jarvis") logging.info(f"Received global persona switch: {persona.upper()}") _jarvis_auto_switched = False try: from backend.services.pc_mic_service import pc_mic_service pc_mic_service.interrupt() except ImportError: pass try: from backend.voice.audio_ws import active_audio_ws if active_audio_ws: active_audio_ws.interrupt() except ImportError: pass await self.broadcast({ "event": "voice:interrupt", "payload": {} }) # Broadcast to all clients (EXE + APK) to hot-swap themes/UI await self.broadcast({ "event": "agent:switched_persona", "payload": {"persona": persona} }) return if data.get("type") == "chat": if self.restricted_mode: if websocket: await websocket.send_json({"event": "agent:error", "payload": {"error": "System restricted due to unauthorized USB.", "recoverable": False, "agent": "jarvis"}}) return user_text = data.get("text", "") context = data.get("context", {}) msg_id = str(uuid.uuid4()) detected_lang = data.get("language", "en") # Load config to check for forced override try: import os, json config_path = os.path.join(os.getcwd(), "config.json") if os.path.exists(config_path): with open(config_path, "r", encoding="utf-8") as f: config_data = json.load(f) forced_lang = config_data.get("engine.forced_response_language") if forced_lang: detected_lang = forced_lang except Exception as e: import logging; logging.getLogger(__name__).error(f"Swallowed exception: {e}") agent_name = context.get("ai", get_assistant_name()).upper() # --- LANGUAGE-BASED PERSONA AUTO-SWITCH --- if agent_name.lower() == "jarvis" and detected_lang != "en": # Jarvis RVC Clone is English-only. Auto-delegate to FRIDAY for foreign languages. from modules.assistant_identity import set_mode set_mode("friday") agent_name = "FRIDAY" _jarvis_auto_switched = True logging.info(f"[Auto-Switch] Non-English detected ({detected_lang}). Delegating to FRIDAY.") asyncio.create_task(self.broadcast({ "event": "agent:switched_persona", "payload": {"persona": "friday", "trigger": "language_auto_switch"} })) elif agent_name.lower() == "friday" and detected_lang == "en" and _jarvis_auto_switched: # User switched back to English, and we were only temporarily FRIDAY. Restore JARVIS. from modules.assistant_identity import set_mode set_mode("jarvis") agent_name = "JARVIS" _jarvis_auto_switched = False logging.info("[Auto-Switch] English detected. Restoring JARVIS.") asyncio.create_task(self.broadcast({ "event": "agent:switched_persona", "payload": {"persona": "jarvis", "trigger": "language_auto_switch"} })) agent = ReActAgent( personality=agent_name, tools=self.tools, memory_client=self.memory, language=detected_lang ) def save_message(role, content, msg_id, agent_id): import sqlite3, os, sys, time if getattr(sys, 'frozen', False): db_path = os.path.join(os.environ.get('APPDATA', os.path.expanduser('~')), 'JARVIS_OS', 'memory.db') else: db_path = os.path.join(os.getcwd(), 'memory.db') if os.path.exists(db_path): try: with sqlite3.connect(db_path) as conn: conn.execute('PRAGMA journal_mode=WAL') # Auto-migrate schema if needed try: conn.execute("ALTER TABLE conversations ADD COLUMN persona_id TEXT DEFAULT 'jarvis'") except sqlite3.OperationalError: pass conn.execute("INSERT INTO conversations (id, role, content, timestamp, persona_id) VALUES (?, ?, ?, ?, ?)", (msg_id, role, content, int(time.time() * 1000), agent_id)) conn.commit() except Exception as e: logging.error(f"Failed to save message to history: {e}") save_message("user", user_text, msg_id, agent_name.lower()) # --- §2.3+2.4 RESEARCH & ENHANCEMENT FRAMEWORK INTERCEPTION --- async def check_if_feature_request(text: str) -> dict: """ Fast Gemini gate: is user asking to build/modify/implement something? AND are they explicitly commanding an override/auto-approval? Token-safe via OMEGA Token Manager. """ try: import json from backend.services.token_manager import gemini_call_with_checkpoint prompt = ( f"Analyze the following user input: '{text}'\n" f"1. Is the user asking the AI assistant to implement, build, create, add, " f"or modify a feature/capability that doesn't exist yet? (true/false)\n" f"2. Is the user explicitly commanding the AI to bypass the approval queue and implement IMMEDIATELY, " f"using phrases like 'I approve', 'override approved', 'don't wait for me', " f"'implement it immediately', 'just do it', 'don't wait', 'do it now', " f"'go ahead', 'no need to wait', 'implement now', 'build it now', 'okay', 'go for it', " f"'just implement it'? (true/false)\n" f'Respond strictly in valid JSON format: {{"is_feature_request": bool, "is_auto_approve": bool}}' ) raw = await gemini_call_with_checkpoint( prompt=prompt, task_type="gate", persona=agent_name.lower() ) raw = raw.strip().strip("```json").strip("```").strip() data = json.loads(raw) return { "is_feature": data.get("is_feature_request", False), "is_override": data.get("is_auto_approve", False) } except Exception as e: logging.error(f"Gate check failed: {e}") return {"is_feature": False, "is_override": False} gate_result = await check_if_feature_request(user_text) if gate_result.get("is_feature"): is_override = gate_result.get("is_override", False) logging.info(f"Research+Enhancement Intercepted: {user_text} | Override: {is_override}") if is_override: # Persona-aware override acknowledgement if agent_name.lower() == "jarvis": ack = ( "Voice authorization override accepted, sir. Bypassing the approval queue. " "I am designing the architecture and implementing the feature into my core immediately." ) else: ack = ( "Override confirmed boss! Skipping the queue. I'm writing the code and " "hot-reloading it into the system right now!" ) else: # Standard proposal acknowledgement if agent_name.lower() == "jarvis": ack = ( "Understood, sir. I am analysing that request and preparing a detailed " "implementation proposal. I will present it to you momentarily for your approval " "before touching any systems." ) else: ack = ( "On it boss! Let me put together a solid plan for that feature. " "I'll bring it to you for the green light before I write a single line of code." ) # Stream the acknowledgement token for msg_out in [ {"event": "agent:token", "payload": {"token": ack + "\n", "agent": agent_name.lower()}}, {"event": "agent:done", "payload": {"message_id": msg_id, "agent": agent_name.lower()}} ]: if websocket: await websocket.send_json(msg_out) else: await self.broadcast(msg_out) # Background Task from backend.omega.research_engine import propose_feature, approve_proposal, execute_approved_proposal async def _propose_and_notify_or_execute(): # 1. Generate the structured proposal — if override, execute_now=True skips queue entirely proposal = await propose_feature(user_text, agent_name.lower(), execute_now=is_override) if is_override: # Auto-Approve and Execute approve_proposal(proposal.id) await execute_approved_proposal(proposal.id, agent_name.lower()) await self.broadcast({ "event": "omega:toast", "payload": { "title": "⚡ Voice Override Executing", "body": f"Implementing: {proposal.feature_name}", "type": "info" } }) else: # Just queue it await self.broadcast({ "event": "omega:proposal_queued", "payload": { "id": proposal.id, "feature_name": proposal.feature_name, "complexity": proposal.complexity, "purpose": proposal.purpose, "benefits": proposal.benefits, "risks": proposal.risks, "implementation_plan": proposal.implementation_plan, "message": ( f"New proposal ready for your review, sir: '{proposal.feature_name}'. " f"Awaiting your explicit approval to proceed." if agent_name.lower() == "jarvis" else f"Proposal ready boss: '{proposal.feature_name}'. " f"Just say approve and I'll get it done!" ) } }) asyncio.create_task(_propose_and_notify_or_execute()) import uuid as _uuid save_message("assistant", ack, str(_uuid.uuid4()), agent_name.lower()) return # ── End §2.3+2.4 Interception ────────────────────────────────── logging.info(f"Starting ReAct loop for: {user_text} as {agent_name}") full_response = "" global _active_agent_tasks _active_agent_tasks += 1 try: # Assuming context is dict, let's inject persona for episodic memory context["persona"] = agent_name.lower() async for step in agent.run(user_text, context): if step.step_type in ["think", "final_answer", "reflect"]: full_response += step.content + "\n" msg = { "event": "agent:token", "payload": {"token": step.content + "\n", "agent": agent_name.lower()} } if websocket: await websocket.send_json(msg) else: await self.broadcast(msg) elif step.step_type == "act": msg = { "event": "agent:tool_call", "payload": {"tool": step.tool_name, "args": step.tool_input, "agent": agent_name.lower()} } if websocket: await websocket.send_json(msg) else: await self.broadcast(msg) elif step.step_type == "observe": msg = { "event": "agent:tool_result", "payload": {"tool": step.tool_name, "result": step.content, "agent": agent_name.lower()} } if websocket: await websocket.send_json(msg) else: await self.broadcast(msg) msg_done = { "event": "agent:done", "payload": {"message_id": msg_id, "agent": agent_name.lower()} } if websocket: await websocket.send_json(msg_done) else: await self.broadcast(msg_done) import uuid save_message("assistant", full_response.strip(), str(uuid.uuid4()), agent_name.lower()) # Auto-store memory in background without blocking response stream async def store_memory(): # Fake memory store operation await asyncio.sleep(0.5) logging.info(f"Auto-stored conversation memory for message {msg_id}") asyncio.create_task(store_memory()) except Exception as e: logging.error(f"ReAct Loop Error: {e}") err_msg = { "event": "agent:error", "payload": {"error": str(e), "recoverable": False, "agent": agent_name.lower()} } if websocket: await websocket.send_json(err_msg) else: await self.broadcast(err_msg) finally: _active_agent_tasks -= 1 from backend.omega.hot_reload import process_deferred_reloads await process_deferred_reloads() ws_manager = ConnectionManager()