import os import sys import time import subprocess import requests # Ensure the parent directory is in sys.path so 'backend.*' imports work parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if parent_dir not in sys.path: sys.path.insert(0, parent_dir) from backend.services.usb_vault import KeyDomain, resolve_vault_key import sqlite3 import os try: with sqlite3.connect('/home/user/app/memory.db') as conn: print("SUPERVISOR DB DUMP:", conn.cursor().execute("SELECT key_name FROM vault_secrets").fetchall()) except Exception as e: print("SUPERVISOR DB DUMP ERROR:", e) def get_runtime_location() -> str: return "cloud" if os.environ.get("SPACE_ID") else "pc" try: if get_runtime_location() == "cloud": GEMINI_API_KEY = resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_CLOUD) HEAD_SUPERVISOR_KEY = resolve_vault_key(KeyDomain.HEAD_SUPERVISOR_CLOUD) else: GEMINI_API_KEY = resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_PC) HEAD_SUPERVISOR_KEY = resolve_vault_key(KeyDomain.HEAD_SUPERVISOR_PC) except Exception as e: print(f"[SUPERVISOR] API Key initialization failed: {e}") GEMINI_API_KEY = None HEAD_SUPERVISOR_KEY = None def heal_code(file_path, error_traceback): """ Sends the broken file and the stack trace to Gemini and asks it to rewrite the code. Returns the fixed code as a string, or None if it fails. """ if not GEMINI_API_KEY: print("[SUPERVISOR] No Gemini API key found. Cannot auto-heal.") return None try: with open(file_path, "r", encoding="utf-8") as f: broken_code = f.read() prompt = f""" You are the JARVIS / FRIDAY Autonomous Supervisor. The main server just crashed with the following error: {error_traceback} This occurred in the file: {file_path} Here is the current broken code: ```python {broken_code} ``` Rewrite the entire file to fix this bug. Return ONLY the raw python code. Do not include markdown formatting or backticks. """ url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent?key={GEMINI_API_KEY}" payload = { "contents": [{"parts": [{"text": prompt}]}] } headers = {"Content-Type": "application/json"} response = requests.post(url, json=payload, headers=headers) if response.status_code == 429: print("[SUPERVISOR] 429 Google limit hit on standard key.") # Try HEAD_SUPERVISOR_KEY first head_response = None if HEAD_SUPERVISOR_KEY: print("[SUPERVISOR] Attempting fallback to HEAD_SUPERVISOR_KEY...") head_url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent?key={HEAD_SUPERVISOR_KEY}" head_response = requests.post(head_url, json=payload, headers=headers) if head_response and head_response.status_code == 200: print("[SUPERVISOR] Head Supervisor Key succeeded.") data = head_response.json() fixed_code = data["candidates"][0]["content"]["parts"][0]["text"] else: print("[SUPERVISOR] Head Supervisor Key exhausted or missing. Failing over to NVIDIA Heavy Compute Tier...") from backend.services.nvidia_vault import call_nvidia_model, NvidiaModel # Try multiple heavy coding models in descending order of power models_to_try = [ NvidiaModel.GLM_5_1, NvidiaModel.DEEPSEEK_V4_PRO, NvidiaModel.NEMOTRON_3_ULTRA_550B ] fixed_code = None for n_model in models_to_try: print(f"[SUPERVISOR] Attempting heal with {n_model.value}...") result = call_nvidia_model(prompt, n_model) if result and "[NVIDIA FALLBACK FAILED]" not in result: fixed_code = result print(f"[SUPERVISOR] Successfully generated fix using {n_model.value}.") break if not fixed_code: raise Exception("All NVIDIA fallback models failed to generate a fix.") else: response.raise_for_status() data = response.json() fixed_code = data["candidates"][0]["content"]["parts"][0]["text"] # Clean up markdown if the LLM hallucinated it anyway if fixed_code.startswith("```python"): fixed_code = fixed_code.replace("```python", "", 1) elif fixed_code.startswith("```"): fixed_code = fixed_code.replace("```", "", 1) if fixed_code.endswith("```"): fixed_code = fixed_code[:-3] fixed_code = fixed_code.strip() # An unvalidated LLM reply must never replace server code: reject # anything that isn't syntactically valid Python or that shrank the # file so much it's likely a truncated/apology reply. import ast as _ast try: _ast.parse(fixed_code) except SyntaxError as syn_err: print(f"[SUPERVISOR] Heal rejected — generated code does not parse: {syn_err}") return None if len(fixed_code) < max(64, len(broken_code) // 3): print("[SUPERVISOR] Heal rejected — generated code suspiciously short vs original.") return None return fixed_code except Exception as e: print(f"[SUPERVISOR] Auto-healing failed: {e}") return None def start_server(): print("[SUPERVISOR] Booting JARVIS / FRIDAY Server...") import collections restart_timestamps = collections.deque() while True: now = time.time() while restart_timestamps and now - restart_timestamps[0] > 60: restart_timestamps.popleft() if len(restart_timestamps) >= 5: print("[SUPERVISOR] CIRCUIT BREAKER TRIPPED! Max 5 restarts/min reached. Exiting.") sys.exit(1) restart_timestamps.append(now) try: # Run the main FastAPI app as a module from the parent directory # so that 'from backend...' imports work natively parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) process = subprocess.Popen( [sys.executable, "-u", "-m", "backend.main"], cwd=parent_dir, stdout=sys.stdout, stderr=subprocess.PIPE, text=True ) stderr_buffer = [] def read_stderr(): for line in process.stderr: sys.stdout.write(line) sys.stdout.flush() stderr_buffer.append(line) import threading t = threading.Thread(target=read_stderr, daemon=True) t.start() process.wait() t.join() stderr_data = "".join(stderr_buffer) if process.returncode != 0: print("\n[SUPERVISOR] FATAL CRASH DETECTED!") print("="*50) # Attempt to extract the file that crashed from the traceback crashed_file = None for line in reversed(stderr_data.splitlines()): if "File " in line and ".py" in line: # Parse out the file path (e.g., ' File "/app/backend/routes/...", line 10') parts = line.split('"') if len(parts) >= 3: potential_file = os.path.abspath(parts[1]) if potential_file.startswith(parent_dir): crashed_file = potential_file break if crashed_file and os.path.exists(crashed_file): print(f"[SUPERVISOR] Attempting AI Auto-Heal on {crashed_file}...") fixed_code = heal_code(crashed_file, stderr_data) if fixed_code: # Keep the pre-heal original so a bad patch is always recoverable. try: import shutil shutil.copy2(crashed_file, crashed_file + ".pre_heal.bak") except Exception as bak_err: print(f"[SUPERVISOR] Warning: could not back up original before heal: {bak_err}") with open(crashed_file, "w", encoding="utf-8") as f: f.write(fixed_code) print("[SUPERVISOR] Code healed successfully. Restarting in 3 seconds...") try: import asyncio from backend.events.omega_event_bus import publish_omega_event, OmegaEvent from backend.services.usb_vault import KeyDomain async def publish_heal(): await publish_omega_event(OmegaEvent( domain=KeyDomain.SUPERVISOR_HEAL_CLOUD if get_runtime_location() == "cloud" else KeyDomain.SUPERVISOR_HEAL_PC, event_type="bug_found", description=f"Auto-healed crash in {os.path.basename(crashed_file)}", persona="jarvis" )) asyncio.run(publish_heal()) except Exception as pub_err: print(f"[SUPERVISOR] Failed to publish heal event: {pub_err}") time.sleep(3) continue else: print("[SUPERVISOR] Healing failed. Restarting blindly in 5 seconds...") time.sleep(5) else: print("[SUPERVISOR] Could not isolate the crashed file. Restarting in 5 seconds...") time.sleep(5) else: # Normal exit (e.g., intentional shutdown) print("[SUPERVISOR] Server shut down normally.") break except KeyboardInterrupt: print("\n[SUPERVISOR] Received KeyboardInterrupt. Shutting down.") process.terminate() break except Exception as e: print(f"[SUPERVISOR] Unknown error: {e}") time.sleep(5) if __name__ == "__main__": start_server()