backend/supervisor.py

import os
import sys
import time
import subprocess
import requests

# Ensure the parent directory is in sys.path so 'backend.*' imports work
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if parent_dir not in sys.path:
    sys.path.insert(0, parent_dir)

from backend.services.usb_vault import KeyDomain, resolve_vault_key
import sqlite3
import os
try:
    with sqlite3.connect('/home/user/app/memory.db') as conn:
        print("SUPERVISOR DB DUMP:", conn.cursor().execute("SELECT key_name FROM vault_secrets").fetchall())
except Exception as e:
    print("SUPERVISOR DB DUMP ERROR:", e)

def get_runtime_location() -> str:
    return "cloud" if os.environ.get("SPACE_ID") else "pc"

try:
    if get_runtime_location() == "cloud":
        GEMINI_API_KEY = resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_CLOUD)
        HEAD_SUPERVISOR_KEY = resolve_vault_key(KeyDomain.HEAD_SUPERVISOR_CLOUD)
    else:
        GEMINI_API_KEY = resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_PC)
        HEAD_SUPERVISOR_KEY = resolve_vault_key(KeyDomain.HEAD_SUPERVISOR_PC)
except Exception as e:
    print(f"[SUPERVISOR] API Key initialization failed: {e}")
    GEMINI_API_KEY = None
    HEAD_SUPERVISOR_KEY = None
def heal_code(file_path, error_traceback):
    """
    Sends the broken file and the stack trace to Gemini and asks it to rewrite the code.
    Returns the fixed code as a string, or None if it fails.
    """
    if not GEMINI_API_KEY:
        print("[SUPERVISOR] No Gemini API key found. Cannot auto-heal.")
        return None

    try:
        with open(file_path, "r", encoding="utf-8") as f:
            broken_code = f.read()

        prompt = f"""
You are the JARVIS / FRIDAY Autonomous Supervisor. 
The main server just crashed with the following error:

{error_traceback}

This occurred in the file: {file_path}

Here is the current broken code:
```python
{broken_code}
```

Rewrite the entire file to fix this bug. 
Return ONLY the raw python code. Do not include markdown formatting or backticks.
"""
        url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent?key={GEMINI_API_KEY}"
        payload = {
            "contents": [{"parts": [{"text": prompt}]}]
        }
        headers = {"Content-Type": "application/json"}
        
        response = requests.post(url, json=payload, headers=headers)
        if response.status_code == 429:
            print("[SUPERVISOR] 429 Google limit hit on standard key.")
            
            # Try HEAD_SUPERVISOR_KEY first
            head_response = None
            if HEAD_SUPERVISOR_KEY:
                print("[SUPERVISOR] Attempting fallback to HEAD_SUPERVISOR_KEY...")
                head_url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent?key={HEAD_SUPERVISOR_KEY}"
                head_response = requests.post(head_url, json=payload, headers=headers)
            
            if head_response and head_response.status_code == 200:
                print("[SUPERVISOR] Head Supervisor Key succeeded.")
                data = head_response.json()
                fixed_code = data["candidates"][0]["content"]["parts"][0]["text"]
            else:
                print("[SUPERVISOR] Head Supervisor Key exhausted or missing. Failing over to NVIDIA Heavy Compute Tier...")
                from backend.services.nvidia_vault import call_nvidia_model, NvidiaModel
                
                # Try multiple heavy coding models in descending order of power
                models_to_try = [
                    NvidiaModel.GLM_5_1,
                    NvidiaModel.DEEPSEEK_V4_PRO,
                    NvidiaModel.NEMOTRON_3_ULTRA_550B
                ]
                
                fixed_code = None
                for n_model in models_to_try:
                    print(f"[SUPERVISOR] Attempting heal with {n_model.value}...")
                    result = call_nvidia_model(prompt, n_model)
                    if result and "[NVIDIA FALLBACK FAILED]" not in result:
                        fixed_code = result
                        print(f"[SUPERVISOR] Successfully generated fix using {n_model.value}.")
                        break
                        
                if not fixed_code:
                    raise Exception("All NVIDIA fallback models failed to generate a fix.")
        else:
            response.raise_for_status()
            data = response.json()
            fixed_code = data["candidates"][0]["content"]["parts"][0]["text"]
        
        # Clean up markdown if the LLM hallucinated it anyway
        if fixed_code.startswith("```python"):
            fixed_code = fixed_code.replace("```python", "", 1)
        elif fixed_code.startswith("```"):
            fixed_code = fixed_code.replace("```", "", 1)
            
        if fixed_code.endswith("```"):
            fixed_code = fixed_code[:-3]

        fixed_code = fixed_code.strip()

        # An unvalidated LLM reply must never replace server code: reject
        # anything that isn't syntactically valid Python or that shrank the
        # file so much it's likely a truncated/apology reply.
        import ast as _ast
        try:
            _ast.parse(fixed_code)
        except SyntaxError as syn_err:
            print(f"[SUPERVISOR] Heal rejected — generated code does not parse: {syn_err}")
            return None
        if len(fixed_code) < max(64, len(broken_code) // 3):
            print("[SUPERVISOR] Heal rejected — generated code suspiciously short vs original.")
            return None

        return fixed_code
    except Exception as e:
        print(f"[SUPERVISOR] Auto-healing failed: {e}")
        return None

def start_server():
    print("[SUPERVISOR] Booting JARVIS / FRIDAY Server...")
    import collections
    restart_timestamps = collections.deque()
    
    while True:
        now = time.time()
        while restart_timestamps and now - restart_timestamps[0] > 60:
            restart_timestamps.popleft()
            
        if len(restart_timestamps) >= 5:
            print("[SUPERVISOR] CIRCUIT BREAKER TRIPPED! Max 5 restarts/min reached. Exiting.")
            sys.exit(1)
            
        restart_timestamps.append(now)
        
        try:
            # Run the main FastAPI app as a module from the parent directory
            # so that 'from backend...' imports work natively
            parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
            process = subprocess.Popen(
                [sys.executable, "-u", "-m", "backend.main"], 
                cwd=parent_dir,
                stdout=sys.stdout,
                stderr=subprocess.PIPE,
                text=True
            )
            
            stderr_buffer = []
            def read_stderr():
                for line in process.stderr:
                    sys.stdout.write(line)
                    sys.stdout.flush()
                    stderr_buffer.append(line)
            
            import threading
            t = threading.Thread(target=read_stderr, daemon=True)
            t.start()
            
            process.wait()
            t.join()
            stderr_data = "".join(stderr_buffer)
            
            if process.returncode != 0:
                print("\n[SUPERVISOR] FATAL CRASH DETECTED!")
                print("="*50)
                
                # Attempt to extract the file that crashed from the traceback
                crashed_file = None
                for line in reversed(stderr_data.splitlines()):
                    if "File " in line and ".py" in line:
                        # Parse out the file path (e.g., '  File "/app/backend/routes/...", line 10')
                        parts = line.split('"')
                        if len(parts) >= 3:
                            potential_file = os.path.abspath(parts[1])
                            if potential_file.startswith(parent_dir):
                                crashed_file = potential_file
                                break
                
                if crashed_file and os.path.exists(crashed_file):
                    print(f"[SUPERVISOR] Attempting AI Auto-Heal on {crashed_file}...")
                    fixed_code = heal_code(crashed_file, stderr_data)
                    
                    if fixed_code:
                        # Keep the pre-heal original so a bad patch is always recoverable.
                        try:
                            import shutil
                            shutil.copy2(crashed_file, crashed_file + ".pre_heal.bak")
                        except Exception as bak_err:
                            print(f"[SUPERVISOR] Warning: could not back up original before heal: {bak_err}")
                        with open(crashed_file, "w", encoding="utf-8") as f:
                            f.write(fixed_code)
                        print("[SUPERVISOR] Code healed successfully. Restarting in 3 seconds...")
                        try:
                            import asyncio
                            from backend.events.omega_event_bus import publish_omega_event, OmegaEvent
                            from backend.services.usb_vault import KeyDomain
                            
                            async def publish_heal():
                                await publish_omega_event(OmegaEvent(
                                    domain=KeyDomain.SUPERVISOR_HEAL_CLOUD if get_runtime_location() == "cloud" else KeyDomain.SUPERVISOR_HEAL_PC,
                                    event_type="bug_found",
                                    description=f"Auto-healed crash in {os.path.basename(crashed_file)}",
                                    persona="jarvis"
                                ))
                            asyncio.run(publish_heal())
                        except Exception as pub_err:
                            print(f"[SUPERVISOR] Failed to publish heal event: {pub_err}")
                        time.sleep(3)
                        continue
                    else:
                        print("[SUPERVISOR] Healing failed. Restarting blindly in 5 seconds...")
                        time.sleep(5)
                else:
                    print("[SUPERVISOR] Could not isolate the crashed file. Restarting in 5 seconds...")
                    time.sleep(5)
            else:
                # Normal exit (e.g., intentional shutdown)
                print("[SUPERVISOR] Server shut down normally.")
                break

        except KeyboardInterrupt:
            print("\n[SUPERVISOR] Received KeyboardInterrupt. Shutting down.")
            process.terminate()
            break
        except Exception as e:
            print(f"[SUPERVISOR] Unknown error: {e}")
            time.sleep(5)

if __name__ == "__main__":
    start_server()