""" Crash telemetry + backup/export — the two remaining v6 platform additions. CRASH TELEMETRY A crash that a subscriber cannot report is a crash nobody can fix, and the end user must never open a terminal to collect one (the standing constraint). So the backend records its own unhandled failures into a bounded local table and exposes them to the owner. Nothing leaves the machine on its own: this is a local ring buffer with an explicit read endpoint, not an uploader. Privacy: the recorder redacts anything that looks like a credential before it is written, so a traceback that happens to contain a key does not turn the crash log into a second copy of the vault. BACKUP / EXPORT Everything a subscriber creates — conversations, memories, automations, personas, tasks — lives in one SQLite file. Export produces a single JSON document of the user-owned tables so it can be copied, versioned, or moved to a new install. Secrets are deliberately NOT exported: vault tables carry ciphertext that is useless on another machine and dangerous in a backup file, so they are listed by name and row count only. """ from __future__ import annotations import json import logging import os import re import sqlite3 import time import traceback from typing import Any from fastapi import APIRouter, HTTPException, Query router = APIRouter() # Tables the user owns and that are safe to hand back to them verbatim. _EXPORTABLE_TABLES = ( "conversations", "nodes", "edges", "skills", "tasks", "personas", "custom_automations", "automation_history", "research_notes", "research_reports", "omega_events", "gaming_sessions", "upgrades", "easter_eggs_triggered", "model_generations", ) # Tables that exist but must never be written into an export file. _SENSITIVE_TABLES = ( "vault_keys", "vault_secrets", "vault_credentials", "credentials", "master_vault_ledger", "token_checkpoints", ) _MAX_CRASHES = 500 # Anything shaped like a credential is replaced before the row is stored. # # Only the optional named group `keep` survives substitution — it holds the # harmless prefix ("Bearer ", "api_key=") that makes the redacted line readable. # Everything else the pattern matches is dropped. An earlier version re-emitted # group 1 unconditionally, which for the bare-token patterns WAS the token, so # `sk-…` was written back out next to the [REDACTED] marker. _REDACTIONS = ( re.compile(r"(?i)(?P[\"']?(?:api[_-]?key|token|password|secret)[\"']?\s*[=:]\s*[\"']?)[^\s\"',}]{8,}"), re.compile(r"(?i)(?Pbearer\s+)[A-Za-z0-9._\-]{16,}"), re.compile(r"(?i)\bsk-[A-Za-z0-9_\-]{16,}"), re.compile(r"\bAIza[0-9A-Za-z_\-]{20,}"), re.compile(r"(?i)\bomega_[0-9a-f]{16,}"), re.compile(r"(?i)\bnvapi-[A-Za-z0-9_\-]{16,}"), re.compile(r"(?i)\bhf_[A-Za-z0-9]{20,}"), ) def _sub(match: re.Match) -> str: return (match.groupdict().get("keep") or "") + "[REDACTED]" def get_db_path() -> str: if "JARVIS_APP_DATA_DIR" in os.environ: return os.path.join(os.environ["JARVIS_APP_DATA_DIR"], "memory.db") project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) return os.path.join(project_root, "memory.db") def redact(text: str) -> str: out = text or "" for pattern in _REDACTIONS: out = pattern.sub(_sub, out) return out def _connect() -> sqlite3.Connection: conn = sqlite3.connect(get_db_path(), timeout=10) conn.execute("PRAGMA journal_mode=WAL") conn.execute( """CREATE TABLE IF NOT EXISTS crash_reports ( id INTEGER PRIMARY KEY AUTOINCREMENT, ts_ms INTEGER NOT NULL, surface TEXT NOT NULL, kind TEXT NOT NULL, message TEXT NOT NULL, stack TEXT, context TEXT )""" ) return conn def record_crash( surface: str, exc: BaseException | None = None, *, kind: str = "", message: str = "", context: dict[str, Any] | None = None, ) -> None: """ Record one crash. Safe to call from an exception handler: it never raises, because a telemetry failure must not become a second crash. """ try: if exc is not None: kind = kind or type(exc).__name__ message = message or str(exc) stack = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) else: stack = "" with _connect() as conn: conn.execute( "INSERT INTO crash_reports (ts_ms, surface, kind, message, stack, context) VALUES (?,?,?,?,?,?)", ( int(time.time() * 1000), str(surface)[:64], str(kind or "Error")[:96], redact(str(message))[:2000], redact(stack)[:16000], redact(json.dumps(context or {}))[:2000], ), ) # Bounded: keep only the most recent _MAX_CRASHES rows. conn.execute( "DELETE FROM crash_reports WHERE id NOT IN (SELECT id FROM crash_reports ORDER BY id DESC LIMIT ?)", (_MAX_CRASHES,), ) conn.commit() except Exception as e: # pragma: no cover - telemetry must never propagate logging.getLogger(__name__).warning(f"crash telemetry write failed: {e}") @router.get("/crashes") async def list_crashes(limit: int = Query(50, ge=1, le=_MAX_CRASHES)): """Most recent crashes, newest first. Values are already redacted at write time.""" try: with _connect() as conn: conn.row_factory = sqlite3.Row rows = conn.execute( "SELECT id, ts_ms, surface, kind, message, stack, context " "FROM crash_reports ORDER BY id DESC LIMIT ?", (limit,), ).fetchall() return {"ok": True, "count": len(rows), "crashes": [dict(r) for r in rows]} except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @router.post("/crashes") async def report_crash(payload: dict): """ Lets the exe front-end and the APK report their own crashes into the same store, so one place answers "what broke" across all three surfaces. """ surface = str(payload.get("surface") or "unknown")[:64] kind = str(payload.get("kind") or "Error")[:96] message = str(payload.get("message") or "")[:2000] stack = str(payload.get("stack") or "")[:16000] if not message and not stack: raise HTTPException(status_code=422, detail="message or stack is required") try: with _connect() as conn: conn.execute( "INSERT INTO crash_reports (ts_ms, surface, kind, message, stack, context) VALUES (?,?,?,?,?,?)", ( int(time.time() * 1000), surface, kind, redact(message), redact(stack), redact(json.dumps(payload.get("context") or {}))[:2000], ), ) conn.execute( "DELETE FROM crash_reports WHERE id NOT IN (SELECT id FROM crash_reports ORDER BY id DESC LIMIT ?)", (_MAX_CRASHES,), ) conn.commit() return {"ok": True, "status": "recorded"} except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @router.delete("/crashes") async def clear_crashes(): """Owner-initiated clear. Empties the ring buffer; touches nothing else.""" try: with _connect() as conn: n = conn.execute("SELECT COUNT(*) FROM crash_reports").fetchone()[0] conn.execute("DELETE FROM crash_reports") conn.commit() return {"ok": True, "cleared": n} except Exception as e: raise HTTPException(status_code=500, detail=str(e)) def _existing_tables(conn: sqlite3.Connection) -> set[str]: return {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} @router.get("/export") async def export_data(max_rows_per_table: int = Query(5000, ge=1, le=100000)): """ Full user-data export as one JSON document. Secrets are never included. Sensitive tables are reported by name and row count so the export is honest about what it is NOT carrying, instead of silently looking complete. """ try: with _connect() as conn: conn.row_factory = sqlite3.Row present = _existing_tables(conn) data: dict[str, list[dict[str, Any]]] = {} truncated: list[str] = [] for table in _EXPORTABLE_TABLES: if table not in present: continue rows = conn.execute( f"SELECT * FROM {table} LIMIT ?", (max_rows_per_table + 1,) ).fetchall() if len(rows) > max_rows_per_table: rows = rows[:max_rows_per_table] truncated.append(table) data[table] = [dict(r) for r in rows] excluded = { t: conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] for t in _SENSITIVE_TABLES if t in present } return { "ok": True, "format": "omega-export/1", "exported_at_ms": int(time.time() * 1000), "source_db": os.path.basename(get_db_path()), "tables": {t: len(rows) for t, rows in data.items()}, "truncated_tables": truncated, "excluded_for_safety": excluded, "data": data, } except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @router.get("/export/manifest") async def export_manifest(): """ What an export would contain, without producing it — so the UI can show size and scope before a subscriber downloads anything. """ try: with _connect() as conn: present = _existing_tables(conn) counts = { t: conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] for t in _EXPORTABLE_TABLES if t in present } excluded = { t: conn.execute(f"SELECT COUNT(*) FROM {t}").fetchone()[0] for t in _SENSITIVE_TABLES if t in present } db_path = get_db_path() return { "ok": True, "db_bytes": os.path.getsize(db_path) if os.path.exists(db_path) else 0, "exportable": counts, "exportable_rows": sum(counts.values()), "excluded_for_safety": excluded, "missing_tables": [t for t in _EXPORTABLE_TABLES if t not in counts], } except Exception as e: raise HTTPException(status_code=500, detail=str(e))