"""Verify the v10 examples pack against the trace API."""

from __future__ import annotations

import json
from pathlib import Path

from ts_bridge.reasoner_v10 import solve_trace


ROOT = Path(__file__).resolve().parent
EXAMPLES = ROOT / "examples" / "v10_examples.json"
OUT = ROOT / "logs" / "eval" / "ts_reasoner_v10_examples_trace.json"


def main() -> None:
    rows = []
    failures = []
    for item in json.loads(EXAMPLES.read_text()):
        trace = solve_trace(item["prompt"], item["category"]).to_dict()
        ok = trace["display_answer"] == item["expected_display_answer"] and bool(trace["solved"]) == bool(item["expected_solved"])
        if item.get("expected_boundary_reason") is not None:
            ok = ok and trace["boundary_reason"] == item["expected_boundary_reason"]
        row = {"example": item, "trace": trace, "ok": ok}
        rows.append(row)
        if not ok:
            failures.append(row)
    payload = {
        "engine": "TensionLM-117M-TS-Reasoner-v10",
        "total": len(rows),
        "correct": len(rows) - len(failures),
        "accuracy": (len(rows) - len(failures)) / len(rows),
        "failures": failures,
        "results": rows,
    }
    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(json.dumps(payload, indent=2))
    print(f"examples={payload['correct']}/{payload['total']} ({100*payload['accuracy']:.1f}%)")
    print(f"Wrote {OUT}")
    if failures:
        raise SystemExit(1)


if __name__ == "__main__":
    main()