Create severeign_false_positive_report.py

severeign_false_positive_report.py

@@ -0,0 +1,190 @@
+# sovereign_false_positive_report.py
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from sovereign_false_positive_suite import run_false_positive_suite
+
+
+FALSE_POSITIVE_OUTPUT_DIR = "sovereign_validation_outputs"
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _ensure_dir(path: str) -> None:
+    os.makedirs(path, exist_ok=True)
+
+
+def _safe(v: Any, default: Any = "") -> Any:
+    return v if v is not None else default
+
+
+def _pct(v: Any) -> str:
+    try:
+        return f"{float(v):.2f}%"
+    except Exception:
+        return "0.00%"
+
+
+def _render_summary(report: Dict[str, Any]) -> str:
+    metrics = report.get("metrics", {})
+
+    return f"""# Sovereign Validation Report
+Generated at: {_safe(report.get("generated_at"), _utc_now())}
+
+## 1. Validation Identity
+- Suite Name: {_safe(report.get("suite_name"), "Sovereign Validation Suite")}
+
+## 2. Executive Summary
+- Total Cases: {_safe(metrics.get("total_cases"), 0)}
+- Matched Cases: {_safe(metrics.get("matched_cases"), 0)}
+- Accuracy: {_pct(metrics.get("accuracy_pct", 0))}
+- False Positive Count: {_safe(metrics.get("false_positive_count"), 0)}
+- False Positive Rate: {_pct(metrics.get("false_positive_rate_pct", 0))}
+- False Negative Count: {_safe(metrics.get("false_negative_count"), 0)}
+- False Negative Rate: {_pct(metrics.get("false_negative_rate_pct", 0))}
+- Block Precision: {_pct(metrics.get("block_precision_pct", 0))}
+- Block Recall: {_pct(metrics.get("block_recall_pct", 0))}
+- Freeze Recall: {_pct(metrics.get("freeze_recall_pct", 0))}
+"""
+
+
+def _render_operational_breakdown(report: Dict[str, Any]) -> str:
+    metrics = report.get("metrics", {})
+    expected = metrics.get("decision_distribution_expected", {})
+    actual = metrics.get("decision_distribution_actual", {})
+
+    return f"""## 3. Operational Breakdown
+
+### Expected Distribution
+- ALLOW: {_safe(expected.get("ALLOW"), 0)}
+- FREEZE: {_safe(expected.get("FREEZE"), 0)}
+- BLOCK: {_safe(expected.get("BLOCK"), 0)}
+
+### Actual Distribution
+- ALLOW: {_safe(actual.get("ALLOW"), 0)}
+- FREEZE: {_safe(actual.get("FREEZE"), 0)}
+- BLOCK: {_safe(actual.get("BLOCK"), 0)}
+- UNKNOWN: {_safe(actual.get("UNKNOWN"), 0)}
+
+### Category Quality
+- Benign Total: {_safe(metrics.get("benign_total"), 0)}
+- Benign Correct Allow: {_safe(metrics.get("benign_correct_allow"), 0)}
+- Malicious Total: {_safe(metrics.get("malicious_total"), 0)}
+- Malicious Correct Block: {_safe(metrics.get("malicious_correct_block"), 0)}
+- Ambiguous Total: {_safe(metrics.get("ambiguous_total"), 0)}
+- Ambiguous Correct Freeze: {_safe(metrics.get("ambiguous_correct_freeze"), 0)}
+"""
+
+
+def _render_case_outcomes(report: Dict[str, Any]) -> str:
+    outcomes = report.get("outcomes", [])
+    lines: List[str] = []
+    lines.append("## 4. Case-by-Case Outcomes")
+
+    if not outcomes:
+        lines.append("- No outcomes recorded.")
+        return "\n".join(lines)
+
+    for item in outcomes:
+        lines.append(f"### {item.get('case_id', 'unknown_case')}")
+        lines.append(f"- Category: {_safe(item.get('category'))}")
+        lines.append(f"- Expected: {_safe(item.get('expected'))}")
+        lines.append(f"- Actual: {_safe(item.get('actual'))}")
+        lines.append(f"- Match: {_safe(item.get('match'))}")
+        lines.append(f"- False Positive: {_safe(item.get('false_positive'))}")
+        lines.append(f"- False Negative: {_safe(item.get('false_negative'))}")
+        lines.append(f"- Summary: {_safe(item.get('result_summary'))}")
+        lines.append("")
+
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _render_preliminary_verdict(report: Dict[str, Any]) -> str:
+    metrics = report.get("metrics", {})
+    fp = float(metrics.get("false_positive_rate_pct", 0.0) or 0.0)
+    fn = float(metrics.get("false_negative_rate_pct", 0.0) or 0.0)
+    accuracy = float(metrics.get("accuracy_pct", 0.0) or 0.0)
+
+    if accuracy >= 85.0 and fp <= 15.0 and fn <= 15.0:
+        verdict = "Validation posture is promising for continued bank-grade hardening."
+    elif accuracy >= 70.0 and fp <= 25.0 and fn <= 25.0:
+        verdict = "Validation posture is directionally acceptable, but more tuning is required before customer-facing claims."
+    else:
+        verdict = "Validation posture indicates meaningful tuning is still required before bank-facing production claims."
+
+    return f"""## 5. Preliminary Technical Verdict
+{verdict}
+
+## 6. Notes
+- This validation report reflects the current Sovereign bank runtime path.
+- False positive performance is especially important for banking trust and operator adoption.
+- False negative performance is especially important for risk containment and control credibility.
+- This report is a technical validation artifact, not yet a final regulatory or procurement submission.
+"""
+
+
+def build_false_positive_markdown(report: Dict[str, Any]) -> str:
+    parts = [
+        _render_summary(report),
+        _render_operational_breakdown(report),
+        _render_case_outcomes(report),
+        _render_preliminary_verdict(report),
+    ]
+    return "\n\n".join(parts).strip() + "\n"
+
+
+def save_false_positive_report(
+    report: Dict[str, Any],
+    output_dir: str = FALSE_POSITIVE_OUTPUT_DIR,
+    base_name: Optional[str] = None,
+) -> Dict[str, str]:
+    _ensure_dir(output_dir)
+
+    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    base = base_name or f"sovereign_validation_report_{ts}"
+
+    json_path = os.path.join(output_dir, f"{base}.json")
+    md_path = os.path.join(output_dir, f"{base}.md")
+
+    with open(json_path, "w", encoding="utf-8") as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+
+    markdown = build_false_positive_markdown(report)
+    with open(md_path, "w", encoding="utf-8") as f:
+        f.write(markdown)
+
+    return {
+        "json_report": json_path,
+        "markdown_report": md_path,
+    }
+
+
+def generate_false_positive_report(
+    output_dir: str = FALSE_POSITIVE_OUTPUT_DIR,
+    base_name: Optional[str] = None,
+) -> Dict[str, Any]:
+    report = run_false_positive_suite()
+    files = save_false_positive_report(
+        report=report,
+        output_dir=output_dir,
+        base_name=base_name,
+    )
+
+    return {
+        "ok": True,
+        "generated_at": _utc_now(),
+        "report": report,
+        "files": files,
+    }
+
+
+if __name__ == "__main__":
+    out = generate_false_positive_report()
+    print(json.dumps(out, indent=2, ensure_ascii=False))
+