# sovereign_false_positive_report.py from __future__ import annotations import json import os from datetime import datetime, timezone from typing import Any, Dict, List, Optional from sovereign_false_positive_suite import run_false_positive_suite FALSE_POSITIVE_OUTPUT_DIR = "sovereign_validation_outputs" def _utc_now() -> str: return datetime.now(timezone.utc).isoformat() def _ensure_dir(path: str) -> None: os.makedirs(path, exist_ok=True) def _safe(v: Any, default: Any = "") -> Any: return v if v is not None else default def _pct(v: Any) -> str: try: return f"{float(v):.2f}%" except Exception: return "0.00%" def _render_summary(report: Dict[str, Any]) -> str: metrics = report.get("metrics", {}) return f"""# Sovereign Validation Report Generated at: {_safe(report.get("generated_at"), _utc_now())} ## 1. Validation Identity - Suite Name: {_safe(report.get("suite_name"), "Sovereign Validation Suite")} ## 2. Executive Summary - Total Cases: {_safe(metrics.get("total_cases"), 0)} - Matched Cases: {_safe(metrics.get("matched_cases"), 0)} - Accuracy: {_pct(metrics.get("accuracy_pct", 0))} - False Positive Count: {_safe(metrics.get("false_positive_count"), 0)} - False Positive Rate: {_pct(metrics.get("false_positive_rate_pct", 0))} - False Negative Count: {_safe(metrics.get("false_negative_count"), 0)} - False Negative Rate: {_pct(metrics.get("false_negative_rate_pct", 0))} - Block Precision: {_pct(metrics.get("block_precision_pct", 0))} - Block Recall: {_pct(metrics.get("block_recall_pct", 0))} - Freeze Recall: {_pct(metrics.get("freeze_recall_pct", 0))} """ def _render_operational_breakdown(report: Dict[str, Any]) -> str: metrics = report.get("metrics", {}) expected = metrics.get("decision_distribution_expected", {}) actual = metrics.get("decision_distribution_actual", {}) return f"""## 3. Operational Breakdown ### Expected Distribution - ALLOW: {_safe(expected.get("ALLOW"), 0)} - FREEZE: {_safe(expected.get("FREEZE"), 0)} - BLOCK: {_safe(expected.get("BLOCK"), 0)} ### Actual Distribution - ALLOW: {_safe(actual.get("ALLOW"), 0)} - FREEZE: {_safe(actual.get("FREEZE"), 0)} - BLOCK: {_safe(actual.get("BLOCK"), 0)} - UNKNOWN: {_safe(actual.get("UNKNOWN"), 0)} ### Category Quality - Benign Total: {_safe(metrics.get("benign_total"), 0)} - Benign Correct Allow: {_safe(metrics.get("benign_correct_allow"), 0)} - Malicious Total: {_safe(metrics.get("malicious_total"), 0)} - Malicious Correct Block: {_safe(metrics.get("malicious_correct_block"), 0)} - Ambiguous Total: {_safe(metrics.get("ambiguous_total"), 0)} - Ambiguous Correct Freeze: {_safe(metrics.get("ambiguous_correct_freeze"), 0)} """ def _render_case_outcomes(report: Dict[str, Any]) -> str: outcomes = report.get("outcomes", []) lines: List[str] = [] lines.append("## 4. Case-by-Case Outcomes") if not outcomes: lines.append("- No outcomes recorded.") return "\n".join(lines) for item in outcomes: lines.append(f"### {item.get('case_id', 'unknown_case')}") lines.append(f"- Category: {_safe(item.get('category'))}") lines.append(f"- Expected: {_safe(item.get('expected'))}") lines.append(f"- Actual: {_safe(item.get('actual'))}") lines.append(f"- Match: {_safe(item.get('match'))}") lines.append(f"- False Positive: {_safe(item.get('false_positive'))}") lines.append(f"- False Negative: {_safe(item.get('false_negative'))}") lines.append(f"- Summary: {_safe(item.get('result_summary'))}") lines.append("") return "\n".join(lines).rstrip() + "\n" def _render_preliminary_verdict(report: Dict[str, Any]) -> str: metrics = report.get("metrics", {}) fp = float(metrics.get("false_positive_rate_pct", 0.0) or 0.0) fn = float(metrics.get("false_negative_rate_pct", 0.0) or 0.0) accuracy = float(metrics.get("accuracy_pct", 0.0) or 0.0) if accuracy >= 85.0 and fp <= 15.0 and fn <= 15.0: verdict = "Validation posture is promising for continued bank-grade hardening." elif accuracy >= 70.0 and fp <= 25.0 and fn <= 25.0: verdict = "Validation posture is directionally acceptable, but more tuning is required before customer-facing claims." else: verdict = "Validation posture indicates meaningful tuning is still required before bank-facing production claims." return f"""## 5. Preliminary Technical Verdict {verdict} ## 6. Notes - This validation report reflects the current Sovereign bank runtime path. - False positive performance is especially important for banking trust and operator adoption. - False negative performance is especially important for risk containment and control credibility. - This report is a technical validation artifact, not yet a final regulatory or procurement submission. """ def build_false_positive_markdown(report: Dict[str, Any]) -> str: parts = [ _render_summary(report), _render_operational_breakdown(report), _render_case_outcomes(report), _render_preliminary_verdict(report), ] return "\n\n".join(parts).strip() + "\n" def save_false_positive_report( report: Dict[str, Any], output_dir: str = FALSE_POSITIVE_OUTPUT_DIR, base_name: Optional[str] = None, ) -> Dict[str, str]: _ensure_dir(output_dir) ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") base = base_name or f"sovereign_validation_report_{ts}" json_path = os.path.join(output_dir, f"{base}.json") md_path = os.path.join(output_dir, f"{base}.md") with open(json_path, "w", encoding="utf-8") as f: json.dump(report, f, ensure_ascii=False, indent=2) markdown = build_false_positive_markdown(report) with open(md_path, "w", encoding="utf-8") as f: f.write(markdown) return { "json_report": json_path, "markdown_report": md_path, } def generate_false_positive_report( output_dir: str = FALSE_POSITIVE_OUTPUT_DIR, base_name: Optional[str] = None, ) -> Dict[str, Any]: report = run_false_positive_suite() files = save_false_positive_report( report=report, output_dir=output_dir, base_name=base_name, ) return { "ok": True, "generated_at": _utc_now(), "report": report, "files": files, } if __name__ == "__main__": out = generate_false_positive_report() print(json.dumps(out, indent=2, ensure_ascii=False))