severeign_false_positive_report.py

# sovereign_false_positive_report.py
from __future__ import annotations

import json
import os
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional

from sovereign_false_positive_suite import run_false_positive_suite


FALSE_POSITIVE_OUTPUT_DIR = "sovereign_validation_outputs"


def _utc_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def _ensure_dir(path: str) -> None:
    os.makedirs(path, exist_ok=True)


def _safe(v: Any, default: Any = "") -> Any:
    return v if v is not None else default


def _pct(v: Any) -> str:
    try:
        return f"{float(v):.2f}%"
    except Exception:
        return "0.00%"


def _render_summary(report: Dict[str, Any]) -> str:
    metrics = report.get("metrics", {})

    return f"""# Sovereign Validation Report
Generated at: {_safe(report.get("generated_at"), _utc_now())}

## 1. Validation Identity
- Suite Name: {_safe(report.get("suite_name"), "Sovereign Validation Suite")}

## 2. Executive Summary
- Total Cases: {_safe(metrics.get("total_cases"), 0)}
- Matched Cases: {_safe(metrics.get("matched_cases"), 0)}
- Accuracy: {_pct(metrics.get("accuracy_pct", 0))}
- False Positive Count: {_safe(metrics.get("false_positive_count"), 0)}
- False Positive Rate: {_pct(metrics.get("false_positive_rate_pct", 0))}
- False Negative Count: {_safe(metrics.get("false_negative_count"), 0)}
- False Negative Rate: {_pct(metrics.get("false_negative_rate_pct", 0))}
- Block Precision: {_pct(metrics.get("block_precision_pct", 0))}
- Block Recall: {_pct(metrics.get("block_recall_pct", 0))}
- Freeze Recall: {_pct(metrics.get("freeze_recall_pct", 0))}
"""


def _render_operational_breakdown(report: Dict[str, Any]) -> str:
    metrics = report.get("metrics", {})
    expected = metrics.get("decision_distribution_expected", {})
    actual = metrics.get("decision_distribution_actual", {})

    return f"""## 3. Operational Breakdown

### Expected Distribution
- ALLOW: {_safe(expected.get("ALLOW"), 0)}
- FREEZE: {_safe(expected.get("FREEZE"), 0)}
- BLOCK: {_safe(expected.get("BLOCK"), 0)}

### Actual Distribution
- ALLOW: {_safe(actual.get("ALLOW"), 0)}
- FREEZE: {_safe(actual.get("FREEZE"), 0)}
- BLOCK: {_safe(actual.get("BLOCK"), 0)}
- UNKNOWN: {_safe(actual.get("UNKNOWN"), 0)}

### Category Quality
- Benign Total: {_safe(metrics.get("benign_total"), 0)}
- Benign Correct Allow: {_safe(metrics.get("benign_correct_allow"), 0)}
- Malicious Total: {_safe(metrics.get("malicious_total"), 0)}
- Malicious Correct Block: {_safe(metrics.get("malicious_correct_block"), 0)}
- Ambiguous Total: {_safe(metrics.get("ambiguous_total"), 0)}
- Ambiguous Correct Freeze: {_safe(metrics.get("ambiguous_correct_freeze"), 0)}
"""


def _render_case_outcomes(report: Dict[str, Any]) -> str:
    outcomes = report.get("outcomes", [])
    lines: List[str] = []
    lines.append("## 4. Case-by-Case Outcomes")

    if not outcomes:
        lines.append("- No outcomes recorded.")
        return "\n".join(lines)

    for item in outcomes:
        lines.append(f"### {item.get('case_id', 'unknown_case')}")
        lines.append(f"- Category: {_safe(item.get('category'))}")
        lines.append(f"- Expected: {_safe(item.get('expected'))}")
        lines.append(f"- Actual: {_safe(item.get('actual'))}")
        lines.append(f"- Match: {_safe(item.get('match'))}")
        lines.append(f"- False Positive: {_safe(item.get('false_positive'))}")
        lines.append(f"- False Negative: {_safe(item.get('false_negative'))}")
        lines.append(f"- Summary: {_safe(item.get('result_summary'))}")
        lines.append("")

    return "\n".join(lines).rstrip() + "\n"


def _render_preliminary_verdict(report: Dict[str, Any]) -> str:
    metrics = report.get("metrics", {})
    fp = float(metrics.get("false_positive_rate_pct", 0.0) or 0.0)
    fn = float(metrics.get("false_negative_rate_pct", 0.0) or 0.0)
    accuracy = float(metrics.get("accuracy_pct", 0.0) or 0.0)

    if accuracy >= 85.0 and fp <= 15.0 and fn <= 15.0:
        verdict = "Validation posture is promising for continued bank-grade hardening."
    elif accuracy >= 70.0 and fp <= 25.0 and fn <= 25.0:
        verdict = "Validation posture is directionally acceptable, but more tuning is required before customer-facing claims."
    else:
        verdict = "Validation posture indicates meaningful tuning is still required before bank-facing production claims."

    return f"""## 5. Preliminary Technical Verdict
{verdict}

## 6. Notes
- This validation report reflects the current Sovereign bank runtime path.
- False positive performance is especially important for banking trust and operator adoption.
- False negative performance is especially important for risk containment and control credibility.
- This report is a technical validation artifact, not yet a final regulatory or procurement submission.
"""


def build_false_positive_markdown(report: Dict[str, Any]) -> str:
    parts = [
        _render_summary(report),
        _render_operational_breakdown(report),
        _render_case_outcomes(report),
        _render_preliminary_verdict(report),
    ]
    return "\n\n".join(parts).strip() + "\n"


def save_false_positive_report(
    report: Dict[str, Any],
    output_dir: str = FALSE_POSITIVE_OUTPUT_DIR,
    base_name: Optional[str] = None,
) -> Dict[str, str]:
    _ensure_dir(output_dir)

    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    base = base_name or f"sovereign_validation_report_{ts}"

    json_path = os.path.join(output_dir, f"{base}.json")
    md_path = os.path.join(output_dir, f"{base}.md")

    with open(json_path, "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=2)

    markdown = build_false_positive_markdown(report)
    with open(md_path, "w", encoding="utf-8") as f:
        f.write(markdown)

    return {
        "json_report": json_path,
        "markdown_report": md_path,
    }


def generate_false_positive_report(
    output_dir: str = FALSE_POSITIVE_OUTPUT_DIR,
    base_name: Optional[str] = None,
) -> Dict[str, Any]:
    report = run_false_positive_suite()
    files = save_false_positive_report(
        report=report,
        output_dir=output_dir,
        base_name=base_name,
    )

    return {
        "ok": True,
        "generated_at": _utc_now(),
        "report": report,
        "files": files,
    }


if __name__ == "__main__":
    out = generate_false_positive_report()
    print(json.dumps(out, indent=2, ensure_ascii=False))