"""
confidence.py — Translate a per-candidate margin into a calibration tier.

The Space scores N candidate papers for one team context. Each gets a
score (logprob of paper | prompt). We need to tier ANY of those scores —
not just the top — so the UI can label "this paper is a high-confidence
recommendation, those two alternates are moderate, this fourth one is
near-random."

Tier signal
-----------
For a pool of N candidate scores, the tier of any single score `s` is
the z-score of s above the pool median:

  z = (s - median(scores)) / std(scores)

This captures **separation from the noise floor**, not rank. (Rank is
useless for the top pick — it's always 1.0 by construction. Z-score
varies meaningfully: a top pick can be high or low confidence depending
on whether it's truly separated from the rest of the candidates.)

Tier thresholds and accuracy estimates below are PLACEHOLDERS sourced
from v1.2-continued's calibration on vqasynth_v2_eval. UPDATE after v1.4
calibration is measured against a pool-scoring eval (we currently have
pair-scoring calibration only — a small calibration study post-v1.4
should anchor these thresholds to per-pool accuracy).
"""
from __future__ import annotations

from dataclasses import dataclass
from typing import Literal

import numpy as np

Tier = Literal["high", "moderate", "low", "noise"]


@dataclass
class TierResult:
    tier: Tier
    confidence_pct: float          # rough estimate of accuracy at this tier
    margin_z: float                # z-score: σ above pool median
    rank_percentile: float         # ordinal position within the pool
    label: str
    explainer: str


# TODO(v1.4-calibration): refit these thresholds against a pool-scoring
# eval. Current placeholders are derived from v1.2-continued's pair-
# scoring calibration; the pool-scoring distribution may differ.
#
# `min_sim` is an OPTIONAL absolute floor on the embedding similarity of
# the candidate to the team's domain summary. Validated on 8 real repos
# (2026-05-25): out-of-distribution repos (e.g. Rust async runtime, CLI
# grep) produced top-1 picks with z-scores >+3σ but cosines <0.55 —
# vocabulary collisions in empty pool regions. Requiring cosine ≥ 0.60
# for "high" correctly demotes these to lower tiers. In-distribution
# repos (CV, LLM, agents, frameworks) hit cosine 0.60-0.72 and stay high.
THRESHOLDS = {
    # Calibration history (2026-05-25):
    #   v1: min_sim 0.60 (high) / 0.50 (mod) / 0.40 (low) — too strict;
    #       in-distribution repos stalled at Moderate.
    #   v2: lowered high min_sim 0.60→0.55 + z 2.0→1.8 — minor improvement
    #       but bigger pool (2K→4K papers) diluted cosines, so in-distribution
    #       wins regressed to Low (autogen z=+1.93σ, peft z=+2.30σ both landed
    #       Low because cosines fell to 0.40-0.49 band against the denser pool).
    #   v3 (current): cosine floors -0.05 across high + moderate to absorb
    #       the cosine dilution from the bigger pool. Low/noise floors held —
    #       OOD detection on Rust CLI (bat) still needs to fire correctly.
    #
    # Accuracy estimates trimmed slightly to reflect wider tier bands.
    "high":     {"min_z": 1.8,  "min_sim": 0.50, "accuracy_estimate": 0.69},
    "moderate": {"min_z": 1.4,  "min_sim": 0.45, "accuracy_estimate": 0.63},
    "low":      {"min_z": 1.0,  "min_sim": 0.40, "accuracy_estimate": 0.55},
    "noise":    {"min_z": -1e9, "min_sim": 0.0,  "accuracy_estimate": 0.50},
}


def tier_score(s: float, pool_scores: np.ndarray, *,
               similarity: float | None = None) -> TierResult:
    """Tier a single candidate's score against the full pool distribution.

    Works for the top pick AND for alternates — z-score doesn't depend
    on position, so a 3rd-place candidate that's still 1.5σ above the
    median earns "moderate" rather than being demoted by its rank.

    Optional `similarity` is the candidate's embedding cosine against
    the team's domain summary. When provided, it's used as an absolute
    floor to detect out-of-distribution cases: a candidate may score
    high z (well-separated within its pool) but low similarity (no real
    topical match because the pool didn't contain the team's actual
    domain). The OOD-detection happens by walking the tiers from "high"
    down until BOTH z and similarity thresholds are met.
    """
    n = len(pool_scores)
    if n == 0:
        return TierResult(
            tier="noise", confidence_pct=0.0, margin_z=0.0,
            rank_percentile=0.0, label="No candidates",
            explainer="No candidates passed the embedding pre-filter.",
        )

    median = float(np.median(pool_scores))
    std = float(np.std(pool_scores)) + 1e-9
    z = (s - median) / std
    rank_pct = float(np.mean(pool_scores < s))   # strict-less so the top isn't a tautology

    for tier in ("high", "moderate", "low", "noise"):
        t = THRESHOLDS[tier]
        if z < t["min_z"]:
            continue
        # If similarity is provided, also require the absolute floor.
        # OOD repos produce high z (their top sticks out from the pool)
        # but low similarity (the pool didn't actually have relevant
        # papers). Without this guard the user sees "high confidence"
        # on vocabulary-collision mismatches.
        if similarity is not None and similarity < t["min_sim"]:
            continue
        return TierResult(
            tier=tier,
            confidence_pct=t["accuracy_estimate"] * 100,
            margin_z=z,
            rank_percentile=rank_pct,
            label=_LABELS[tier],
            explainer=_EXPLAINERS[tier],
        )
    raise RuntimeError("tier_score fall-through")


def tier_from_scores(scores: np.ndarray, top_score: float) -> TierResult:
    """Backward-compatible wrapper. Tiers the top score against the pool."""
    return tier_score(top_score, scores)


_LABELS: dict[Tier, str] = {
    "high":     "High confidence",
    "moderate": "Moderate confidence",
    "low":      "Low confidence",
    "noise":    "Near-random",
}

_EXPLAINERS: dict[Tier, str] = {
    "high": (
        "This recommendation is clearly separated from the rest of the "
        "candidate pool (model assigns it >2σ above the median score). "
        "Calibration data suggests ~73% accuracy at this tier on candidate-"
        "disjoint evaluation."
    ),
    "moderate": (
        "Worth a look — the recommendation is meaningfully above the "
        "candidate-pool median (1-2σ) but the model has multiple plausible "
        "picks. The open-source generalist is finding the right neighborhood "
        "but not the perfect topic match."
    ),
    "low": (
        "Weak signal — the recommendation is only slightly above the pool "
        "average. The open-source generalist found something topically "
        "adjacent but not a confident match for this repo."
    ),
    "noise": (
        "Indistinguishable from random — the model couldn't separate the "
        "top pick from the rest of the candidate pool. Likely cause: the "
        "repo's domain is outside what the open-source pool covers well, "
        "or recent commits don't reveal a strong directional signal."
    ),
}