test_grader_exhaustive.py

"""Exhaustive local test of ALL grader paths to find 0.0 or 1.0 scores."""
from environment.graders import grade
from environment.env import VoiceAuthenticityEnv, TASKS, DIFFICULTY_MAP

violations = []
total = 0

difficulties = ["easy", "medium", "medium_hard", "hard", "extreme"]

# All possible action histories
action_histories = [
    ["final_classify"],
    ["request_temporal_features", "final_classify"],
    ["request_spectral_features", "final_classify"],
    ["request_comparison", "final_classify"],
    ["analyze_evidence", "final_classify"],
    ["request_temporal_features", "request_spectral_features", "final_classify"],
    ["request_temporal_features", "request_spectral_features", "request_comparison", "final_classify"],
    ["request_temporal_features", "request_spectral_features", "request_comparison", "analyze_evidence", "final_classify"],
    ["request_temporal_features", "analyze_evidence", "final_classify"],
    ["analyze_evidence", "request_temporal_features", "final_classify"],
]

labels = [0, 1]
true_labels = [0, 1]
confidences = [0.0, 0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.99, 1.0]
reasonings = [
    "",
    "test",
    "real human natural jitter",
    "synthetic fake generated smooth",
    "real but also synthetic",
    "no keywords here at all just random text padding to exceed minimum length",
]

for diff in difficulties:
    for tl in true_labels:
        for pl in labels:
            for conf in confidences:
                for reasoning in reasonings:
                    for history in action_histories:
                        action = {"label": pl, "confidence": conf, "reasoning": reasoning}
                        result = grade(tl, action, diff, history)
                        score = result["score"]
                        total += 1
                        if score <= 0.0 or score >= 1.0:
                            violations.append({
                                "score": score,
                                "true_label": tl,
                                "pred_label": pl,
                                "confidence": conf,
                                "difficulty": diff,
                                "reasoning": reasoning[:30],
                                "history": history,
                            })

# Also test via the environment step() directly
print("Testing via environment step()...")
env_violations = []
for task in TASKS:
    env = VoiceAuthenticityEnv(task)
    for seed in range(20):
        env.reset(seed=seed)
        
        # Test jump-to-classify
        for label in [0, 1]:
            for conf in [0.0, 0.5, 1.0]:
                env.reset(seed=seed)
                obs, reward, done, info = env.step({
                    "action_type": "final_classify",
                    "label": label,
                    "confidence": conf,
                    "reasoning": "test reasoning text"
                })
                total += 1
                if reward <= 0.0 or reward >= 1.0:
                    env_violations.append(f"task={task} seed={seed} label={label} conf={conf} reward={reward}")

        # Test full protocol
        env.reset(seed=seed)
        obs, r1, _, _ = env.step({"action_type": "request_temporal_features"})
        total += 1
        if r1 <= 0.0 or r1 >= 1.0:
            env_violations.append(f"temporal task={task} seed={seed} reward={r1}")
        
        obs, r2, _, _ = env.step({"action_type": "request_spectral_features"})
        total += 1
        if r2 <= 0.0 or r2 >= 1.0:
            env_violations.append(f"spectral task={task} seed={seed} reward={r2}")
        
        obs, r3, _, _ = env.step({"action_type": "request_comparison"})
        total += 1
        if r3 <= 0.0 or r3 >= 1.0:
            env_violations.append(f"comparison task={task} seed={seed} reward={r3}")
        
        obs, r4, _, _ = env.step({"action_type": "analyze_evidence"})
        total += 1
        if r4 <= 0.0 or r4 >= 1.0:
            env_violations.append(f"analyze task={task} seed={seed} reward={r4}")
        
        obs, r5, done, info = env.step({
            "action_type": "final_classify",
            "label": 0, "confidence": 0.7,
            "reasoning": "natural speech with jitter variation"
        })
        total += 1
        if r5 <= 0.0 or r5 >= 1.0:
            env_violations.append(f"classify task={task} seed={seed} reward={r5}")

print(f"\nTested {total} combinations")
print(f"\nGrader violations: {len(violations)}")
for v in violations[:20]:
    print(f"  {v}")
print(f"\nEnv step violations: {len(env_violations)}")
for v in env_violations[:20]:
    print(f"  {v}")

if not violations and not env_violations:
    print("\nALL SCORES STRICTLY IN (0, 1) - PASS")
else:
    print("\nFAILED - found violations!")