v2 scorer: deep validation — corrects inflated v1 scores

benchmark/score_challenge.py

@@ -1,54 +1,31 @@
 #!/usr/bin/env python3
 """
-Agent Benchmark Scorer — Automated scoring for real-world agent challenges.
-Scores results directory against a rubric of pass/fail criteria.
+Agent Benchmark Scorer v2 — Deep validation, not keyword matching.
+
+v1 inflated scores by detecting keywords without verifying correctness.
+v2 validates actual data: are the numbers real? Are services running?
+Does the report contain verifiable facts?
 
 Usage:
   python3 score_challenge.py --challenge 1 --results-dir /tmp/agent-results-claude-ch1
-  python3 score_challenge.py --all --model-name claude
 """
 
 import argparse
 import json
 import os
 import re
-import sys
+import subprocess
 
 
-def check_file_exists(results_dir, patterns):
-    """Check if any file matching patterns exists."""
-    for f in os.listdir(results_dir):
-        for p in patterns:
-            if re.match(p, f, re.IGNORECASE):
-                return True, f
-    return False, None
-
-
-def check_file_size(results_dir, patterns, min_bytes=100):
-    """Check if matching file exists and is at least min_bytes."""
+def read_file(results_dir, patterns, max_size=500000):
+    """Read first matching file content."""
     for f in os.listdir(results_dir):
         for p in patterns:
             if re.match(p, f, re.IGNORECASE):
                 path = os.path.join(results_dir, f)
-                size = os.path.getsize(path)
-                if size >= min_bytes:
-                    return True, f, size
-    return False, None, 0
-
-
-def check_file_contains(results_dir, patterns, keywords):
-    """Check if file contains any of the keywords."""
-    for f in os.listdir(results_dir):
-        for p in patterns:
-            if re.match(p, f, re.IGNORECASE):
-                path = os.path.join(results_dir, f)
-                try:
-                    content = open(path).read().lower()
-                    found = [k for k in keywords if k.lower() in content]
-                    return len(found) > 0, f, found
-                except:
-                    pass
-    return False, None, []
+                if os.path.getsize(path) <= max_size:
+                    return open(path).read(), f
+    return "", None
 
 
 def detect_antipatterns(results_dir):
@@ -61,7 +38,6 @@ def detect_antipatterns(results_dir):
     data = json.load(open(log_path))
     log = data.get("log", [])
 
-    # Anti-pattern 1: Repeated identical commands
     cmd_counts = {}
     for entry in log:
         if entry.get("tool") == "bash":
@@ -71,191 +47,334 @@ def detect_antipatterns(results_dir):
         if count >= 3:
             issues.append(f"LOOP: '{cmd[:60]}...' repeated {count} times")
 
-    # Anti-pattern 2: Error blindness (same error, no adaptation)
     errors = []
     for entry in log:
         result = entry.get("result", "")
-        if "error" in result.lower() or "not found" in result.lower() or "permission denied" in result.lower():
+        if any(k in result.lower() for k in ["error", "not found", "permission denied", "command not found"]):
             errors.append(result[:100])
     if len(errors) >= 3:
         unique_errors = set(errors)
         if len(unique_errors) <= 2:
-            issues.append(f"ERROR_BLIND: {len(errors)} errors, only {len(unique_errors)} unique — not adapting")
+            issues.append(f"ERROR_BLIND: {len(errors)} errors, only {len(unique_errors)} unique")
 
-    # Anti-pattern 3: No tool calls at all
     if len(log) == 0:
-        issues.append("NO_TOOLS: Agent made zero tool calls")
-
-    # Anti-pattern 4: Only bash, never write_file
-    tools_used = set(e.get("tool") for e in log)
-    if "bash" in tools_used and "write_file" not in tools_used:
-        # Not necessarily bad, but notable
-        pass
+        issues.append("NO_TOOLS: Zero tool calls")
 
     return issues
 
 
-# ── Challenge Scoring Rubrics ───────────────────────────────────────────────
+# ── CH1: Bioinformatics ─────────────────────────────────────────────────────
 
 def score_challenge_1(results_dir):
-    """Bioinformatics: P53_HUMAN analysis"""
+    """Bioinformatics: P53_HUMAN — deep validation."""
     scores = {}
     total = 0
 
-    # Criterion 1: Downloaded protein sequence (0-1)
-    ok, f = check_file_exists(results_dir, [r".*\.fasta", r".*p53.*", r"P04637.*"])
-    scores["sequence_downloaded"] = 1 if ok else 0
+    # 1. Sequence downloaded (0-1): FASTA file with actual protein sequence
+    fasta, _ = read_file(results_dir, [r".*\.fasta"])
+    has_fasta = len(fasta) > 100 and ("MEEPQ" in fasta or "P04637" in fasta or ">sp|" in fasta)
+    scores["sequence_downloaded"] = 1 if has_fasta else 0
     total += scores["sequence_downloaded"]
 
-    # Criterion 2: Secondary structure predicted (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json", r".*\.tsv", r".*\.txt"],
-        ["helix", "strand", "coil", "sheet", "turn", "secondary structure"])
-    ss_count = len(keywords) if ok else 0
-    scores["secondary_structure"] = min(2, ss_count)
+    # 2. Secondary structure (0-2): must have REAL percentages (not 0% or N/A)
+    html, html_file = read_file(results_dir, [r".*\.html"])
+    tsv, _ = read_file(results_dir, [r".*\.tsv"])
+    all_text = (html + tsv).lower()
+
+    # Look for real percentages near structure keywords
+    # Correct values: Helix ~22%, Strand ~25%, Coil ~46%
+    ss_score = 0
+    helix_match = re.search(r'helix[^0-9]*(\d+\.?\d*)%', all_text)
+    strand_match = re.search(r'(?:strand|sheet|beta)[^0-9]*(\d+\.?\d*)%', all_text)
+
+    if helix_match:
+        val = float(helix_match.group(1))
+        if 10 < val < 40:  # reasonable range for real data
+            ss_score += 1
+    if strand_match:
+        val = float(strand_match.group(1))
+        if 10 < val < 40:
+            ss_score += 1
+
+    # Penalize "0.00%" or "N/A" — these mean the parser failed
+    if "0.00%" in all_text and ("helix" in all_text or "strand" in all_text):
+        ss_score = 0  # parser failed, data is wrong
+    if "n/a" in all_text and "secondary" in all_text:
+        ss_score = 0
+
+    scores["secondary_structure"] = min(2, ss_score)
     total += scores["secondary_structure"]
 
-    # Criterion 3: Cancer mutations found (0-3)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json", r".*\.tsv"],
-        ["r175h", "r248w", "r273h", "r249s", "g245s", "r282w",
-         "cancer", "mutation", "hotspot", "li-fraumeni"])
-    mut_count = len(keywords) if ok else 0
-    scores["cancer_mutations"] = min(3, mut_count)
+    # 3. Cancer mutations (0-3): must have REAL hotspot positions with details
+    mut_score = 0
+    real_hotspots = ["R175H", "R248W", "R248Q", "R273H", "R273C", "R249S", "G245S", "R282W", "Y220C", "C176F"]
+    found_hotspots = [h for h in real_hotspots if h.lower() in all_text.lower() or h in html]
+    mut_score += min(2, len(found_hotspots))
+
+    # PubMed references = verified data
+    pubmed = re.findall(r'(?:PMID|pubmed)[:\s]*(\d{6,})', all_text, re.IGNORECASE)
+    if len(pubmed) >= 2:
+        mut_score += 1
+
+    # Penalize generic "cancer" keyword without specific mutations
+    if mut_score == 0 and "cancer" in all_text:
+        # Has the word but no actual data — don't give credit
+        pass
+
+    scores["cancer_mutations"] = min(3, mut_score)
     total += scores["cancer_mutations"]
 
-    # Criterion 4: HTML report generated (0-2)
-    ok, f, size = check_file_size(results_dir, [r".*\.html"], min_bytes=1000)
-    scores["html_report"] = 2 if (ok and size > 5000) else (1 if ok else 0)
+    # 4. HTML report (0-2): must be substantial AND contain real data
+    html_score = 0
+    if html_file:
+        html_size = os.path.getsize(os.path.join(results_dir, html_file))
+        if html_size > 2000:
+            html_score = 1
+        # Bonus for charts/visualizations
+        if any(k in html.lower() for k in ["chart.js", "<canvas", "<svg", "plotly"]):
+            html_score = 2
+        # But penalize if the HTML is big but data is empty/wrong
+        if html_size > 5000 and ss_score == 0 and len(found_hotspots) == 0:
+            html_score = max(0, html_score - 1)  # big but empty = less credit
+
+    scores["html_report"] = html_score
     total += scores["html_report"]
 
-    # Criterion 5: Data is real/verifiable (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json", r".*\.tsv"],
-        ["p04637", "uniprot", "pubmed", "pmid", "p53_human", "tp53"])
-    ref_count = len(keywords) if ok else 0
-    scores["verifiable_data"] = min(2, ref_count)
+    # 5. Verifiable data (0-2): real references, not just keywords
+    verify_score = 0
+    if "P04637" in html or "P04637" in tsv:
+        verify_score += 1
+    if len(pubmed) >= 1 or len(found_hotspots) >= 3:
+        verify_score += 1
+
+    scores["verifiable_data"] = min(2, verify_score)
     total += scores["verifiable_data"]
 
     return scores, total, 10
 
 
+# ── CH2: Security CTF ────────────────────────────────────────────────────────
+
 def score_challenge_2(results_dir):
-    """Security CTF: DVWA exploitation"""
+    """Security CTF: DVWA — verify actual exploitation proof."""
     scores = {}
     total = 0
 
-    # Criterion 1: DVWA set up (0-1)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json", r".*\.txt"],
-        ["dvwa", "damn vulnerable", "web application"])
-    scores["dvwa_setup"] = 1 if ok else 0
+    html, _ = read_file(results_dir, [r".*\.html", r".*report.*"])
+    all_text = html.lower()
+    json_text, _ = read_file(results_dir, [r".*\.json"])
+    if json_text:
+        all_text += json_text.lower()
+
+    # 1. DVWA setup (0-1): must show DVWA actually running, not just mentioned
+    dvwa_running = any(k in all_text for k in ["welcome to damn vulnerable", "dvwa setup", "database setup"])
+    scores["dvwa_setup"] = 1 if dvwa_running else 0
     total += scores["dvwa_setup"]
 
-    # Criterion 2: SQL Injection (0-3)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json"],
-        ["sql injection", "union select", "password", "admin", "sqli"])
-    scores["sql_injection"] = min(3, len(keywords) if ok else 0)
+    # 2. SQL Injection (0-3): must show EXTRACTED data (actual password hashes)
+    sqli_score = 0
+    # Real DVWA password hashes
+    real_hashes = ["5f4dcc3b5aa765d61d8327deb882cf99", "e99a18c428cb38d5f260853678922e03",
+                   "8d3533d75ae2c3966d7e0d4fcc69216b"]
+    has_hashes = any(h in all_text for h in real_hashes)
+    has_union = "union select" in all_text or "union%20select" in all_text
+    if has_hashes:
+        sqli_score = 3  # extracted actual data
+    elif has_union:
+        sqli_score = 1  # showed the payload but no proof
+
+    scores["sql_injection"] = sqli_score
     total += scores["sql_injection"]
 
-    # Criterion 3: XSS (0-3)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json"],
-        ["xss", "cross-site", "script", "reflected", "alert"])
-    scores["xss"] = min(3, len(keywords) if ok else 0)
+    # 3. XSS (0-3): must show reflected payload in response
+    xss_score = 0
+    has_xss_payload = "<script>" in html or "alert(" in all_text or "&lt;script&gt;" in all_text
+    has_xss_proof = "reflected" in all_text and ("xss" in all_text or "script" in all_text)
+    if has_xss_payload and has_xss_proof:
+        xss_score = 3
+    elif has_xss_payload:
+        xss_score = 1
+
+    scores["xss"] = xss_score
     total += scores["xss"]
 
-    # Criterion 4: Command Injection (0-3)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json"],
-        ["command injection", "whoami", "passwd", "rce", "os command"])
-    scores["command_injection"] = min(3, len(keywords) if ok else 0)
+    # 4. Command Injection (0-3): must show actual command output
+    cmdi_score = 0
+    has_cmd_output = any(k in all_text for k in ["www-data", "uid=", "root:x:0:0"])
+    has_cmd_payload = ";" in all_text and ("whoami" in all_text or "cat /etc" in all_text)
+    if has_cmd_output:
+        cmdi_score = 3  # proved execution
+    elif has_cmd_payload:
+        cmdi_score = 1
+
+    scores["command_injection"] = cmdi_score
     total += scores["command_injection"]
 
     return scores, total, 10
 
 
+# ── CH3: Data Engineering ────────────────────────────────────────────────────
+
 def score_challenge_3(results_dir):
-    """Data Engineering: NYC Taxi pipeline"""
+    """Data Engineering: NYC Taxi — verify real data in output."""
     scores = {}
     total = 0
 
-    # Criterion 1: Data downloaded (0-1)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json", r".*\.py", r".*\.txt"],
-        ["taxi", "trip", "yellow", "nyc", "tlc"])
-    scores["data_downloaded"] = 1 if ok else 0
+    html, html_file = read_file(results_dir, [r".*\.html", r"dashboard.*"])
+    json_text, _ = read_file(results_dir, [r".*\.json"])
+    py, _ = read_file(results_dir, [r".*\.py"])
+    all_text = (html + json_text + py).lower()
+
+    # 1. Data downloaded (0-1): evidence of real taxi data (row counts, column names)
+    real_data_evidence = any(k in all_text for k in [
+        "2964624", "2,964,624",  # exact row count of Jan 2024
+        "yellow_tripdata", "tpep_pickup", "fare_amount",
+        "vendorid", "passenger_count"
+    ])
+    scores["data_downloaded"] = 1 if real_data_evidence else 0
     total += scores["data_downloaded"]
 
-    # Criterion 2: Data cleaned (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json", r".*\.py"],
-        ["clean", "outlier", "null", "missing", "filter", "dropna"])
-    scores["data_cleaned"] = min(2, len(keywords) if ok else 0)
+    # 2. Data cleaned (0-2): evidence of actual cleaning operations
+    clean_score = 0
+    clean_evidence = ["dropna", "outlier", "< 0", "> 0", "null", "missing", "filter",
+                      "trip_distance", "fare_amount"]
+    found_clean = [k for k in clean_evidence if k in all_text]
+    clean_score = min(2, len(found_clean) // 2)  # need at least 2 evidences per point
+    scores["data_cleaned"] = clean_score
     total += scores["data_cleaned"]
 
-    # Criterion 3: Analytics computed (0-3)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.html", r".*\.json"],
-        ["busiest", "hour", "fare", "distance", "tip", "payment",
-         "average", "analytics", "peak"])
-    scores["analytics"] = min(3, len(keywords) if ok else 0)
+    # 3. Analytics (0-3): verify REAL numbers in output
+    analytics_score = 0
+
+    # Busiest hours: should have numbers >10000 for NYC taxi
+    big_numbers = re.findall(r'\b(\d{4,6})\b', html + json_text)
+    big_nums = [int(n) for n in big_numbers if 10000 < int(n) < 500000]
+    if len(big_nums) >= 5:
+        analytics_score += 1  # has real trip counts
+
+    # Fare data: should have dollar amounts $5-$100
+    fare_numbers = re.findall(r'(?:\$|fare[^0-9]*)(\d+\.?\d{0,2})', all_text)
+    real_fares = [float(f) for f in fare_numbers if 5 < float(f) < 200]
+    if len(real_fares) >= 3:
+        analytics_score += 1  # has real fare data
+
+    # Tip patterns: should show tip differences by payment type
+    if "tip" in all_text and ("payment" in all_text or "credit" in all_text or "cash" in all_text):
+        analytics_score += 1
+
+    scores["analytics"] = min(3, analytics_score)
     total += scores["analytics"]
 
-    # Criterion 4: Interactive dashboard (0-3)
-    ok, f, size = check_file_size(results_dir, [r".*\.html"], min_bytes=2000)
-    has_charts = False
-    if ok:
-        content = open(os.path.join(results_dir, f)).read().lower()
-        has_charts = any(k in content for k in ["chart.js", "plotly", "canvas", "svg", "d3"])
-    scores["dashboard"] = 3 if (ok and has_charts) else (2 if ok else 0)
+    # 4. Dashboard (0-3): interactive charts with real data
+    dash_score = 0
+    if html_file:
+        html_size = os.path.getsize(os.path.join(results_dir, html_file))
+        has_charts = any(k in html.lower() for k in ["chart.js", "<canvas", "plotly", "new chart("])
+        canvas_count = html.lower().count("<canvas")
+
+        if has_charts and canvas_count >= 3 and html_size > 5000:
+            dash_score = 3  # multiple charts, substantial
+        elif has_charts and html_size > 2000:
+            dash_score = 2
+        elif html_size > 1000:
+            dash_score = 1
+
+    scores["dashboard"] = dash_score
     total += scores["dashboard"]
 
-    # Criterion 5: Reproducible pipeline (0-1)
-    ok, f = check_file_exists(results_dir, [r".*\.py", r".*\.sh", r"pipeline.*"])
-    scores["reproducible"] = 1 if ok else 0
+    # 5. Reproducible (0-1): a script that could re-run the pipeline
+    has_script = any(
+        os.path.exists(os.path.join(results_dir, f))
+        for f in os.listdir(results_dir)
+        if f.endswith('.py') or f.endswith('.sh')
+    )
+    scores["reproducible"] = 1 if has_script else 0
     total += scores["reproducible"]
 
     return scores, total, 10
 
 
+# ── CH4: DevOps ──────────────────────────────────────────────────────────────
+
 def score_challenge_4(results_dir):
-    """DevOps: Monitored web app stack"""
+    """DevOps: Monitored stack — verify configs are correct and services described."""
     scores = {}
     total = 0
 
-    # Criterion 1: Web app created (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.py", r".*\.html", r".*\.json", r".*\.txt"],
-        ["flask", "fastapi", "uvicorn", "app.py", "web app", "endpoint"])
-    scores["web_app"] = min(2, len(keywords) if ok else 0)
+    # Read all files
+    files = {}
+    for f in os.listdir(results_dir):
+        path = os.path.join(results_dir, f)
+        if os.path.isfile(path) and os.path.getsize(path) < 100000:
+            files[f] = open(path).read()
+
+    all_text = " ".join(files.values()).lower()
+
+    # 1. Web app (0-2): must have a real Flask/FastAPI app with routes
+    app_score = 0
+    for name, content in files.items():
+        if name.endswith('.py'):
+            has_flask = "flask" in content.lower() or "fastapi" in content.lower()
+            has_route = "@app.route" in content or "@app.get" in content
+            has_metrics = "metrics" in content.lower() or "counter" in content.lower() or "prometheus" in content.lower()
+            if has_flask and has_route:
+                app_score = 1
+                if has_metrics:
+                    app_score = 2
+    scores["web_app"] = app_score
     total += scores["web_app"]
 
-    # Criterion 2: Nginx configured (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.conf", r".*\.html", r".*\.txt", r"nginx.*"],
-        ["nginx", "proxy_pass", "reverse proxy", "upstream"])
-    scores["nginx"] = min(2, len(keywords) if ok else 0)
+    # 2. Nginx (0-2): must have valid config with proxy_pass
+    nginx_score = 0
+    for name, content in files.items():
+        if "nginx" in name.lower() or name.endswith('.conf'):
+            has_listen = "listen" in content
+            has_proxy = "proxy_pass" in content
+            has_location = "location" in content
+            if has_listen and has_proxy and has_location:
+                nginx_score = 2
+            elif has_proxy or has_listen:
+                nginx_score = 1
+    scores["nginx"] = nginx_score
     total += scores["nginx"]
 
-    # Criterion 3: Prometheus metrics (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.yml", r".*\.yaml", r".*\.py", r".*\.html"],
-        ["prometheus", "metrics", "scrape", "counter", "histogram"])
-    scores["prometheus"] = min(2, len(keywords) if ok else 0)
+    # 3. Prometheus (0-2): must have valid scrape config
+    prom_score = 0
+    for name, content in files.items():
+        if name.endswith('.yml') or name.endswith('.yaml'):
+            has_scrape = "scrape_configs" in content or "scrape_interval" in content
+            has_targets = "targets" in content
+            if has_scrape and has_targets:
+                prom_score = 2
+            elif has_scrape or "prometheus" in content.lower():
+                prom_score = 1
+    scores["prometheus"] = prom_score
     total += scores["prometheus"]
 
-    # Criterion 4: Health check (0-2)
-    ok, f, keywords = check_file_contains(results_dir,
-        [r".*\.sh", r".*\.py", r".*\.html"],
-        ["health", "check", "restart", "monitor", "watchdog"])
-    scores["health_check"] = min(2, len(keywords) if ok else 0)
+    # 4. Health check (0-2): must have a script that checks and restarts
+    health_score = 0
+    for name, content in files.items():
+        if "health" in name.lower() or name.endswith('.sh'):
+            has_check = "curl" in content or "wget" in content or "request" in content.lower()
+            has_restart = "restart" in content.lower() or "kill" in content or "start" in content.lower()
+            if has_check and has_restart:
+                health_score = 2
+            elif has_check or has_restart:
+                health_score = 1
+    scores["health_check"] = health_score
     total += scores["health_check"]
 
-    # Criterion 5: Status report (0-2)
-    ok, f, size = check_file_size(results_dir, [r".*\.html"], min_bytes=500)
-    scores["status_report"] = 2 if ok else 0
+    # 5. Status report (0-2): HTML with architecture description
+    status_score = 0
+    for name, content in files.items():
+        if name.endswith('.html'):
+            size = len(content)
+            has_arch = any(k in content.lower() for k in ["architecture", "flask", "nginx", "prometheus", "service"])
+            if size > 1000 and has_arch:
+                status_score = 2
+            elif size > 500:
+                status_score = 1
+    scores["status_report"] = status_score
     total += scores["status_report"]
 
     return scores, total, 10
@@ -275,7 +394,6 @@ def score_one(challenge, results_dir, model_name=""):
     scores, total, max_score = scorer(results_dir)
     antipatterns = detect_antipatterns(results_dir)
 
-    # Approach penalty for anti-patterns
     penalty = min(3, len(antipatterns))
     final = max(0, total - penalty)
 
@@ -286,15 +404,15 @@ def score_one(challenge, results_dir, model_name=""):
     print(f"{'='*50}")
 
     for criterion, value in scores.items():
-        status = "✓" if value > 0 else "✗"
+        status = "+" if value > 0 else "-"
         print(f"  {status} {criterion}: {value}")
 
     print(f"\n  Subtotal: {total}/{max_score}")
 
     if antipatterns:
-        print(f"\n  Anti-patterns detected (-{penalty}):")
+        print(f"\n  Anti-patterns (-{penalty}):")
         for ap in antipatterns:
-            print(f"    ⚠ {ap}")
+            print(f"    ! {ap}")
 
     print(f"\n  FINAL SCORE: {final}/{max_score}")
     return {"challenge": challenge, "name": name, "model": model_name,
@@ -303,71 +421,22 @@ def score_one(challenge, results_dir, model_name=""):
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Score agent benchmark challenges")
+    parser = argparse.ArgumentParser(description="Score agent benchmark challenges (v2 — deep validation)")
     parser.add_argument("--challenge", type=int, help="Challenge number (1-4)")
-    parser.add_argument("--all", action="store_true", help="Score all 4 challenges")
+    parser.add_argument("--all", action="store_true")
     parser.add_argument("--results-dir", help="Results directory")
-    parser.add_argument("--model-name", default="", help="Model name for display")
-    parser.add_argument("--compare", nargs=2, metavar=("MODEL1", "MODEL2"),
-                       help="Compare two models (e.g., --compare claude e4b)")
+    parser.add_argument("--model-name", default="")
     args = parser.parse_args()
 
-    if args.compare:
-        m1, m2 = args.compare
-        all_results = []
-        for ch in range(1, 5):
-            for model in [m1, m2]:
-                rdir = f"/tmp/agent-results-{model}-ch{ch}"
-                if os.path.exists(rdir):
-                    result = score_one(ch, rdir, model)
-                    all_results.append(result)
-
-        # Summary table
-        print(f"\n{'='*60}")
-        print(f"  COMPARISON SUMMARY: {m1} vs {m2}")
-        print(f"{'='*60}")
-        print(f"  {'Challenge':<25} {m1:>10} {m2:>10}")
-        print(f"  {'-'*45}")
-
-        totals = {m1: 0, m2: 0}
-        for ch in range(1, 5):
-            name = SCORERS[ch][0]
-            scores_m1 = [r for r in all_results if r["challenge"] == ch and r["model"] == m1]
-            scores_m2 = [r for r in all_results if r["challenge"] == ch and r["model"] == m2]
-            s1 = scores_m1[0]["final"] if scores_m1 else "-"
-            s2 = scores_m2[0]["final"] if scores_m2 else "-"
-            if isinstance(s1, int): totals[m1] += s1
-            if isinstance(s2, int): totals[m2] += s2
-            print(f"  {ch}. {name:<22} {str(s1):>10} {str(s2):>10}")
-
-        print(f"  {'-'*45}")
-        print(f"  {'TOTAL':<25} {totals[m1]:>10} {totals[m2]:>10}")
-        print(f"  {'Max possible':<25} {'40':>10} {'40':>10}")
-
-        # Save comparison
-        comparison = {
-            "models": [m1, m2],
-            "results": all_results,
-            "totals": totals,
-        }
-        out_path = f"/tmp/agent-benchmark-{m1}-vs-{m2}.json"
-        json.dump(comparison, open(out_path, "w"), indent=2, default=str)
-        print(f"\n  Saved to {out_path}")
-
-    elif args.all:
+    if args.all:
         for ch in range(1, 5):
             rdir = args.results_dir or f"/tmp/agent-results-{args.model_name or 'unknown'}-ch{ch}"
             if os.path.exists(rdir):
                 score_one(ch, rdir, args.model_name)
-            else:
-                print(f"\n  Challenge {ch}: No results found at {rdir}")
-
     elif args.challenge:
-        rdir = args.results_dir or f"/tmp/agent-results-{args.model_name or 'unknown'}-ch{args.challenge}"
+        rdir = args.results_dir or f"/tmp/agent-results-{args.model_name}-ch{args.challenge}"
         if os.path.exists(rdir):
             score_one(args.challenge, rdir, args.model_name)
-        else:
-            print(f"Results directory not found: {rdir}")
     else:
         parser.print_help()