"""
Wikipedia Speedrun - Flask app for Hugging Face Spaces.
Features:
- Dashboard: Benchmark results with Pareto chart and leaderboard
- Play: Navigate Wikipedia with inline clickable links
- Watch AI: Spectate an AI agent playing live
"""
import os
import time
import json
import threading
import urllib.parse
from pathlib import Path
from dataclasses import dataclass, field
from flask import Flask, render_template_string, request, session, redirect, url_for, jsonify
import requests
from bs4 import BeautifulSoup
import numpy as np
app = Flask(__name__)
app.secret_key = os.environ.get("SECRET_KEY", "wiki-speedrun-dev-key")
# Session config for HF Spaces (behind reverse proxy)
app.config["SESSION_COOKIE_SAMESITE"] = "Lax"
app.config["SESSION_COOKIE_SECURE"] = False # HF proxy handles HTTPS
# ====================
# Data Loading
# ====================
def get_data_path() -> Path:
return Path(__file__).parent / "data"
def load_benchmark_data() -> dict:
data_path = get_data_path()
jsonl_path = data_path / "benchmark_results.jsonl"
if jsonl_path.exists():
results = []
with open(jsonl_path, encoding="utf-8") as f:
for line in f:
if line.strip():
try:
results.append(json.loads(line))
except json.JSONDecodeError:
continue
if results:
return {"results": results}
json_path = data_path / "benchmark_results.json"
if json_path.exists():
with open(json_path, encoding="utf-8") as f:
return json.load(f)
return {"results": []}
def load_model_pricing() -> dict[str, float]:
models_path = get_data_path() / "openrouter_models.json"
if not models_path.exists():
return {}
with open(models_path, encoding="utf-8") as f:
models = json.load(f)
return {m["id"]: float(m.get("pricing", {}).get("prompt", "0")) * 1_000_000 for m in models}
@dataclass
class AgentStats:
agent: str
agent_type: str
games: int
wins: int
win_rate: float
avg_clicks: float
avg_time: float
cost_per_game: float
def compute_agent_stats() -> list[AgentStats]:
data = load_benchmark_data()
pricing = load_model_pricing()
agent_data: dict[str, list] = {}
for r in data.get("results", []):
agent = r.get("agent", "")
agent_data.setdefault(agent, []).append(r)
stats = []
for agent, games in agent_data.items():
wins = [g for g in games if g.get("won", False)]
win_clicks = [g.get("clicks", 0) for g in wins] if wins else [0]
agent_type = "embedding" if agent.startswith("live-") else "llm"
total_tokens = sum(g.get("tokens_used", 0) for g in games)
model_id = None
if agent.startswith("llm-"):
model_part = agent.replace("llm-", "")
for mid in pricing:
short_mid = mid.split("/")[-1] if "/" in mid else mid
if model_part.startswith(short_mid) or short_mid.startswith(model_part.rstrip(".")):
model_id = mid
break
price = pricing.get(model_id, 0) if model_id else 0
total_cost = (total_tokens / 1_000_000) * price
cost_per_game = total_cost / len(games) if games else 0
stats.append(AgentStats(
agent=agent, agent_type=agent_type, games=len(games), wins=len(wins),
win_rate=len(wins) / len(games) * 100 if games else 0,
avg_clicks=sum(win_clicks) / len(win_clicks) if win_clicks else 0,
avg_time=sum(g.get("time_seconds", 0) for g in games) / len(games) if games else 0,
cost_per_game=cost_per_game,
))
stats.sort(key=lambda s: (-s.win_rate, s.avg_clicks))
return stats
# ====================
# Wikipedia Scraping
# ====================
_wiki_cache = {}
def fetch_wiki_page(title: str) -> tuple[str, list[str]]:
if title in _wiki_cache:
return _wiki_cache[title]
encoded = urllib.parse.quote(title.replace(" ", "_"))
url = f"https://en.wikipedia.org/api/rest_v1/page/html/{encoded}"
headers = {
"User-Agent": "WikiSpeedrun/1.0 (https://huggingface.co/spaces/jwlutz/wiki-speedrun; educational benchmark project)",
"Accept": "text/html,application/xhtml+xml",
}
resp = requests.get(url, headers=headers, timeout=30)
resp.raise_for_status()
soup = BeautifulSoup(resp.text, "html.parser")
for sel in [".navbox", ".infobox", ".sidebar", ".reference", ".reflist",
".mw-editsection", "figure", ".thumb", ".ambox", ".mbox", ".toc",
".hatnote", ".shortdescription", "style", "script", "base"]:
for el in soup.select(sel):
el.decompose()
links = []
for a in soup.find_all("a", href=True):
href = a.get("href", "")
# Skip red links (non-existent pages) - they have class="new"
if "new" in a.get("class", []):
continue
if href.startswith("./") and ":" not in href:
path = href.replace("./", "").split("#")[0].split("?")[0]
decoded = urllib.parse.unquote(path).replace("_", " ")
if decoded and decoded not in links:
links.append(decoded)
result = (str(soup), links)
_wiki_cache[title] = result
return result
def process_wiki_html(html: str, available_links: list[str], target: str, nav_base: str = "") -> str:
soup = BeautifulSoup(html, "html.parser")
available_lower = {l.lower(): l for l in available_links}
target_lower = target.lower()
for a in soup.find_all("a", href=True):
href = a.get("href", "")
if not href.startswith("./"):
a["class"] = a.get("class", []) + ["disabled"]
if "href" in a.attrs:
del a["href"]
continue
path = href.replace("./", "").split("#")[0].split("?")[0]
decoded = urllib.parse.unquote(path).replace("_", " ")
norm = decoded.lower()
if norm == target_lower:
a["class"] = a.get("class", []) + ["target-link"]
a["href"] = f"{nav_base}&title={urllib.parse.quote(decoded)}"
elif norm in available_lower:
a["class"] = a.get("class", []) + ["available"]
a["href"] = f"{nav_base}&title={urllib.parse.quote(available_lower[norm])}"
else:
a["class"] = a.get("class", []) + ["disabled"]
if "href" in a.attrs:
del a["href"]
return str(soup)
# ====================
# AI Agent (Embedding-based)
# ====================
_embedding_model = None
_model_ready = False
_model_warming_up = False
def get_embedding_model():
global _embedding_model
if _embedding_model is None:
try:
from sentence_transformers import SentenceTransformer
_embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
except ImportError:
return None
return _embedding_model
def warmup_model():
"""Warmup embedding model in background thread."""
global _model_ready, _model_warming_up
_model_warming_up = True
print("Warming up embedding model...")
model = get_embedding_model()
if model:
# Do a test encode to fully initialize
model.encode(["warmup test"], convert_to_numpy=True)
_model_ready = True
print("Model ready!")
else:
print("Warning: sentence-transformers not available")
_model_warming_up = False
def is_model_ready() -> bool:
return _model_ready
def ai_choose_link(available_links: list[str], target: str, path_so_far: list[str]) -> tuple[str, float]:
"""AI agent picks best link using embeddings. Returns (link, similarity_score)."""
if target in available_links:
return target, 1.0
model = get_embedding_model()
if model is None:
# Fallback: random choice
import random
return random.choice(available_links), 0.0
# Avoid revisits
visited = set(path_so_far)
candidates = [l for l in available_links if l not in visited]
if not candidates:
candidates = available_links
# Encode target + candidates
all_texts = [target] + candidates
embeddings = model.encode(all_texts, convert_to_numpy=True)
embeddings = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
target_emb = embeddings[0]
candidate_embs = embeddings[1:]
similarities = np.dot(candidate_embs, target_emb)
best_idx = int(np.argmax(similarities))
return candidates[best_idx], float(similarities[best_idx])
# ====================
# AI Spectator State
# ====================
@dataclass
class AIGame:
start: str
target: str
path: list[str] = field(default_factory=list)
links: list[str] = field(default_factory=list)
html: str = ""
won: bool = False
start_time: float = 0
last_choice: str = ""
last_score: float = 0
pending_choice: str = ""
pending_score: float = 0
error: str = ""
running: bool = False
_ai_games: dict[str, AIGame] = {}
def ai_game_choose(game_id: str) -> tuple[str, float]:
"""AI chooses next link but doesn't navigate yet. Returns (choice, score)."""
game = _ai_games.get(game_id)
if not game or not game.running:
return "", 0.0
current = game.path[-1]
# Check win
if current.lower() == game.target.lower():
game.won = True
game.running = False
return "", 0.0
# Choose next link
choice, score = ai_choose_link(game.links, game.target, game.path)
game.pending_choice = choice
game.pending_score = score
return choice, score
def ai_game_advance(game_id: str):
"""Navigate to the pending choice. Skips invalid links and retries."""
game = _ai_games.get(game_id)
if not game or not game.running or not game.pending_choice:
return
choice = game.pending_choice
game.last_choice = choice
game.last_score = game.pending_score
game.pending_choice = ""
game.pending_score = 0
# Navigate - retry on 404
max_retries = 5
for attempt in range(max_retries):
try:
html, links = fetch_wiki_page(choice)
game.path.append(choice)
game.html = html
game.links = links
# Check win after navigate
if choice.lower() == game.target.lower():
game.won = True
game.running = False
return # Success
except Exception as e:
error_str = str(e)
if "404" in error_str or "Not Found" in error_str:
# Remove invalid link and choose another
if choice in game.links:
game.links.remove(choice)
if not game.links:
game.error = "No valid links remaining"
game.running = False
return
# Choose a new link
choice, score = ai_choose_link(game.links, game.target, game.path)
game.last_choice = choice
game.last_score = score
else:
game.error = error_str
game.running = False
return
game.error = f"Failed after {max_retries} retries"
game.running = False
# ====================
# HTML Templates
# ====================
BASE_TEMPLATE = """
Wikipedia Speedrun
{{ content | safe }}
"""
# ====================
# Routes - Dashboard
# ====================
@app.route("/")
def dashboard():
"""Dashboard with full Plotly charts."""
NAV_BAR = ''
try:
from ui.components.data_loader import (
get_agent_summaries,
get_results,
get_agent_cost_summaries,
get_failure_analysis,
get_dashboard_stats,
get_problems,
)
from ui.components.charts import (
create_win_rate_chart,
create_clicks_boxplot,
create_time_scatter,
create_difficulty_heatmap,
create_pareto_chart,
create_leaderboard_table,
create_failure_table,
)
# Load data
stats = get_dashboard_stats()
summaries = get_agent_summaries()
cost_summaries = get_agent_cost_summaries()
results = get_results()
problems = get_problems()
failures = get_failure_analysis()
if not summaries:
content = f"""
No Benchmark Data
Add results to data/benchmark_results.jsonl to see the dashboard.
Play Now
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=0)
# Generate charts as HTML
pareto_html = create_pareto_chart(cost_summaries).to_html(full_html=False, include_plotlyjs="cdn")
win_rate_html = create_win_rate_chart(summaries).to_html(full_html=False, include_plotlyjs=False)
clicks_html = create_clicks_boxplot(results).to_html(full_html=False, include_plotlyjs=False)
time_html = create_time_scatter(summaries).to_html(full_html=False, include_plotlyjs=False)
# Try difficulty heatmap (may fail if no difficulty data)
try:
difficulty_html = create_difficulty_heatmap(results, problems).to_html(full_html=False, include_plotlyjs=False)
except Exception:
difficulty_html = "No difficulty data available.
"
# Generate tables
leaderboard_html = create_leaderboard_table(cost_summaries)
failure_html = create_failure_table(failures)
# Format stats for display
cheapest_cost = f"${stats['cheapest_good_cost']:.4f}" if stats['cheapest_good_cost'] > 0 else "FREE"
content = f"""
{stats['total_games']}
Total Games
{stats['agents_tested']}
Agents Tested
{stats['best_agent'].replace('llm-','').replace('live-','')}
Best Agent ({stats['best_win_rate']:.0f}%)
{stats['cheapest_good_agent'].replace('llm-','').replace('live-','')}
Best Value ({cheapest_cost}/game)
Cost vs Performance
Points on the yellow frontier represent optimal cost-performance tradeoffs. Green = embedding models, Blue = LLM models.
{pareto_html}
Agent Leaderboard
{leaderboard_html}
Win Rate by Agent
{win_rate_html}
Click Distribution (Wins Only)
{clicks_html}
Efficiency: Clicks vs Time
{time_html}
Win Rate by Difficulty
{difficulty_html}
Hardest Problems (Most Failures)
All failures are timeouts (25 clicks max). These problems defeated the most agents.
{failure_html}
Think You Can Beat the AI?
Try the Wikipedia Speedrun yourself and see how you compare to our AI agents.
Play Now
Watch AI
"""
except Exception as e:
content = f"""
Error Loading Dashboard
{e}
Make sure benchmark data exists in data/benchmark_results.jsonl
Play Now
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=0)
# ====================
# Routes - Play
# ====================
@app.route("/play")
def play_start():
content = """
Start a New Game
Navigate from one Wikipedia article to another using only the links on each page.
Difficulty Presets
Easy: Cat → DogMedium: Coffee → MinecraftHard: Lamprey → Costco hot dog
Loading Wikipedia pages...
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=0)
@app.route("/start", methods=["POST"])
def start_game():
start = request.form.get("start", "").strip()
target = request.form.get("target", "").strip()
visualize = "1" if request.form.get("visualize") == "1" else "0"
if not start or not target:
return redirect(url_for("play_start"))
try:
fetch_wiki_page(start)
fetch_wiki_page(target)
except Exception as e:
return render_template_string(BASE_TEMPLATE, content=f"""
Error Could not find article: {e}
Back Error: {e}
"
links = []
clicks = len(path) - 1
# Check if target is in available links
target_available = target.lower() in [l.lower() for l in links]
# Build navigation URL with state (for URL-based state passing)
nav_base = f"/navigate?target={urllib.parse.quote(target)}&path={urllib.parse.quote(path_str)}&t={start_time}&viz={'1' if visualize else '0'}"
if visualize:
# Full visual mode with Wikipedia content
processed = process_wiki_html(html, links, target, nav_base)
content = f"""
Target: {target}
Clicks: {clicks}
Time: 0.0s
Give Up
📍 {' → '.join(path)}
Green = Click to navigate ({len(links)} links)
Gold = TARGET (click to win!)
{current}
{processed}
"""
else:
# Minimal mode - just timer, path, and link dropdown
sorted_links = sorted(links, key=str.lower)
options = "".join(f'{l} ' for l in sorted_links)
target_btn = ""
if target_available:
target_btn = f'🎯 Click to WIN: {target} '
content = f"""
Current: {current}
📍 {' → '.join(path)}
{target_btn}
Choose next article ({len(links)} links available):
-- Select a link --
{options}
Go
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=start_time)
@app.route("/navigate")
def navigate():
# Read state from URL params
title = request.args.get("title", "")
target = request.args.get("target", "")
path_str = request.args.get("path", "")
start_time = request.args.get("t", "0")
visualize = request.args.get("viz", "1")
if not title or not target:
return redirect(url_for("play_start"))
# Append new title to path
new_path = f"{path_str}|{title}" if path_str else title
return redirect(url_for("game", start=request.args.get("start", ""),
target=target, current=title, path=new_path,
t=start_time, viz=visualize))
@app.route("/win")
def win_page():
# Read from URL params
clicks = request.args.get("clicks", "0")
elapsed = request.args.get("time", "0")
path_str = request.args.get("path", "")
path = path_str.split("|") if path_str else []
content = f"""
You Won!
{clicks} clicks in {elapsed} seconds
Your path:
Play Again
Watch AI
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=0)
@app.route("/give-up")
def give_up():
session.clear()
return redirect(url_for("play_start"))
# ====================
# Routes - Watch AI
# ====================
@app.route("/watch")
def watch_start():
model_status = ""
if not is_model_ready():
model_status = """
Warming up model... The embedding model is loading. This may take a moment on first run.
"""
content = f"""
Watch an AI Agent Play
Watch a live embedding agent navigate Wikipedia in real-time!
{model_status}
About the AI Agent
The AI uses sentence-transformers to compute semantic similarity between article titles and the target.
At each step, it chooses the link that is most similar to the target article's title.
Model: all-MiniLM-L6-v2 (384 dimensions)
Loading Wikipedia pages...
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=0)
@app.route("/watch/start", methods=["POST"])
def watch_start_game():
start = request.form.get("start", "").strip()
target = request.form.get("target", "").strip()
if not start or not target:
return redirect(url_for("watch_start"))
# Wait for model to be ready
if not is_model_ready():
return render_template_string(BASE_TEMPLATE, content="""
Model Not Ready Please wait for the model to finish warming up.
Back Error Could not find article: {e}
Back
AI's Analysis: Chose "{game.last_choice} " (similarity: {game.last_score:.3f})
"""
# URL-encode the game_id for use in URLs
gid_encoded = urllib.parse.quote(game_id, safe='')
content = f"""
AI Playing
Target: {game.target}
Clicks: {clicks}
Time: {elapsed:.1f}s
Stop
{' -> '.join(game.path)}
{ai_info}
Green = Available links ({len(game.links)})
Gold = TARGET
{current}
{processed}
"""
return render_template_string(BASE_TEMPLATE, content=content, start_time=0)
@app.route("/watch/step")
def watch_step():
"""AI chooses a link, returns it for scroll animation."""
# Read game_id from URL params (HF Spaces doesn't preserve cookies)
game_id = request.args.get("gid", "")
if not game_id or game_id not in _ai_games:
return jsonify({"error": "no game"}), 404
game = _ai_games[game_id]
if game.won or game.error:
return jsonify({"done": True})
choice, score = ai_game_choose(game_id)
return jsonify({"choice": choice, "score": score})
@app.route("/watch/advance")
def watch_advance():
"""Navigate to the pending choice after animation."""
# Read game_id from URL params (HF Spaces doesn't preserve cookies)
game_id = request.args.get("gid", "")
if game_id and game_id in _ai_games:
ai_game_advance(game_id)
return "", 204
@app.route("/watch/stop")
def watch_stop():
# Read game_id from URL params (HF Spaces doesn't preserve cookies)
game_id = request.args.get("gid", "")
if game_id and game_id in _ai_games:
del _ai_games[game_id]
return redirect(url_for("watch_start"))
@app.route("/watch/status")
def watch_status():
"""Return model warmup status."""
return jsonify({"ready": is_model_ready(), "warming_up": _model_warming_up})
# ====================
# Main
# ====================
if __name__ == "__main__":
port = int(os.environ.get("PORT", 7860))
print(f"\n=== Wikipedia Speedrun ===")
print(f"Open http://localhost:{port} in your browser\n")
# Start model warmup in background thread
warmup_thread = threading.Thread(target=warmup_model, daemon=True)
warmup_thread.start()
app.run(host="0.0.0.0", port=port, debug=False)