{
  "checkpoint": "runs\\sovereign_specialists\\q-retriever-50m-v10\\final.pt",
  "holdout": "data\\model_suite\\q_retriever\\holdout\\q_retriever_v7_holdout.jsonl",
  "scorer": "exact_match",
  "n": 30,
  "mean": 1.0,
  "ci_lower": 1.0,
  "per_row": [
    {
      "i": 0,
      "score": 1.0,
      "user": "Q: capital of Germany\n\nPASSAGE:\nBerlin is the capital of Germany.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 1,
      "score": 1.0,
      "user": "Q: capital of Germany\n\nPASSAGE:\nParis is the capital of France.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 2,
      "score": 1.0,
      "user": "Q: freezing point of water\n\nPASSAGE:\nWater freezes at 0 degrees Celsius.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 3,
      "score": 1.0,
      "user": "Q: freezing point of water\n\nPASSAGE:\nWater boils at 100 C.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 4,
      "score": 1.0,
      "user": "Q: net income amount\n\nPASSAGE:\nNet income for the period was $12 billion, up from $11 billion prior year.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 5,
      "score": 1.0,
      "user": "Q: net income amount\n\nPASSAGE:\nTotal revenue was $54 billion in the quarter.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 6,
      "score": 1.0,
      "user": "Q: how do I list files in a directory in python\n\nPASSAGE:\nUse `os.listdir(path)` or `pathlib.Path(path).iterdir()` to li",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 7,
      "score": 1.0,
      "user": "Q: how do I list files in a directory in python\n\nPASSAGE:\nls is the Unix command.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 8,
      "score": 1.0,
      "user": "Q: what was free cash flow\n\nPASSAGE:\nFree cash flow for the year was 30 billion dollars.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 9,
      "score": 1.0,
      "user": "Q: what was free cash flow\n\nPASSAGE:\nOperating cash flow was $35 billion.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 10,
      "score": 1.0,
      "user": "Q: Python venv\n\nPASSAGE:\nUse `python -m venv .venv` to create a new isolated Python environment.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 11,
      "score": 1.0,
      "user": "Q: Python venv\n\nPASSAGE:\n`conda create -n myenv` makes a Conda environment.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 12,
      "score": 1.0,
      "user": "Q: capital of Germany\n\nPASSAGE:\nFree cash flow for the year was $30 billion.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 13,
      "score": 1.0,
      "user": "Q: capital of Germany\n\nPASSAGE:\nUse `pip install <package>` to install.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 14,
      "score": 1.0,
      "user": "Q: what was net income\n\nPASSAGE:\nBerlin is the capital of Germany.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 15,
      "score": 1.0,
      "user": "Q: what was net income\n\nPASSAGE:\nThe speed of light is about 3e8 m/s.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 16,
      "score": 1.0,
      "user": "Q: how to git commit\n\nPASSAGE:\nThe Nile is the world's longest river.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 17,
      "score": 1.0,
      "user": "Q: speed of light\n\nPASSAGE:\nThe speed of light in vacuum is approximately 299,792,458 meters per second.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 18,
      "score": 1.0,
      "user": "Q: speed of light\n\nPASSAGE:\nThe speed of sound in air is about 343 m/s.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 19,
      "score": 1.0,
      "user": "Q: Pythagorean theorem\n\nPASSAGE:\nThe Pythagorean theorem states that a^2 + b^2 = c^2 for a right triangle's sides.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 20,
      "score": 1.0,
      "user": "Q: Pythagorean theorem\n\nPASSAGE:\nHeron's formula gives area from side lengths.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 21,
      "score": 1.0,
      "user": "Q: when did World War II end\n\nPASSAGE:\nWorld War II ended in 1945, with Japan's surrender in September.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 22,
      "score": 1.0,
      "user": "Q: when did World War II end\n\nPASSAGE:\nWorld War I ended in 1918.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 23,
      "score": 1.0,
      "user": "Q: capital of Japan\n\nPASSAGE:\nTokyo is the capital of Japan.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 24,
      "score": 1.0,
      "user": "Q: capital of Japan\n\nPASSAGE:\nKyoto was the former capital of Japan before 1868.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 25,
      "score": 1.0,
      "user": "Q: CPR compression rate\n\nPASSAGE:\nPerform chest compressions at 100-120 per minute, about 2 inches deep.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 26,
      "score": 1.0,
      "user": "Q: CPR compression rate\n\nPASSAGE:\nHeimlich maneuver is for choking, not cardiac arrest.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 27,
      "score": 1.0,
      "user": "Q: EPS\n\nPASSAGE:\nDiluted EPS for the quarter was $2.40.",
      "pred": "1",
      "gold": "1"
    },
    {
      "i": 28,
      "score": 1.0,
      "user": "Q: EPS\n\nPASSAGE:\nDividend per share was $1.20.",
      "pred": "0",
      "gold": "0"
    },
    {
      "i": 29,
      "score": 1.0,
      "user": "Q: how to install Python on Windows\n\nPASSAGE:\nTo install Node.js on Windows, download the LTS installer from nodejs.org.",
      "pred": "0",
      "gold": "0"
    }
  ]
}