evals/matched_public_capability_lm_eval_0_4_12.json

{
  "protocol": {
    "backend": "hf-multimodal",
    "batch_size": 8,
    "chat_template": false,
    "dtype": "bfloat16",
    "fewshot": {
      "gsm8k": 5,
      "hellaswag": 0,
      "mmlu": 0,
      "truthfulqa_mc2": 0,
      "winogrande": 0
    },
    "harness": {
      "name": "lm-evaluation-harness",
      "raw_result_git_hash": {
        "candidate": "8eebb67",
        "note": "This per-result field is derived from the launch working directory, not the installed evaluator package. The upstream value is unavailable and is not inferred.",
        "untouched_upstream": null,
        "used_for_protocol_matching": false
      },
      "version": "0.4.12"
    },
    "limit": null,
    "seeds": {
      "fewshot_seed": 1234,
      "numpy_seed": 1234,
      "random_seed": 0,
      "torch_seed": 1234
    },
    "system_instruction": false,
    "task_versions": {
      "gsm8k": 3.0,
      "hellaswag": 1.0,
      "mmlu": "2",
      "truthfulqa_mc2": 3.0,
      "winogrande": 1.0
    },
    "transformers_version": "5.14.1"
  },
  "schema_version": 1,
  "scope": "matched_public_capability_evaluation",
  "sources": {
    "candidate_result_sha256": "f963936d03b9cf28d511e7be5e07b9b9af5d9b0be0465ab07bbc24acdaaad512",
    "untouched_upstream_result_sha256": "28168fcb1a6105c32fffba5b9d5cd6250b45fbd2da8b89b864be0dcfef944ede"
  },
  "tasks": {
    "gsm8k": {
      "interpretation": "Generated-answer exact match with strict extraction is primary; flexible extraction is secondary. Higher is better.",
      "metrics": {
        "exact_match,flexible-extract": {
          "candidate": {
            "stderr": 0.01291940810865643,
            "value": 0.6732373009855952
          },
          "delta_candidate_minus_upstream": -0.0037907505686125553,
          "role": "secondary",
          "untouched_upstream": {
            "stderr": 0.012880360794851822,
            "value": 0.6770280515542078
          }
        },
        "exact_match,strict-match": {
          "candidate": {
            "stderr": 0.012714401009923647,
            "value": 0.6921910538286581
          },
          "delta_candidate_minus_upstream": -0.0007581501137224889,
          "role": "primary",
          "untouched_upstream": {
            "stderr": 0.012705685723131698,
            "value": 0.6929492039423806
          }
        }
      },
      "sample_count": 1319
    },
    "hellaswag": {
      "interpretation": "Length-normalized multiple-choice accuracy is primary; raw accuracy is secondary. Higher is better.",
      "metrics": {
        "acc,none": {
          "candidate": {
            "stderr": 0.00458670271601444,
            "value": 0.3031268671579367
          },
          "delta_candidate_minus_upstream": 0.00019916351324439363,
          "role": "secondary",
          "untouched_upstream": {
            "stderr": 0.004585850835623589,
            "value": 0.3029277036446923
          }
        },
        "acc_norm,none": {
          "candidate": {
            "stderr": 0.004760354191370993,
            "value": 0.35012945628360886
          },
          "delta_candidate_minus_upstream": 0.0004979087831109563,
          "role": "primary",
          "untouched_upstream": {
            "stderr": 0.004758790172436568,
            "value": 0.3496315475004979
          }
        }
      },
      "sample_count": 10042
    },
    "mmlu": {
      "interpretation": "Accuracy over all MMLU subjects; higher is better.",
      "metrics": {
        "acc,none": {
          "candidate": {
            "stderr": 0.004086530540962004,
            "value": 0.40542657741062527
          },
          "delta_candidate_minus_upstream": -0.0030622418458908784,
          "role": "primary",
          "untouched_upstream": {
            "stderr": 0.004092076960230109,
            "value": 0.40848881925651614
          }
        }
      },
      "sample_count": 14042
    },
    "truthfulqa_mc2": {
      "interpretation": "Mean normalized probability mass assigned to true answer options; higher is better.",
      "metrics": {
        "acc,none": {
          "candidate": {
            "stderr": 0.01657668864641618,
            "value": 0.46375219954067515
          },
          "delta_candidate_minus_upstream": -0.01264179784361441,
          "role": "primary",
          "untouched_upstream": {
            "stderr": 0.016538783742581047,
            "value": 0.47639399738428956
          }
        }
      },
      "sample_count": 817
    },
    "winogrande": {
      "interpretation": "Multiple-choice accuracy; higher is better.",
      "metrics": {
        "acc,none": {
          "candidate": {
            "stderr": 0.014047122916440271,
            "value": 0.4861878453038674
          },
          "delta_candidate_minus_upstream": 0.0007892659826361226,
          "role": "primary",
          "untouched_upstream": {
            "stderr": 0.014046492383275955,
            "value": 0.48539857932123126
          }
        }
      },
      "sample_count": 1267
    }
  }
}